From 8a8c60889cf67b581d7adb4245e0bcc02cdfdc93 Mon Sep 17 00:00:00 2001 From: ChiaPing Tsai Date: Sun, 9 Oct 2016 16:52:54 -0700 Subject: [PATCH] HBASE-16794 TestDispatchMergingRegionsProcedure#testMergeRegionsConcurrently is flaky Signed-off-by: Matteo Bertozzi --- .../regionserver/CompactSplitThread.java | 5 +++ .../TestDispatchMergingRegionsProcedure.java | 45 ++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java index c1f82b98ade..a454f0e8827 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java @@ -723,6 +723,11 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi return compactionThroughputController; } + @VisibleForTesting + public long getCompletedMergeTaskCount() { + return mergePool.getCompletedTaskCount(); + } + @VisibleForTesting /** * Shutdown the long compaction thread pool. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java index 601f22fb58b..a7dd4a8151c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDispatchMergingRegionsProcedure.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hbase.master.procedure; +import java.io.IOException; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -36,6 +38,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.D import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -125,20 +128,21 @@ public class TestDispatchMergingRegionsProcedure { regionsToMerge[0] = tableRegions.get(0); regionsToMerge[1] = tableRegions.get(1); + final int initCompletedTaskCount = countOfCompletedMergeTaskCount(); long procId = procExec.submitProcedure(new DispatchMergingRegionsProcedure( procExec.getEnvironment(), tableName, regionsToMerge, true)); ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); - assertRegionCount(tableName, 2); + assertRegionCount(tableName, 2, 1, initCompletedTaskCount); } /** * This tests two concurrent region merges */ - @Test(timeout=90000) + @Test(timeout=60000) public void testMergeRegionsConcurrently() throws Exception { - final TableName tableName = TableName.valueOf("testMergeTwoRegions"); + final TableName tableName = TableName.valueOf("testMergeRegionsConcurrently"); final ProcedureExecutor procExec = getMasterProcedureExecutor(); List tableRegions = createTable(tableName, 4); @@ -150,6 +154,7 @@ public class TestDispatchMergingRegionsProcedure { regionsToMerge2[0] = tableRegions.get(2); regionsToMerge2[1] = tableRegions.get(3); + final int initCompletedTaskCount = countOfCompletedMergeTaskCount(); long procId1 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( procExec.getEnvironment(), tableName, regionsToMerge1, true)); long procId2 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( @@ -158,8 +163,28 @@ public class TestDispatchMergingRegionsProcedure { ProcedureTestingUtility.waitProcedure(procExec, procId2); ProcedureTestingUtility.assertProcNotFailed(procExec, procId1); ProcedureTestingUtility.assertProcNotFailed(procExec, procId2); + assertRegionCount(tableName, 2, 2, initCompletedTaskCount); + } - assertRegionCount(tableName, 2); + private void waitForCompletedMergeTask(int expectedTaskCount, int initCompletedTaskCount) + throws IOException, InterruptedException { + while (true) { + long currentCompletedTaskCount = countOfCompletedMergeTaskCount() - initCompletedTaskCount; + if (currentCompletedTaskCount == expectedTaskCount) { + return; + } + LOG.info("There are " + (expectedTaskCount - currentCompletedTaskCount) + + " merge requests are not completed, wait 100 ms"); + TimeUnit.MILLISECONDS.sleep(100); + } + } + + private static int countOfCompletedMergeTaskCount() { + int completedTaskCount = 0; + for (RegionServerThread server : UTIL.getMiniHBaseCluster().getRegionServerThreads()) { + completedTaskCount += server.getRegionServer().getCompactSplitThread().getCompletedMergeTaskCount(); + } + return completedTaskCount; } @Test(timeout=60000) @@ -173,6 +198,7 @@ public class TestDispatchMergingRegionsProcedure { regionsToMerge[0] = tableRegions.get(0); regionsToMerge[1] = tableRegions.get(1); + final int initCompletedTaskCount = countOfCompletedMergeTaskCount(); long procId1 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( procExec.getEnvironment(), tableName, regionsToMerge, true), nonceGroup, nonce); long procId2 = procExec.submitProcedure(new DispatchMergingRegionsProcedure( @@ -185,7 +211,7 @@ public class TestDispatchMergingRegionsProcedure { ProcedureTestingUtility.waitProcedure(procExec, procId2); ProcedureTestingUtility.assertProcNotFailed(procExec, procId2); - assertRegionCount(tableName, 2); + assertRegionCount(tableName, 2, 1, initCompletedTaskCount); } @Test(timeout=60000) @@ -202,6 +228,7 @@ public class TestDispatchMergingRegionsProcedure { regionsToMerge[0] = tableRegions.get(0); regionsToMerge[1] = tableRegions.get(1); + final int initCompletedTaskCount = countOfCompletedMergeTaskCount(); long procId = procExec.submitProcedure( new DispatchMergingRegionsProcedure( procExec.getEnvironment(), tableName, regionsToMerge, true)); @@ -211,7 +238,7 @@ public class TestDispatchMergingRegionsProcedure { MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); - assertRegionCount(tableName, 2); + assertRegionCount(tableName, 2, 1, initCompletedTaskCount); } @Test(timeout = 60000) @@ -256,6 +283,12 @@ public class TestDispatchMergingRegionsProcedure { return tableRegions; } + public List assertRegionCount(final TableName tableName, final int nregions, + int expectedTaskCount, int initCompletedTaskCount) throws Exception { + waitForCompletedMergeTask(expectedTaskCount, initCompletedTaskCount); + return assertRegionCount(tableName, nregions); + } + private ProcedureExecutor getMasterProcedureExecutor() { return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); }