Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-22153 Fix the flaky TestRestartCluster

This commit is contained in:
Duo Zhang 2019-04-08 10:26:28 +08:00 committed by Apache9
parent b2587c874b
commit a871d3139a
1 changed files with 17 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
View File

@ -27,7 +27,6 @@ import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
@ -46,7 +45,6 @@ import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.JVMClusterUtil;
import org.apache.hadoop.hbase.util.Threads;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@ -86,21 +84,27 @@ public class TestRestartCluster {
splitWALCoordinatedByZK); splitWALCoordinatedByZK);
} }
@After public void tearDown() throws Exception { @After
public void tearDown() throws Exception {
UTIL.shutdownMiniCluster(); UTIL.shutdownMiniCluster();
} }
private ServerStateNode getServerStateNode(ServerName serverName) {
return UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
.getServerNode(serverName);
}
@Test @Test
public void testClusterRestartFailOver() throws Exception { public void testClusterRestartFailOver() throws Exception {
UTIL.startMiniCluster(3); UTIL.startMiniCluster(3);
UTIL.waitFor(60000, () -> UTIL.getMiniHBaseCluster().getMaster().isInitialized()); UTIL.waitFor(60000, () -> UTIL.getMiniHBaseCluster().getMaster().isInitialized());
//wait for all SCPs finished // wait for all SCPs finished
UTIL.waitFor(20000, () -> UTIL.getHBaseCluster().getMaster().getProcedures().stream() UTIL.waitFor(60000, () -> UTIL.getHBaseCluster().getMaster().getProcedures().stream()
.noneMatch(p -> p instanceof ServerCrashProcedure)); .noneMatch(p -> p instanceof ServerCrashProcedure));
TableName tableName = TABLES[0]; TableName tableName = TABLES[0];
ServerName testServer = UTIL.getHBaseCluster().getRegionServer(0).getServerName(); ServerName testServer = UTIL.getHBaseCluster().getRegionServer(0).getServerName();
ServerStateNode serverNode = UTIL.getHBaseCluster().getMaster().getAssignmentManager() UTIL.waitFor(10000, () -> getServerStateNode(testServer) != null);
.getRegionStates().getServerNode(testServer); ServerStateNode serverNode = getServerStateNode(testServer);
Assert.assertNotNull(serverNode); Assert.assertNotNull(serverNode);
Assert.assertTrue("serverNode should be ONLINE when cluster runs normally", Assert.assertTrue("serverNode should be ONLINE when cluster runs normally",
serverNode.isInState(ServerState.ONLINE)); serverNode.isInState(ServerState.ONLINE));
@ -124,7 +128,7 @@ public class TestRestartCluster {
Assert.assertNotNull("serverNode should not be null when restart whole cluster", serverNode); Assert.assertNotNull("serverNode should not be null when restart whole cluster", serverNode);
Assert.assertFalse(serverNode.isInState(ServerState.ONLINE)); Assert.assertFalse(serverNode.isInState(ServerState.ONLINE));
LOG.info("start to find the procedure of SCP for the severName we choose"); LOG.info("start to find the procedure of SCP for the severName we choose");
UTIL.waitFor(20000, UTIL.waitFor(60000,
() -> UTIL.getHBaseCluster().getMaster().getProcedures().stream() () -> UTIL.getHBaseCluster().getMaster().getProcedures().stream()
.anyMatch(procedure -> (procedure instanceof ServerCrashProcedure) .anyMatch(procedure -> (procedure instanceof ServerCrashProcedure)
&& ((ServerCrashProcedure) procedure).getServerName().equals(testServer))); && ((ServerCrashProcedure) procedure).getServerName().equals(testServer)));
@ -133,11 +137,11 @@ public class TestRestartCluster {
LOG.info("start to submit the SCP for the same serverName {} which should fail", testServer); LOG.info("start to submit the SCP for the same serverName {} which should fail", testServer);
Assert.assertFalse( Assert.assertFalse(
UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(testServer)); UTIL.getHBaseCluster().getMaster().getServerManager().expireServer(testServer));
Procedure procedure = UTIL.getHBaseCluster().getMaster().getProcedures().stream() Procedure<?> procedure = UTIL.getHBaseCluster().getMaster().getProcedures().stream()
.filter(p -> (p instanceof ServerCrashProcedure) .filter(p -> (p instanceof ServerCrashProcedure)
&& ((ServerCrashProcedure) p).getServerName().equals(testServer)) && ((ServerCrashProcedure) p).getServerName().equals(testServer))
.findAny().get(); .findAny().get();
UTIL.waitFor(20000, () -> procedure.isFinished()); UTIL.waitFor(60000, () -> procedure.isFinished());
LOG.info("even when the SCP is finished, the duplicate SCP should not be scheduled for {}", LOG.info("even when the SCP is finished, the duplicate SCP should not be scheduled for {}",
testServer); testServer);
Assert.assertFalse( Assert.assertFalse(
@ -150,9 +154,7 @@ public class TestRestartCluster {
@Test @Test
public void testClusterRestart() throws Exception { public void testClusterRestart() throws Exception {
UTIL.startMiniCluster(3); UTIL.startMiniCluster(3);
while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) { UTIL.waitFor(60000, () -> UTIL.getMiniHBaseCluster().getMaster().isInitialized());
Threads.sleep(1);
}
LOG.info("\n\nCreating tables"); LOG.info("\n\nCreating tables");
for(TableName TABLE : TABLES) { for(TableName TABLE : TABLES) {
UTIL.createTable(TABLE, FAMILY); UTIL.createTable(TABLE, FAMILY);
@ -319,7 +321,7 @@ public class TestRestartCluster {
} }
@Parameterized.Parameters @Parameterized.Parameters
public static Collection coordinatedByZK() { public static Collection<?> coordinatedByZK() {
return Arrays.asList(false, true); return Arrays.asList(false, true);
} }
} }