HBASE-19927 TestFullLogReconstruction flakey

hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java

@@ -2634,12 +2634,12 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
   /**
    * Expire a region server's session
    * @param index which RS
-   * @throws Exception
    */
-  public void expireRegionServerSession(int index) throws Exception {
+  public HRegionServer expireRegionServerSession(int index) throws Exception {
     HRegionServer rs = getMiniHBaseCluster().getRegionServer(index);
     expireSession(rs.getZooKeeper(), false);
     decrementMinRegionServerCount();
+    return rs;
   }
 
   private void decrementMinRegionServerCount() {

hbase-server/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java

@@ -20,7 +20,9 @@ package org.apache.hadoop.hbase;
 import static org.junit.Assert.assertEquals;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -37,15 +39,11 @@ public class TestFullLogReconstruction {
   public static final HBaseClassTestRule CLASS_RULE =
       HBaseClassTestRule.forClass(TestFullLogReconstruction.class);
 
-  private final static HBaseTestingUtility
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-      TEST_UTIL = new HBaseTestingUtility();
 
   private final static TableName TABLE_NAME = TableName.valueOf("tabletest");
   private final static byte[] FAMILY = Bytes.toBytes("family");
 
-  /**
-   * @throws java.lang.Exception
-   */
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     Configuration c = TEST_UTIL.getConfiguration();
@@ -60,22 +58,17 @@ public class TestFullLogReconstruction {
     TEST_UTIL.startMiniCluster(3);
   }
 
-  /**
-   * @throws java.lang.Exception
-   */
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
   /**
-   * Test the whole reconstruction loop. Build a table with regions aaa to zzz
+   * Test the whole reconstruction loop. Build a table with regions aaa to zzz and load every one of
-   * and load every one of them multiple times with the same date and do a flush
+   * them multiple times with the same date and do a flush at some point. Kill one of the region
-   * at some point. Kill one of the region servers and scan the table. We should
+   * servers and scan the table. We should see all the rows.
-   * see all the rows.
-   * @throws Exception
    */
-  @Test (timeout=300000)
+  @Test
   public void testReconstruction() throws Exception {
     Table table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, FAMILY);
 
@@ -85,11 +78,25 @@ public class TestFullLogReconstruction {
 
     assertEquals(initialCount, count);
 
-    for(int i = 0; i < 4; i++) {
+    for (int i = 0; i < 4; i++) {
       TEST_UTIL.loadTable(table, FAMILY);
     }
+    HRegionServer rs = TEST_UTIL.expireRegionServerSession(0);
+    // make sure that the RS is fully down before reading, so that we will read the data from other
+    // RSes.
+    TEST_UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {
+
+      @Override
+      public boolean evaluate() throws Exception {
+        return !rs.isAlive();
+      }
+
+      @Override
+      public String explainFailure() throws Exception {
+        return rs + " is still alive";
+      }
+    });
 
-    TEST_UTIL.expireRegionServerSession(0);
     int newCount = TEST_UTIL.countRows(table);
     assertEquals(count, newCount);
     table.close();