From e1cd10b002a07a35aa7666fcfbd01b54cfcff1bf Mon Sep 17 00:00:00 2001
From: zhangduo <zhangduo@apache.org>
Date: Sun, 4 Feb 2018 12:58:35 +0800
Subject: [PATCH] HBASE-19927 TestFullLogReconstruction flakey

---
 .../hadoop/hbase/HBaseTestingUtility.java     |  4 +-
 .../hbase/TestFullLogReconstruction.java      | 39 +++++++++++--------
 2 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 6007b0762a8..ecd2fa5c436 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2634,12 +2634,12 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
   /**
    * Expire a region server's session
    * @param index which RS
-   * @throws Exception
    */
-  public void expireRegionServerSession(int index) throws Exception {
+  public HRegionServer expireRegionServerSession(int index) throws Exception {
     HRegionServer rs = getMiniHBaseCluster().getRegionServer(index);
     expireSession(rs.getZooKeeper(), false);
     decrementMinRegionServerCount();
+    return rs;
   }
 
   private void decrementMinRegionServerCount() {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
index da3bf2c3982..13c616f7e2c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestFullLogReconstruction.java
@@ -20,7 +20,9 @@ package org.apache.hadoop.hbase;
 import static org.junit.Assert.assertEquals;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -37,15 +39,11 @@ public class TestFullLogReconstruction {
   public static final HBaseClassTestRule CLASS_RULE =
       HBaseClassTestRule.forClass(TestFullLogReconstruction.class);
 
-  private final static HBaseTestingUtility
-      TEST_UTIL = new HBaseTestingUtility();
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
 
   private final static TableName TABLE_NAME = TableName.valueOf("tabletest");
   private final static byte[] FAMILY = Bytes.toBytes("family");
 
-  /**
-   * @throws java.lang.Exception
-   */
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     Configuration c = TEST_UTIL.getConfiguration();
@@ -60,22 +58,17 @@ public class TestFullLogReconstruction {
     TEST_UTIL.startMiniCluster(3);
   }
 
-  /**
-   * @throws java.lang.Exception
-   */
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
   }
 
   /**
-   * Test the whole reconstruction loop. Build a table with regions aaa to zzz
-   * and load every one of them multiple times with the same date and do a flush
-   * at some point. Kill one of the region servers and scan the table. We should
-   * see all the rows.
-   * @throws Exception
+   * Test the whole reconstruction loop. Build a table with regions aaa to zzz and load every one of
+   * them multiple times with the same date and do a flush at some point. Kill one of the region
+   * servers and scan the table. We should see all the rows.
    */
-  @Test (timeout=300000)
+  @Test
   public void testReconstruction() throws Exception {
     Table table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, FAMILY);
 
@@ -85,11 +78,25 @@ public class TestFullLogReconstruction {
 
     assertEquals(initialCount, count);
 
-    for(int i = 0; i < 4; i++) {
+    for (int i = 0; i < 4; i++) {
       TEST_UTIL.loadTable(table, FAMILY);
     }
+    HRegionServer rs = TEST_UTIL.expireRegionServerSession(0);
+    // make sure that the RS is fully down before reading, so that we will read the data from other
+    // RSes.
+    TEST_UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {
+
+      @Override
+      public boolean evaluate() throws Exception {
+        return !rs.isAlive();
+      }
+
+      @Override
+      public String explainFailure() throws Exception {
+        return rs + " is still alive";
+      }
+    });
 
-    TEST_UTIL.expireRegionServerSession(0);
     int newCount = TEST_UTIL.countRows(table);
     assertEquals(count, newCount);
     table.close();