HBASE-8934 Fix bunch of flaky tests

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1502627 13f79535-47bb-0310-9956-ffa450edef68

hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java

@@ -1071,6 +1071,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
       desc.addFamily(hcd);
     }
     getHBaseAdmin().createTable(desc, startKey, endKey, numRegions);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(getConfiguration(), tableName);
   }
 
@@ -1095,6 +1097,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
       desc.addFamily(hcd);
     }
     getHBaseAdmin().createTable(desc);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(c, tableName);
   }
 
@@ -1117,6 +1121,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
       desc.addFamily(hcd);
     }
     getHBaseAdmin().createTable(desc);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(c, tableName);
   }
 
@@ -1150,6 +1156,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
       desc.addFamily(hcd);
     }
     getHBaseAdmin().createTable(desc);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(new Configuration(getConfiguration()), tableName);
   }
 
@@ -1171,6 +1179,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
       desc.addFamily(hcd);
     }
     getHBaseAdmin().createTable(desc);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(new Configuration(getConfiguration()), tableName);
   }
 
@@ -1194,6 +1204,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
       i++;
     }
     getHBaseAdmin().createTable(desc);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(new Configuration(getConfiguration()), tableName);
   }
 
@@ -1211,6 +1223,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
     HColumnDescriptor hcd = new HColumnDescriptor(family);
     desc.addFamily(hcd);
     getHBaseAdmin().createTable(desc, splitRows);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(getConfiguration(), tableName);
   }
 
@@ -1230,6 +1244,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
       desc.addFamily(hcd);
     }
     getHBaseAdmin().createTable(desc, splitRows);
+    // HBaseAdmin only waits for regions to appear in META we should wait until they are assigned
+    waitUntilAllRegionsAssigned(tableName);
     return new HTable(getConfiguration(), tableName);
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java

@@ -248,9 +248,23 @@ public class TestHFileArchiving {
     UTIL.deleteTable(TABLE_NAME);
     LOG.debug("Deleted table");
 
-    // then get the files in the archive directory.
+    assertArchiveFiles(fs, storeFiles, 30000);
+  }
+
+  private void assertArchiveFiles(FileSystem fs, List<String> storeFiles, long timeout) throws IOException {
+    long end = System.currentTimeMillis() + timeout;
     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
-    List<String> archivedFiles = getAllFileNames(fs, archiveDir);
+    List<String> archivedFiles = new ArrayList<String>();
+
+    // We have to ensure that the DeleteTableHandler is finished. HBaseAdmin.deleteXXX() can return before all files
+    // are archived. We should fix HBASE-5487 and fix synchronous operations from admin.
+    while (System.currentTimeMillis() < end) {
+      archivedFiles = getAllFileNames(fs, archiveDir);
+      if (archivedFiles.size() >= storeFiles.size()) {
+        break;
+      }
+    }
+
     Collections.sort(storeFiles);
     Collections.sort(archivedFiles);
 
@@ -267,6 +281,7 @@ public class TestHFileArchiving {
       archivedFiles.containsAll(storeFiles));
   }
 
+
   /**
    * Test that the store files are archived when a column family is removed.
    * @throws Exception
@@ -310,23 +325,7 @@ public class TestHFileArchiving {
     // then delete the table so the hfiles get archived
     UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM);
 
-    // then get the files in the archive directory.
+    assertArchiveFiles(fs, storeFiles, 30000);
-    Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
-    List<String> archivedFiles = getAllFileNames(fs, archiveDir);
-    Collections.sort(storeFiles);
-    Collections.sort(archivedFiles);
-
-    LOG.debug("Store files:");
-    for (int i = 0; i < storeFiles.size(); i++) {
-      LOG.debug(i + " - " + storeFiles.get(i));
-    }
-    LOG.debug("Archive files:");
-    for (int i = 0; i < archivedFiles.size(); i++) {
-      LOG.debug(i + " - " + archivedFiles.get(i));
-    }
-
-    assertTrue("Archived files are missing some of the store files!",
-      archivedFiles.containsAll(storeFiles));
 
     UTIL.deleteTable(TABLE_NAME);
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java

@@ -39,7 +39,21 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.InvalidFamilyOperationException;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.TableNotDisabledException;
+import org.apache.hadoop.hbase.TableNotEnabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
 import org.apache.hadoop.hbase.catalog.CatalogTracker;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.exceptions.InvalidFamilyOperationException;
@@ -1077,7 +1091,7 @@ public class TestAdmin {
      new HColumnDescriptor("/cfamily/name");
   }
 
-  @Test(timeout=36000)
+  @Test(timeout=300000)
   public void testEnableDisableAddColumnDeleteColumn() throws Exception {
     ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
     byte [] tableName = Bytes.toBytes("testMasterAdmin");
@@ -1320,10 +1334,15 @@ public class TestAdmin {
             .getServerName().getServerName());
       }
     }
-    Thread.sleep(1000);
+    boolean isInList = ProtobufUtil.getOnlineRegions(rs).contains(info);
-    onlineRegions = ProtobufUtil.getOnlineRegions(rs);
+    long timeout = System.currentTimeMillis() + 10000;
+    while ((System.currentTimeMillis() < timeout) && (isInList)) {
+      Thread.sleep(100);
+      isInList = ProtobufUtil.getOnlineRegions(rs).contains(info);
+    }
+
     assertFalse("The region should not be present in online regions list.",
-        onlineRegions.contains(info));
+      isInList);
   }
 
   @Test
@@ -1372,7 +1391,7 @@ public class TestAdmin {
     }
 
     boolean isInList = ProtobufUtil.getOnlineRegions(rs).contains(info);
-    long timeout = System.currentTimeMillis() + 2000;
+    long timeout = System.currentTimeMillis() + 10000;
     while ((System.currentTimeMillis() < timeout) && (isInList)) {
       Thread.sleep(100);
       isInList = ProtobufUtil.getOnlineRegions(rs).contains(info);

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java

@@ -166,7 +166,14 @@ public class TestScannerTimeout {
     scan.setCaching(SCANNER_CACHING);
     LOG.info("************ TEST3686A");
     MetaReader.fullScanMetaAndPrint(TEST_UTIL.getHBaseCluster().getMaster().getCatalogTracker());
-    HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME);
+    // Set a very high timeout, we want to test what happens when a RS
+    // fails but the region is recovered before the lease times out.
+    // Since the RS is already created, this conf is client-side only for
+    // this new table
+    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
+    conf.setInt(
+        HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, SCANNER_TIMEOUT*100);
+    HTable table = new HTable(conf, TABLE_NAME);
     LOG.info("START ************ TEST3686A---22");
 
     ResultScanner r = table.getScanner(scan);

hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java

@@ -77,7 +77,7 @@ public class TestRegionServerCoprocessorExceptionWithRemove {
     TEST_UTIL.shutdownMiniCluster();
   }
 
-  @Test(timeout=30000)
+  @Test(timeout=60000)
   public void testExceptionFromCoprocessorDuringPut()
       throws IOException, InterruptedException {
     // Set watches on the zookeeper nodes for all of the regionservers in the

hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestChangingEncoding.java

@@ -69,7 +69,7 @@ public class TestChangingEncoding {
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static final Configuration conf = TEST_UTIL.getConfiguration();
 
-  private static final int TIMEOUT_MS = 240000;
+  private static final int TIMEOUT_MS = 600000;
 
   private HBaseAdmin admin;
   private HColumnDescriptor hcd;