HBASE-1982 [EC2] Handle potentially large and uneven instance startup times

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@892958 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -247,6 +247,8 @@ Release 0.21.0 - Unreleased
                Andrew Purtell)
    HBASE-2028  Add HTable.incrementColumnValue support to shell (Lars George
                via Andrew Purtell)
+   HBASE-1982  [EC2] Handle potentially large and uneven instance startup
+               times
 
   NEW FEATURES
    HBASE-1901  "General" partitioner for "hbase-48" bulk (behind the api, write

src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java

@@ -440,6 +440,24 @@ public class HBaseAdmin {
     return connection.isTableDisabled(tableName);
   }
 
+  /**
+   * @param tableName name of table to check
+   * @return true if all regions of the table are available
+   * @throws IOException
+   */
+  public boolean isTableAvailable(byte[] tableName) throws IOException {
+    return connection.isTableAvailable(tableName);
+  }
+
+  /**
+   * @param tableName name of table to check
+   * @return true if all regions of the table are available
+   * @throws IOException
+   */
+  public boolean isTableAvailable(String tableName) throws IOException {
+    return connection.isTableAvailable(Bytes.toBytes(tableName));
+  }
+
   /**
    * Add a column to an existing table.
    * Asynchronous operation.

src/java/org/apache/hadoop/hbase/client/HConnection.java

@@ -77,7 +77,14 @@ public interface HConnection {
    * @throws IOException
    */
   public boolean isTableDisabled(byte[] tableName) throws IOException;
-  
+
+  /**
+   * @param tableName
+   * @return true if all regions of the table are available, false otherwise
+   * @throws IOException
+   */
+  public boolean isTableAvailable(byte[] tableName) throws IOException;
+
   /**
    * List all the userspace tables.  In other words, scan the META table.
    *

src/java/org/apache/hadoop/hbase/client/HConnectionManager.java

@@ -29,6 +29,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -463,6 +464,29 @@ public class HConnectionManager implements HConstants {
       return testTableOnlineState(tableName, false);
     }
 
+    public boolean isTableAvailable(final byte[] tableName) throws IOException {
+      final AtomicBoolean available = new AtomicBoolean(true);
+      MetaScannerVisitor visitor = new MetaScannerVisitor() {
+        @Override
+        public boolean processRow(Result row) throws IOException {
+          byte[] value = row.getValue(CATALOG_FAMILY, REGIONINFO_QUALIFIER);
+          HRegionInfo info = Writables.getHRegionInfoOrNull(value);
+          if (info != null) {
+            if (Bytes.equals(tableName, info.getTableDesc().getName())) {
+              value = row.getValue(CATALOG_FAMILY, SERVER_QUALIFIER);
+              if (value == null) {
+                available.set(false);
+                return false;
+              }
+            }
+          }
+          return true;
+        }        
+      };
+      MetaScanner.metaScan(conf, visitor);
+      return available.get();
+    }
+
     /*
      * If online == true
      *   Returns true if all regions are online

src/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -752,6 +752,9 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
         if (!this.regionManager.areAllMetaRegionsOnline()) {
           throw new NotAllMetaRegionsOnlineException();
         }
+        if (!this.serverManager.canAssignUserRegions()) {
+          throw new IOException("not enough servers to create table yet");
+        }
         createTable(newRegion);
         LOG.info("created table " + desc.getNameAsString());
         break;
@@ -1155,9 +1158,11 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
   }
 
   private static void printUsageAndExit() {
-    System.err.println("Usage: Master start|stop");
+    System.err.println("Usage: Master [opts] start|stop");
     System.err.println(" start  Start Master. If local mode, start Master and RegionServer in same JVM");
     System.err.println(" stop   Start cluster shutdown; Master signals RegionServer shutdown");
+    System.err.println(" where [opts] are:");
+    System.err.println("   --minServers=<servers>    Minimum RegionServers needed to host user tables.");
     System.exit(0);
   }
 
@@ -1168,6 +1173,13 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
     HBaseConfiguration conf = new HBaseConfiguration();
     // Process command-line args.
     for (String cmd: args) {
+
+      if (cmd.startsWith("--minServers=")) {
+        conf.setInt("hbase.regions.server.count.min",
+          Integer.valueOf(cmd.substring(13)));
+        continue;
+      }
+
       if (cmd.equalsIgnoreCase("start")) {
         try {
           // Print out vm stats before starting up.

src/java/org/apache/hadoop/hbase/master/RegionManager.java

@@ -56,7 +56,6 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
 
 /**
  * Class to manage assigning regions to servers, state of root and meta, etc.
@@ -414,6 +413,12 @@ public class RegionManager implements HConstants {
           // and are on-line
           continue;
         }
+        if (!i.isMetaRegion() && 
+            !master.getServerManager().canAssignUserRegions()) {
+          LOG.debug("user region " + i.getRegionNameAsString() +
+            " is in transition but not enough servers yet");
+          continue;
+        }
         if (s.isUnassigned()) {
           regionsToAssign.add(s);
         }

src/java/org/apache/hadoop/hbase/master/ServerManager.java

@@ -93,6 +93,8 @@ public class ServerManager implements HConstants {
 
   private final ServerMonitor serverMonitorThread;
 
+  private int minimumServerCount;
+
   /*
    * Dumps into log current stats on dead servers and number of servers
    * TODO: Make this a metric; dump metrics into log.
@@ -136,6 +138,7 @@ public class ServerManager implements HConstants {
     this.nobalancingCount = c.getInt("hbase.regions.nobalancing.count", 4);
     int metaRescanInterval = c.getInt("hbase.master.meta.thread.rescanfrequency",
       60 * 1000);
+    this.minimumServerCount = c.getInt("hbase.regions.server.count.min", 0);
     this.serverMonitorThread = new ServerMonitor(metaRescanInterval,
       this.master.getShutdownRequested());
     this.serverMonitorThread.start();
@@ -844,4 +847,16 @@ public class ServerManager implements HConstants {
       m.putAll(this.loadToServers.headMap(l));
     }
   }
+
+  public boolean canAssignUserRegions() {
+    if (minimumServerCount == 0) {
+      return true;
+    }
+    return (numServers() >= minimumServerCount);
+  }
+
+  public void setMinimumServerCount(int minimumServerCount) {
+    this.minimumServerCount = minimumServerCount;
+  }
+
 }

src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java

@@ -41,7 +41,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
 public class MiniHBaseCluster implements HConstants {
   static final Log LOG = LogFactory.getLog(MiniHBaseCluster.class.getName());
   private HBaseConfiguration conf;
-  private LocalHBaseCluster hbaseCluster;
+  public LocalHBaseCluster hbaseCluster;
 
   /**
    * Start a MiniHBaseCluster. 

src/test/org/apache/hadoop/hbase/master/TestMinimumServerCount.java

@@ -0,0 +1,68 @@
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.HBaseClusterTestCase;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+
+public class TestMinimumServerCount extends HBaseClusterTestCase {
+
+  static final String TABLE_NAME = "TestTable";
+
+  public TestMinimumServerCount() {
+    // start cluster with one region server only
+    super(1, true);
+  }
+
+  boolean isTableAvailable(String tableName) throws IOException {
+    boolean available = true;
+    HTable meta = new HTable(conf, ".META.");
+    ResultScanner scanner = meta.getScanner(HConstants.CATALOG_FAMILY);
+    Result result;
+    while ((result = scanner.next()) != null) {
+      // set available to false if a region of the table is found with no
+      // assigned server
+      byte[] value = result.getValue(HConstants.CATALOG_FAMILY,
+        HConstants.SERVER_QUALIFIER);
+      if (value == null) {
+        available = false;
+        break;
+      }
+    }
+    return available;
+  }
+
+  public void testMinimumServerCount() throws Exception {
+    HBaseAdmin admin = new HBaseAdmin(conf);
+
+    // create and disable table
+    admin.createTable(createTableDescriptor(TABLE_NAME));
+    admin.disableTable(TABLE_NAME);
+    assertFalse(admin.isTableEnabled(TABLE_NAME));
+
+    // reach in and set minimum server count
+    cluster.hbaseCluster.getMaster().getServerManager()
+      .setMinimumServerCount(2);
+
+    // now try to enable the table
+    try {
+      admin.enableTable(TABLE_NAME);
+    } catch (IOException ex) {
+      // ignore
+    }
+    Thread.sleep(10 * 1000);
+    assertFalse(admin.isTableAvailable(TABLE_NAME));
+    
+    // now start another region server
+    cluster.startRegionServer();
+
+    // sleep a bit for assignment
+    Thread.sleep(10 * 1000);
+    assertTrue(admin.isTableAvailable(TABLE_NAME));
+  }
+
+}