HBASE-2870: Add Backup CLI Option to HMaster

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@984116 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -831,6 +831,7 @@ Release 0.21.0 - Unreleased
    HBASE-2792  Create a better way to chain log cleaners
                (Chongxin Li via Stack)
    HBASE-2844  Capping the number of regions (Pranav Khaitan via Stack)
+   HBASE-2870  Add Backup CLI Option to HMaster (Nicolas Spiegelberg via Stack)
 
   NEW FEATURES
    HBASE-1961  HBase EC2 scripts

bin/hbase-config.sh

@@ -73,6 +73,8 @@ done
 HBASE_CONF_DIR="${HBASE_CONF_DIR:-$HBASE_HOME/conf}"
 # List of hbase regions servers.
 HBASE_REGIONSERVERS="${HBASE_REGIONSERVERS:-$HBASE_CONF_DIR/regionservers}"
+# List of hbase secondary masters.
+HBASE_BACKUP_MASTERS="${HBASE_BACKUP_MASTERS:-$HBASE_CONF_DIR/backup-masters}"
 
 # Source the hbase-env.sh.  Will have JAVA_HOME defined.
 if [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then

bin/hbase-daemons.sh

@@ -45,6 +45,9 @@ case $command in
   (zookeeper)
     exec "$bin/zookeepers.sh" $args
     ;;
+  (master-backup)
+    exec "$bin/master-backup.sh" $args
+    ;;
   (*)
     exec "$bin/regionservers.sh" $args
     ;;

bin/local-master-backup.sh

@@ -21,6 +21,7 @@ run_master () {
   DN=$2
   export HBASE_IDENT_STRING="$USER-$DN"
   HBASE_MASTER_ARGS="\
+    --backup \
     -D hbase.master.port=`expr 60000 + $DN` \
     -D hbase.master.info.port=`expr 60010 + $DN`"
   "$bin"/hbase-daemon.sh $1 master $HBASE_MASTER_ARGS

bin/start-hbase.sh

@@ -49,4 +49,6 @@ else
   "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master 
   "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
     --hosts "${HBASE_REGIONSERVERS}" start regionserver
+  "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
+    --hosts "${HBASE_BACKUP_MASTERS}" start master-backup
 fi

bin/stop-hbase.sh

@@ -60,5 +60,10 @@ done
 distMode=`$bin/hbase org.apache.hadoop.hbase.HBaseConfTool hbase.cluster.distributed`
 if [ "$distMode" == 'true' ] 
 then
+  # TODO: store backup masters in ZooKeeper and have the primary send them a shutdown message
+  # stop any backup masters
+  "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
+    --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup
+
   "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper
 fi

src/main/java/org/apache/hadoop/hbase/HConstants.java

@@ -81,6 +81,12 @@ public final class HConstants {
   /** default port for master web api */
   public static final int DEFAULT_MASTER_INFOPORT = 60010;
 
+  /** Parameter name for the master type being backup (waits for primary to go inactive). */
+  public static final String MASTER_TYPE_BACKUP = "hbase.master.backup";
+
+  /** by default every master is a possible primary master unless the conf explicitly overrides it */
+  public static final boolean DEFAULT_MASTER_TYPE_BACKUP = false;
+
   /** Name of ZooKeeper quorum configuration parameter. */
   public static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum";
 

src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -230,6 +230,22 @@ public class HMaster extends Thread implements HMasterInterface,
     this.zkMasterAddressWatcher =
       new ZKMasterAddressWatcher(this.zooKeeperWrapper, this.shutdownRequested);
     zooKeeperWrapper.registerListener(zkMasterAddressWatcher);
+
+    // if we're a backup master, stall until a primary to writes his address
+    if(conf.getBoolean(HConstants.MASTER_TYPE_BACKUP, HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {
+      // this will only be a minute or so while the cluster starts up,
+      // so don't worry about setting watches on the parent znode
+      while (!zooKeeperWrapper.masterAddressExists()) {
+        try {
+          LOG.debug("Waiting for master address ZNode to be written " +
+            "(Also watching cluster state node)");
+          Thread.sleep(conf.getInt("zookeeper.session.timeout", 60 * 1000));
+        } catch (InterruptedException e) {
+          // interrupted = user wants to kill us.  Don't continue
+          throw new IOException("Interrupted waiting for master address");
+        }
+      }
+    }
     this.zkMasterAddressWatcher.writeAddressToZooKeeper(this.address, true);
     this.regionServerOperationQueue =
       new RegionServerOperationQueue(this.conf, this.closed);
@@ -1265,6 +1281,7 @@ public class HMaster extends Thread implements HMasterInterface,
     Options opt = new Options();
     opt.addOption("minServers", true, "Minimum RegionServers needed to host user tables");
     opt.addOption("D", true, "Override HBase Configuration Settings");
+    opt.addOption("backup", false, "Do not try to become HMaster until the primary fails");
     try {
       CommandLine cmd = new GnuParser().parse(opt, args);
 
@@ -1287,6 +1304,11 @@ public class HMaster extends Thread implements HMasterInterface,
         }
       }
       
+      // check if we are the backup master - override the conf if so
+      if (cmd.hasOption("backup")) {
+        conf.setBoolean(HConstants.MASTER_TYPE_BACKUP, true);
+      }
+
       if (cmd.getArgList().contains("start")) {
         try {
           // Print out vm stats before starting up.

src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWrapper.java

@@ -465,6 +465,13 @@ public class ZooKeeperWrapper implements Watcher {
     return true;
   }
   
+  /**
+   * @return true if zookeeper has a master address.
+   */
+  public boolean masterAddressExists() {
+    return checkExistenceOf(masterElectionZNode);
+  }
+
   private HServerAddress readAddress(String znode, Watcher watcher) {
     try {
       LOG.debug("<" + instanceName + ">" + "Trying to read " + znode);