HBASE-1551 HBase should manage multiple node ZooKeeper quorum

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@792636 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -452,6 +452,7 @@ Release 0.20.0 - Unreleased
    HBASE-1624  Don't sort Puts if only one in list in HCM#processBatchOfRows
    HBASE-1626  Allow emitting Deletes out of new TableReducer
                (Lars George via Stack)
+   HBASE-1551  HBase should manage multiple node ZooKeeper quorum
 
   OPTIMIZATIONS
    HBASE-1412  Change values for delete column and column family in KeyValue

bin/hbase-daemons.sh

@@ -37,6 +37,20 @@ bin=`cd "$bin"; pwd`
 
 . $bin/hbase-config.sh
 
-exec "$bin/regionservers.sh" --config "${HBASE_CONF_DIR}" \
+remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@"
- cd "${HBASE_HOME}" \; \
+args="--config ${HBASE_CONF_DIR} $remote_cmd"
- "$bin/hbase-daemon.sh" --config "${HBASE_CONF_DIR}" "$@"
+
+command=$2
+case $command in
+  (regionserver)
+    exec "$bin/regionservers.sh" $args
+    ;;
+  (zookeeper)
+    exec "$bin/zookeepers.sh" $args
+    ;;
+  (*)
+    echo $usage
+    exit 1
+    ;;
+esac
+

bin/hbase-zookeeper.sh

@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-#
-#/**
-# * Copyright 2009 The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements.  See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership.  The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License.  You may obtain a copy of the License at
-# *
-# *     http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-# 
-# Run a hbase command on all slave hosts.
-# Modelled after $HADOOP_HOME/bin/hadoop-daemons.sh
-
-usage="Usage: hbase-daemons.sh [--config <hbase-confdir>] \
- [start|stop] command args..."
-
-# if no args specified, show usage
-if [ $# -le 1 ]; then
-  echo $usage
-  exit 1
-fi
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-. $bin/hbase-config.sh
-
-exec "$bin/zookeeper.sh" --config "${HBASE_CONF_DIR}" \
- cd "${HBASE_HOME}" \; \
- "$bin/hbase-daemon.sh" --config "${HBASE_CONF_DIR}" "$@"

bin/start-hbase.sh

@@ -38,8 +38,8 @@ if [ $errCode -ne 0 ]
 then
   exit $errCode
 fi
-"$bin"/hbase-zookeeper.sh --config "${HBASE_CONF_DIR}" \
+
-  start zookeeper
+"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper
 "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master 
 "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
   --hosts "${HBASE_REGIONSERVERS}" start regionserver

bin/stop-hbase.sh

@@ -30,5 +30,4 @@ bin=`cd "$bin"; pwd`
 . "$bin"/hbase-config.sh
 
 "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master
-"$bin"/hbase-zookeeper.sh --config "${HBASE_CONF_DIR}" \
+"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper
-  stop zookeeper

bin/zookeepers.sh

@@ -20,18 +20,17 @@
 # * limitations under the License.
 # */
 # 
-# Run a shell command on all regionserver hosts.
+# Run a shell command on all zookeeper hosts.
 #
 # Environment Variables
 #
-#   HADOOP_CONF_DIR  Alternate conf dir. Default is ${HADOOP_HOME}/conf.
 #   HBASE_CONF_DIR  Alternate hbase conf dir. Default is ${HBASE_HOME}/conf.
-#   HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
+#   HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
-#   HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
+#   HBASE_SSH_OPTS Options passed to ssh when running remote commands.
 #
 # Modelled after $HADOOP_HOME/bin/slaves.sh.
 
-usage="Usage: zookeeper [--config <hbase-confdir>] command..."
+usage="Usage: zookeepers [--config <hbase-confdir>] command..."
 
 # if no args specified, show usage
 if [ $# -le 0 ]; then
@@ -53,10 +52,14 @@ if [ "$HBASE_MANAGES_ZK" = "" ]; then
 fi
 
 if [ "$HBASE_MANAGES_ZK" = "true" ]; then
- ssh $HBASE_SSH_OPTS localhost $"${@// /\\ }" 2>&1 | sed "s/^/localhost: /" &
+  hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool`
- if [ "$HBASE_SLAVE_SLEEP" != "" ]; then
+  cmd=$"${@// /\\ }"
-   sleep $HBASE_SLAVE_SLEEP
+  for zookeeper in $hosts; do
- fi
+   ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" &
+   if [ "$HBASE_SLAVE_SLEEP" != "" ]; then
+     sleep $HBASE_SLAVE_SLEEP
+   fi
+  done
 fi
 
 wait

conf/hbase-default.xml

@@ -199,6 +199,21 @@
     unstable region servers caused by an OOME.
     </description>
   </property>
+  <property>
+    <name>hbase.zookeeper.dns.interface</name>
+    <value>default</value>
+    <description>The name of the Network Interface from which a ZooKeeper server
+      should report its IP address.
+    </description>
+  </property>
+  <property>
+    <name>hbase.zookeeper.dns.nameserver</name>
+    <value>default</value>
+    <description>The host name or IP address of the name server (DNS)
+      which a ZooKeeper server should use to determine the host name used by the
+      master for communication and display purposes.
+    </description>
+  </property>
   <property>
     <name>hbase.regionserver.dns.interface</name>
     <value>default</value>

src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java

@@ -19,8 +19,11 @@
  */
 package org.apache.hadoop.hbase.zookeeper;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PrintWriter;
+import java.net.UnknownHostException;
 import java.util.Properties;
 import java.util.Map.Entry;
 
@@ -28,11 +31,11 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.net.DNS;
 import org.apache.zookeeper.server.ServerConfig;
 import org.apache.zookeeper.server.ZooKeeperServerMain;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
 import org.apache.zookeeper.server.quorum.QuorumPeerMain;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
 
 /**
  * HBase's version of ZooKeeper's QuorumPeer. When HBase is set to manage
@@ -53,28 +56,24 @@ public class HQuorumPeer implements HConstants {
   private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
 
   /**
-   * Parse ZooKeeper configuration and run a QuorumPeer.
+   * Parse ZooKeeper configuration from HBase XML config and run a QuorumPeer.
-   * While parsing the zoo.cfg, we substitute variables with values from
-   * hbase-site.xml.
    * @param args String[] of command line arguments. Not used.
-   * @throws IOException
    */
-  public static void main(String[] args) throws IOException {
+  public static void main(String[] args) {
     HBaseConfiguration conf = new HBaseConfiguration();
-    Properties zkProperties = makeZKProps(conf);
-
-    QuorumPeerConfig zkConfig = new QuorumPeerConfig();
     try {
+      Properties zkProperties = makeZKProps(conf);
+      writeMyID(zkProperties);
+      QuorumPeerConfig zkConfig = new QuorumPeerConfig();
       zkConfig.parseProperties(zkProperties);
-    } catch (ConfigException e) {
+      runZKServer(zkConfig);
+    } catch (Exception e) {
       e.printStackTrace();
       System.exit(-1);
     }
-
-    startZKServer(zkConfig);
   }
 
-  private static void startZKServer(QuorumPeerConfig zkConfig) throws IOException {
+  private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException, IOException {
     if (zkConfig.isDistributed()) {
       QuorumPeerMain qp = new QuorumPeerMain();
       qp.runFromConfig(zkConfig);
@@ -86,6 +85,48 @@ public class HQuorumPeer implements HConstants {
     }
   }
 
+  private static void writeMyID(Properties properties) throws UnknownHostException, IOException {
+    HBaseConfiguration conf = new HBaseConfiguration();
+    String myAddress = DNS.getDefaultHost(
+        conf.get("hbase.zookeeper.dns.interface","default"),
+        conf.get("hbase.zookeeper.dns.nameserver","default"));
+
+    long myId = -1;
+
+    for (Entry<Object, Object> entry : properties.entrySet()) {
+      String key = entry.getKey().toString().trim();
+      String value = entry.getValue().toString().trim();
+      if (key.startsWith("server.")) {
+        int dot = key.indexOf('.');
+        long id = Long.parseLong(key.substring(dot + 1));
+        String[] parts = value.split(":");
+        String address = parts[0];
+        if (myAddress.equals(address)) {
+          myId = id;
+          break;
+        }
+      }
+    }
+
+    if (myId == -1) {
+      throw new IOException("Could not find my address: " + myAddress +
+                            " in list of ZooKeeper quorum servers");
+    }
+
+    String dataDirStr = properties.get("dataDir").toString().trim();
+    File dataDir = new File(dataDirStr);
+    if (!dataDir.isDirectory()) {
+      if (!dataDir.mkdirs()) {
+        throw new IOException("Unable to create data dir " + dataDir);
+      }
+    }
+
+    File myIdFile = new File(dataDir, "myid");
+    PrintWriter w = new PrintWriter(myIdFile);
+    w.println(myId);
+    w.close();
+  }
+
   /**
    * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
    * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
@@ -193,11 +234,11 @@ public class HQuorumPeer implements HConstants {
       if (key.startsWith("server.")) {
         if(conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) &&
             value.startsWith("localhost")) {
-           String msg = "The server in zoo.cfg cannot be set to localhost " +
+          String msg = "The server in zoo.cfg cannot be set to localhost " +
               "in a fully-distributed setup because it won't be reachable. " +
               "See \"Getting Started\" for more information.";
-           LOG.fatal(msg);
+          LOG.fatal(msg);
-           throw new IOException(msg);
+          throw new IOException(msg);
         }
       }
       newValue.append(value.substring(varEnd));

src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java

@@ -0,0 +1,33 @@
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.util.Properties;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+
+/**
+ * Tool for reading ZooKeeper servers from HBase XML configuation and producing
+ * a line-by-line list for use by bash scripts.
+ */
+public class ZKServerTool implements HConstants {
+  /**
+   * Run the tool.
+   * @param args Command line arguments. First arg is path to zookeepers file.
+   */
+  public static void main(String args[]) {
+    HBaseConfiguration conf = new HBaseConfiguration();
+    // Note that we do not simply grab the property ZOOKEEPER_QUORUM from
+    // the HBaseConfiguration because the user may be using a zoo.cfg file.
+    Properties zkProps = HQuorumPeer.makeZKProps(conf);
+    for (Entry<Object, Object> entry : zkProps.entrySet()) {
+      String key = entry.getKey().toString().trim();
+      String value = entry.getValue().toString().trim();
+      if (key.startsWith("server.")) {
+        String[] parts = value.split(":");
+        String host = parts[0];
+        System.out.println(host);
+      }
+    }
+  }
+}

src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java

@@ -25,7 +25,6 @@ import java.util.Map;
 import java.util.Properties;
 
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestCase;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;