From 002e9e8bdc2c842565a4e9bfc4e1dfb28c479bc5 Mon Sep 17 00:00:00 2001 From: Nitay Joffe Date: Thu, 9 Jul 2009 19:04:43 +0000 Subject: [PATCH] HBASE-1551 HBase should manage multiple node ZooKeeper quorum git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@792636 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + bin/hbase-daemons.sh | 20 ++++- bin/hbase-zookeeper.sh | 42 ----------- bin/start-hbase.sh | 4 +- bin/stop-hbase.sh | 3 +- bin/{zookeeper.sh => zookeepers.sh} | 21 +++--- conf/hbase-default.xml | 15 ++++ .../hadoop/hbase/zookeeper/HQuorumPeer.java | 73 +++++++++++++++---- .../hadoop/hbase/zookeeper/ZKServerTool.java | 33 +++++++++ .../hbase/zookeeper/HQuorumPeerTest.java | 1 - 10 files changed, 138 insertions(+), 75 deletions(-) delete mode 100755 bin/hbase-zookeeper.sh rename bin/{zookeeper.sh => zookeepers.sh} (72%) create mode 100644 src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java diff --git a/CHANGES.txt b/CHANGES.txt index cf497b95d57..439fda74a9f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -452,6 +452,7 @@ Release 0.20.0 - Unreleased HBASE-1624 Don't sort Puts if only one in list in HCM#processBatchOfRows HBASE-1626 Allow emitting Deletes out of new TableReducer (Lars George via Stack) + HBASE-1551 HBase should manage multiple node ZooKeeper quorum OPTIMIZATIONS HBASE-1412 Change values for delete column and column family in KeyValue diff --git a/bin/hbase-daemons.sh b/bin/hbase-daemons.sh index 166af335ea5..5af94080113 100755 --- a/bin/hbase-daemons.sh +++ b/bin/hbase-daemons.sh @@ -37,6 +37,20 @@ bin=`cd "$bin"; pwd` . $bin/hbase-config.sh -exec "$bin/regionservers.sh" --config "${HBASE_CONF_DIR}" \ - cd "${HBASE_HOME}" \; \ - "$bin/hbase-daemon.sh" --config "${HBASE_CONF_DIR}" "$@" +remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@" +args="--config ${HBASE_CONF_DIR} $remote_cmd" + +command=$2 +case $command in + (regionserver) + exec "$bin/regionservers.sh" $args + ;; + (zookeeper) + exec "$bin/zookeepers.sh" $args + ;; + (*) + echo $usage + exit 1 + ;; +esac + diff --git a/bin/hbase-zookeeper.sh b/bin/hbase-zookeeper.sh deleted file mode 100755 index 859faabc745..00000000000 --- a/bin/hbase-zookeeper.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2009 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a hbase command on all slave hosts. -# Modelled after $HADOOP_HOME/bin/hadoop-daemons.sh - -usage="Usage: hbase-daemons.sh [--config ] \ - [start|stop] command args..." - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -. $bin/hbase-config.sh - -exec "$bin/zookeeper.sh" --config "${HBASE_CONF_DIR}" \ - cd "${HBASE_HOME}" \; \ - "$bin/hbase-daemon.sh" --config "${HBASE_CONF_DIR}" "$@" diff --git a/bin/start-hbase.sh b/bin/start-hbase.sh index 324fa643c2d..c1209e04afe 100755 --- a/bin/start-hbase.sh +++ b/bin/start-hbase.sh @@ -38,8 +38,8 @@ if [ $errCode -ne 0 ] then exit $errCode fi -"$bin"/hbase-zookeeper.sh --config "${HBASE_CONF_DIR}" \ - start zookeeper + +"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ --hosts "${HBASE_REGIONSERVERS}" start regionserver diff --git a/bin/stop-hbase.sh b/bin/stop-hbase.sh index 071ae55257f..876673ed877 100755 --- a/bin/stop-hbase.sh +++ b/bin/stop-hbase.sh @@ -30,5 +30,4 @@ bin=`cd "$bin"; pwd` . "$bin"/hbase-config.sh "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master -"$bin"/hbase-zookeeper.sh --config "${HBASE_CONF_DIR}" \ - stop zookeeper +"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper diff --git a/bin/zookeeper.sh b/bin/zookeepers.sh similarity index 72% rename from bin/zookeeper.sh rename to bin/zookeepers.sh index a379fb0b268..7ea07a3fa16 100755 --- a/bin/zookeeper.sh +++ b/bin/zookeepers.sh @@ -20,18 +20,17 @@ # * limitations under the License. # */ # -# Run a shell command on all regionserver hosts. +# Run a shell command on all zookeeper hosts. # # Environment Variables # -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. # HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. # # Modelled after $HADOOP_HOME/bin/slaves.sh. -usage="Usage: zookeeper [--config ] command..." +usage="Usage: zookeepers [--config ] command..." # if no args specified, show usage if [ $# -le 0 ]; then @@ -53,10 +52,14 @@ if [ "$HBASE_MANAGES_ZK" = "" ]; then fi if [ "$HBASE_MANAGES_ZK" = "true" ]; then - ssh $HBASE_SSH_OPTS localhost $"${@// /\\ }" 2>&1 | sed "s/^/localhost: /" & - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi + hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool` + cmd=$"${@// /\\ }" + for zookeeper in $hosts; do + ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" & + if [ "$HBASE_SLAVE_SLEEP" != "" ]; then + sleep $HBASE_SLAVE_SLEEP + fi + done fi wait diff --git a/conf/hbase-default.xml b/conf/hbase-default.xml index af2d8210ef2..da67cdf0133 100644 --- a/conf/hbase-default.xml +++ b/conf/hbase-default.xml @@ -199,6 +199,21 @@ unstable region servers caused by an OOME. + + hbase.zookeeper.dns.interface + default + The name of the Network Interface from which a ZooKeeper server + should report its IP address. + + + + hbase.zookeeper.dns.nameserver + default + The host name or IP address of the name server (DNS) + which a ZooKeeper server should use to determine the host name used by the + master for communication and display purposes. + + hbase.regionserver.dns.interface default diff --git a/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java b/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java index 9252f401cdb..63784cc9cea 100644 --- a/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java +++ b/src/java/org/apache/hadoop/hbase/zookeeper/HQuorumPeer.java @@ -19,8 +19,11 @@ */ package org.apache.hadoop.hbase.zookeeper; +import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.PrintWriter; +import java.net.UnknownHostException; import java.util.Properties; import java.util.Map.Entry; @@ -28,11 +31,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.net.DNS; import org.apache.zookeeper.server.ServerConfig; import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.apache.zookeeper.server.quorum.QuorumPeerMain; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; /** * HBase's version of ZooKeeper's QuorumPeer. When HBase is set to manage @@ -53,28 +56,24 @@ public class HQuorumPeer implements HConstants { private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length(); /** - * Parse ZooKeeper configuration and run a QuorumPeer. - * While parsing the zoo.cfg, we substitute variables with values from - * hbase-site.xml. + * Parse ZooKeeper configuration from HBase XML config and run a QuorumPeer. * @param args String[] of command line arguments. Not used. - * @throws IOException */ - public static void main(String[] args) throws IOException { + public static void main(String[] args) { HBaseConfiguration conf = new HBaseConfiguration(); - Properties zkProperties = makeZKProps(conf); - - QuorumPeerConfig zkConfig = new QuorumPeerConfig(); try { + Properties zkProperties = makeZKProps(conf); + writeMyID(zkProperties); + QuorumPeerConfig zkConfig = new QuorumPeerConfig(); zkConfig.parseProperties(zkProperties); - } catch (ConfigException e) { + runZKServer(zkConfig); + } catch (Exception e) { e.printStackTrace(); System.exit(-1); } - - startZKServer(zkConfig); } - private static void startZKServer(QuorumPeerConfig zkConfig) throws IOException { + private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException, IOException { if (zkConfig.isDistributed()) { QuorumPeerMain qp = new QuorumPeerMain(); qp.runFromConfig(zkConfig); @@ -86,6 +85,48 @@ public class HQuorumPeer implements HConstants { } } + private static void writeMyID(Properties properties) throws UnknownHostException, IOException { + HBaseConfiguration conf = new HBaseConfiguration(); + String myAddress = DNS.getDefaultHost( + conf.get("hbase.zookeeper.dns.interface","default"), + conf.get("hbase.zookeeper.dns.nameserver","default")); + + long myId = -1; + + for (Entry entry : properties.entrySet()) { + String key = entry.getKey().toString().trim(); + String value = entry.getValue().toString().trim(); + if (key.startsWith("server.")) { + int dot = key.indexOf('.'); + long id = Long.parseLong(key.substring(dot + 1)); + String[] parts = value.split(":"); + String address = parts[0]; + if (myAddress.equals(address)) { + myId = id; + break; + } + } + } + + if (myId == -1) { + throw new IOException("Could not find my address: " + myAddress + + " in list of ZooKeeper quorum servers"); + } + + String dataDirStr = properties.get("dataDir").toString().trim(); + File dataDir = new File(dataDirStr); + if (!dataDir.isDirectory()) { + if (!dataDir.mkdirs()) { + throw new IOException("Unable to create data dir " + dataDir); + } + } + + File myIdFile = new File(dataDir, "myid"); + PrintWriter w = new PrintWriter(myIdFile); + w.println(myId); + w.close(); + } + /** * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg. * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse @@ -193,11 +234,11 @@ public class HQuorumPeer implements HConstants { if (key.startsWith("server.")) { if(conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) && value.startsWith("localhost")) { - String msg = "The server in zoo.cfg cannot be set to localhost " + + String msg = "The server in zoo.cfg cannot be set to localhost " + "in a fully-distributed setup because it won't be reachable. " + "See \"Getting Started\" for more information."; - LOG.fatal(msg); - throw new IOException(msg); + LOG.fatal(msg); + throw new IOException(msg); } } newValue.append(value.substring(varEnd)); diff --git a/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java b/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java new file mode 100644 index 00000000000..349f48e67ac --- /dev/null +++ b/src/java/org/apache/hadoop/hbase/zookeeper/ZKServerTool.java @@ -0,0 +1,33 @@ +package org.apache.hadoop.hbase.zookeeper; + +import java.util.Properties; +import java.util.Map.Entry; + +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; + +/** + * Tool for reading ZooKeeper servers from HBase XML configuation and producing + * a line-by-line list for use by bash scripts. + */ +public class ZKServerTool implements HConstants { + /** + * Run the tool. + * @param args Command line arguments. First arg is path to zookeepers file. + */ + public static void main(String args[]) { + HBaseConfiguration conf = new HBaseConfiguration(); + // Note that we do not simply grab the property ZOOKEEPER_QUORUM from + // the HBaseConfiguration because the user may be using a zoo.cfg file. + Properties zkProps = HQuorumPeer.makeZKProps(conf); + for (Entry entry : zkProps.entrySet()) { + String key = entry.getKey().toString().trim(); + String value = entry.getValue().toString().trim(); + if (key.startsWith("server.")) { + String[] parts = value.split(":"); + String host = parts[0]; + System.out.println(host); + } + } + } +} diff --git a/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java b/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java index b75c15f6463..b8420075d06 100644 --- a/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java +++ b/src/test/org/apache/hadoop/hbase/zookeeper/HQuorumPeerTest.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Properties; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestCase; import org.apache.hadoop.hbase.HConstants; import org.apache.zookeeper.server.quorum.QuorumPeerConfig;