HBASE-1551 HBase should manage multiple node ZooKeeper quorum
git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@792636 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
2a08fa99ad
commit
002e9e8bdc
|
@ -452,6 +452,7 @@ Release 0.20.0 - Unreleased
|
|||
HBASE-1624 Don't sort Puts if only one in list in HCM#processBatchOfRows
|
||||
HBASE-1626 Allow emitting Deletes out of new TableReducer
|
||||
(Lars George via Stack)
|
||||
HBASE-1551 HBase should manage multiple node ZooKeeper quorum
|
||||
|
||||
OPTIMIZATIONS
|
||||
HBASE-1412 Change values for delete column and column family in KeyValue
|
||||
|
|
|
@ -37,6 +37,20 @@ bin=`cd "$bin"; pwd`
|
|||
|
||||
. $bin/hbase-config.sh
|
||||
|
||||
exec "$bin/regionservers.sh" --config "${HBASE_CONF_DIR}" \
|
||||
cd "${HBASE_HOME}" \; \
|
||||
"$bin/hbase-daemon.sh" --config "${HBASE_CONF_DIR}" "$@"
|
||||
remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@"
|
||||
args="--config ${HBASE_CONF_DIR} $remote_cmd"
|
||||
|
||||
command=$2
|
||||
case $command in
|
||||
(regionserver)
|
||||
exec "$bin/regionservers.sh" $args
|
||||
;;
|
||||
(zookeeper)
|
||||
exec "$bin/zookeepers.sh" $args
|
||||
;;
|
||||
(*)
|
||||
echo $usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
|
|
|
@ -1,42 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
#/**
|
||||
# * Copyright 2009 The Apache Software Foundation
|
||||
# *
|
||||
# * Licensed to the Apache Software Foundation (ASF) under one
|
||||
# * or more contributor license agreements. See the NOTICE file
|
||||
# * distributed with this work for additional information
|
||||
# * regarding copyright ownership. The ASF licenses this file
|
||||
# * to you under the Apache License, Version 2.0 (the
|
||||
# * "License"); you may not use this file except in compliance
|
||||
# * with the License. You may obtain a copy of the License at
|
||||
# *
|
||||
# * http://www.apache.org/licenses/LICENSE-2.0
|
||||
# *
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# * See the License for the specific language governing permissions and
|
||||
# * limitations under the License.
|
||||
# */
|
||||
#
|
||||
# Run a hbase command on all slave hosts.
|
||||
# Modelled after $HADOOP_HOME/bin/hadoop-daemons.sh
|
||||
|
||||
usage="Usage: hbase-daemons.sh [--config <hbase-confdir>] \
|
||||
[start|stop] command args..."
|
||||
|
||||
# if no args specified, show usage
|
||||
if [ $# -le 1 ]; then
|
||||
echo $usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bin=`dirname "$0"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
|
||||
. $bin/hbase-config.sh
|
||||
|
||||
exec "$bin/zookeeper.sh" --config "${HBASE_CONF_DIR}" \
|
||||
cd "${HBASE_HOME}" \; \
|
||||
"$bin/hbase-daemon.sh" --config "${HBASE_CONF_DIR}" "$@"
|
|
@ -38,8 +38,8 @@ if [ $errCode -ne 0 ]
|
|||
then
|
||||
exit $errCode
|
||||
fi
|
||||
"$bin"/hbase-zookeeper.sh --config "${HBASE_CONF_DIR}" \
|
||||
start zookeeper
|
||||
|
||||
"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper
|
||||
"$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master
|
||||
"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \
|
||||
--hosts "${HBASE_REGIONSERVERS}" start regionserver
|
||||
|
|
|
@ -30,5 +30,4 @@ bin=`cd "$bin"; pwd`
|
|||
. "$bin"/hbase-config.sh
|
||||
|
||||
"$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master
|
||||
"$bin"/hbase-zookeeper.sh --config "${HBASE_CONF_DIR}" \
|
||||
stop zookeeper
|
||||
"$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper
|
||||
|
|
|
@ -20,18 +20,17 @@
|
|||
# * limitations under the License.
|
||||
# */
|
||||
#
|
||||
# Run a shell command on all regionserver hosts.
|
||||
# Run a shell command on all zookeeper hosts.
|
||||
#
|
||||
# Environment Variables
|
||||
#
|
||||
# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf.
|
||||
# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf.
|
||||
# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
|
||||
# HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
|
||||
# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands.
|
||||
# HBASE_SSH_OPTS Options passed to ssh when running remote commands.
|
||||
#
|
||||
# Modelled after $HADOOP_HOME/bin/slaves.sh.
|
||||
|
||||
usage="Usage: zookeeper [--config <hbase-confdir>] command..."
|
||||
usage="Usage: zookeepers [--config <hbase-confdir>] command..."
|
||||
|
||||
# if no args specified, show usage
|
||||
if [ $# -le 0 ]; then
|
||||
|
@ -53,10 +52,14 @@ if [ "$HBASE_MANAGES_ZK" = "" ]; then
|
|||
fi
|
||||
|
||||
if [ "$HBASE_MANAGES_ZK" = "true" ]; then
|
||||
ssh $HBASE_SSH_OPTS localhost $"${@// /\\ }" 2>&1 | sed "s/^/localhost: /" &
|
||||
if [ "$HBASE_SLAVE_SLEEP" != "" ]; then
|
||||
sleep $HBASE_SLAVE_SLEEP
|
||||
fi
|
||||
hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool`
|
||||
cmd=$"${@// /\\ }"
|
||||
for zookeeper in $hosts; do
|
||||
ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" &
|
||||
if [ "$HBASE_SLAVE_SLEEP" != "" ]; then
|
||||
sleep $HBASE_SLAVE_SLEEP
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
wait
|
|
@ -199,6 +199,21 @@
|
|||
unstable region servers caused by an OOME.
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>hbase.zookeeper.dns.interface</name>
|
||||
<value>default</value>
|
||||
<description>The name of the Network Interface from which a ZooKeeper server
|
||||
should report its IP address.
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>hbase.zookeeper.dns.nameserver</name>
|
||||
<value>default</value>
|
||||
<description>The host name or IP address of the name server (DNS)
|
||||
which a ZooKeeper server should use to determine the host name used by the
|
||||
master for communication and display purposes.
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>hbase.regionserver.dns.interface</name>
|
||||
<value>default</value>
|
||||
|
|
|
@ -19,8 +19,11 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.zookeeper;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.PrintWriter;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.Properties;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
|
@ -28,11 +31,11 @@ import org.apache.commons.logging.Log;
|
|||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.net.DNS;
|
||||
import org.apache.zookeeper.server.ServerConfig;
|
||||
import org.apache.zookeeper.server.ZooKeeperServerMain;
|
||||
import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
|
||||
import org.apache.zookeeper.server.quorum.QuorumPeerMain;
|
||||
import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
|
||||
|
||||
/**
|
||||
* HBase's version of ZooKeeper's QuorumPeer. When HBase is set to manage
|
||||
|
@ -53,28 +56,24 @@ public class HQuorumPeer implements HConstants {
|
|||
private static final int ZK_CFG_PROPERTY_SIZE = ZK_CFG_PROPERTY.length();
|
||||
|
||||
/**
|
||||
* Parse ZooKeeper configuration and run a QuorumPeer.
|
||||
* While parsing the zoo.cfg, we substitute variables with values from
|
||||
* hbase-site.xml.
|
||||
* Parse ZooKeeper configuration from HBase XML config and run a QuorumPeer.
|
||||
* @param args String[] of command line arguments. Not used.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void main(String[] args) throws IOException {
|
||||
public static void main(String[] args) {
|
||||
HBaseConfiguration conf = new HBaseConfiguration();
|
||||
Properties zkProperties = makeZKProps(conf);
|
||||
|
||||
QuorumPeerConfig zkConfig = new QuorumPeerConfig();
|
||||
try {
|
||||
Properties zkProperties = makeZKProps(conf);
|
||||
writeMyID(zkProperties);
|
||||
QuorumPeerConfig zkConfig = new QuorumPeerConfig();
|
||||
zkConfig.parseProperties(zkProperties);
|
||||
} catch (ConfigException e) {
|
||||
runZKServer(zkConfig);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
System.exit(-1);
|
||||
}
|
||||
|
||||
startZKServer(zkConfig);
|
||||
}
|
||||
|
||||
private static void startZKServer(QuorumPeerConfig zkConfig) throws IOException {
|
||||
private static void runZKServer(QuorumPeerConfig zkConfig) throws UnknownHostException, IOException {
|
||||
if (zkConfig.isDistributed()) {
|
||||
QuorumPeerMain qp = new QuorumPeerMain();
|
||||
qp.runFromConfig(zkConfig);
|
||||
|
@ -86,6 +85,48 @@ public class HQuorumPeer implements HConstants {
|
|||
}
|
||||
}
|
||||
|
||||
private static void writeMyID(Properties properties) throws UnknownHostException, IOException {
|
||||
HBaseConfiguration conf = new HBaseConfiguration();
|
||||
String myAddress = DNS.getDefaultHost(
|
||||
conf.get("hbase.zookeeper.dns.interface","default"),
|
||||
conf.get("hbase.zookeeper.dns.nameserver","default"));
|
||||
|
||||
long myId = -1;
|
||||
|
||||
for (Entry<Object, Object> entry : properties.entrySet()) {
|
||||
String key = entry.getKey().toString().trim();
|
||||
String value = entry.getValue().toString().trim();
|
||||
if (key.startsWith("server.")) {
|
||||
int dot = key.indexOf('.');
|
||||
long id = Long.parseLong(key.substring(dot + 1));
|
||||
String[] parts = value.split(":");
|
||||
String address = parts[0];
|
||||
if (myAddress.equals(address)) {
|
||||
myId = id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (myId == -1) {
|
||||
throw new IOException("Could not find my address: " + myAddress +
|
||||
" in list of ZooKeeper quorum servers");
|
||||
}
|
||||
|
||||
String dataDirStr = properties.get("dataDir").toString().trim();
|
||||
File dataDir = new File(dataDirStr);
|
||||
if (!dataDir.isDirectory()) {
|
||||
if (!dataDir.mkdirs()) {
|
||||
throw new IOException("Unable to create data dir " + dataDir);
|
||||
}
|
||||
}
|
||||
|
||||
File myIdFile = new File(dataDir, "myid");
|
||||
PrintWriter w = new PrintWriter(myIdFile);
|
||||
w.println(myId);
|
||||
w.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
|
||||
* If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
|
||||
|
@ -193,11 +234,11 @@ public class HQuorumPeer implements HConstants {
|
|||
if (key.startsWith("server.")) {
|
||||
if(conf.get(CLUSTER_DISTRIBUTED).equals(CLUSTER_IS_DISTRIBUTED) &&
|
||||
value.startsWith("localhost")) {
|
||||
String msg = "The server in zoo.cfg cannot be set to localhost " +
|
||||
String msg = "The server in zoo.cfg cannot be set to localhost " +
|
||||
"in a fully-distributed setup because it won't be reachable. " +
|
||||
"See \"Getting Started\" for more information.";
|
||||
LOG.fatal(msg);
|
||||
throw new IOException(msg);
|
||||
LOG.fatal(msg);
|
||||
throw new IOException(msg);
|
||||
}
|
||||
}
|
||||
newValue.append(value.substring(varEnd));
|
||||
|
|
|
@ -0,0 +1,33 @@
|
|||
package org.apache.hadoop.hbase.zookeeper;
|
||||
|
||||
import java.util.Properties;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
|
||||
/**
|
||||
* Tool for reading ZooKeeper servers from HBase XML configuation and producing
|
||||
* a line-by-line list for use by bash scripts.
|
||||
*/
|
||||
public class ZKServerTool implements HConstants {
|
||||
/**
|
||||
* Run the tool.
|
||||
* @param args Command line arguments. First arg is path to zookeepers file.
|
||||
*/
|
||||
public static void main(String args[]) {
|
||||
HBaseConfiguration conf = new HBaseConfiguration();
|
||||
// Note that we do not simply grab the property ZOOKEEPER_QUORUM from
|
||||
// the HBaseConfiguration because the user may be using a zoo.cfg file.
|
||||
Properties zkProps = HQuorumPeer.makeZKProps(conf);
|
||||
for (Entry<Object, Object> entry : zkProps.entrySet()) {
|
||||
String key = entry.getKey().toString().trim();
|
||||
String value = entry.getValue().toString().trim();
|
||||
if (key.startsWith("server.")) {
|
||||
String[] parts = value.split(":");
|
||||
String host = parts[0];
|
||||
System.out.println(host);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -25,7 +25,6 @@ import java.util.Map;
|
|||
import java.util.Properties;
|
||||
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.HBaseTestCase;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
|
||||
|
|
Loading…
Reference in New Issue