diff --git a/CHANGES.txt b/CHANGES.txt index 2a794123bc5..ca47e1d1bb1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -462,6 +462,7 @@ Release 0.21.0 - Unreleased HBASE-2274 [stargate] filter support: JSON descriptors HBASE-2316 Need an ability to run shell tests w/o invoking junit (Alexey Kovyrin via Stack) + HBASE-2327 [EC2] Allocate elastic IP addresses for ZK and master nodes OPTIMIZATIONS HBASE-410 [testing] Speed up the test suite diff --git a/contrib/ec2/bin/create-hbase-image b/contrib/ec2/bin/create-hbase-image index 32f1bb748c0..733e6c3d13b 100755 --- a/contrib/ec2/bin/create-hbase-image +++ b/contrib/ec2/bin/create-hbase-image @@ -76,7 +76,7 @@ scp $SSH_OPTS $EC2_PRIVATE_KEY "root@$HOSTNAME:/mnt" scp $SSH_OPTS $EC2_CERT "root@$HOSTNAME:/mnt" # Connect to it -ssh $SSH_OPTS "root@$HOSTNAME" "sh -c \"INSTANCE_TYPE=$type ARCH=$arch /mnt/create-hbase-image-remote\"" +ssh $SSH_OPTS "root@$HOSTNAME" "sh -c \"INSTANCE_TYPE=$type ARCH=$arch HBASE_URL=$HBASE_URL HADOOP_URL=$HADOOP_URL LZO_URL=$LZO_URL JAVA_URL=$JAVA_URL /mnt/create-hbase-image-remote\"" # Register image ec2-register $TOOL_OPTS $S3_BUCKET/hbase-$HBASE_VERSION-$arch.manifest.xml diff --git a/contrib/ec2/bin/hbase-ec2-env.sh b/contrib/ec2/bin/hbase-ec2-env.sh index 0d8fe5426ee..c2ea5dc9ac6 100644 --- a/contrib/ec2/bin/hbase-ec2-env.sh +++ b/contrib/ec2/bin/hbase-ec2-env.sh @@ -36,8 +36,14 @@ EC2_ROOT_SSH_KEY= # The version of HBase to use. HBASE_VERSION=@HBASE_VERSION@ +HBASE_URL=http://hbase.s3.amazonaws.com/hbase/hbase-$HBASE_VERSION.tar.gz + # The version of Hadoop to use. -HADOOP_VERSION=0.20.1 +HADOOP_VERSION=0.20.2 + +HADOOP_URL=http://hbase.s3.amazonaws.com/hadoop/hadoop-$HADOOP_VERSION.tar.gz + +LZO_URL=http://hbase.s3.amazonaws.com/hadoop/lzo-linux-$HADOOP_VERSION.tar.gz # The Amazon S3 bucket where the HBase AMI is stored. # Change this value only if you are creating your own (private) AMI @@ -53,14 +59,14 @@ ENABLE_WEB_PORTS=false # The repository descriptor will be fetched into /etc/yum/repos.d. EXTRA_PACKAGES= -# Use only c1.xlarge unless you know what you are doing -MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-c1.xlarge} +# Use only m1.large or c1.xlarge unless you know what you are doing +MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-m1.large} -# Use only c1.xlarge unless you know what you are doing -SLAVE_INSTANCE_TYPE=${SLAVE_INSTANCE_TYPE:-c1.xlarge} +# Use only m1.large or c1.xlarge unless you know what you are doing +SLAVE_INSTANCE_TYPE=${SLAVE_INSTANCE_TYPE:-m1.large} -# Use only c1.medium unless you know what you are doing -ZOO_INSTANCE_TYPE=${ZOO_INSTANCE_TYPE:-c1.medium} +# Use only m1.small or c1.medium unless you know what you are doing +ZOO_INSTANCE_TYPE=${ZOO_INSTANCE_TYPE:-m1.small} ############################################################################ @@ -89,13 +95,14 @@ TOOL_OPTS=`echo -K "$EC2_PRIVATE_KEY" -C "$EC2_CERT" --request-timeout $REQUEST_ CLUSTER_MASTER=$CLUSTER-master # Cached values for a given cluster -MASTER_PRIVATE_IP_PATH=~/.hbase-private-$CLUSTER_MASTER -MASTER_IP_PATH=~/.hbase-$CLUSTER_MASTER -MASTER_ZONE_PATH=~/.hbase-zone-$CLUSTER_MASTER +MASTER_IP_PATH=$HOME/.hbase-${CLUSTER_MASTER}-ip +MASTER_ADDR_PATH=$HOME/.hbase-${CLUSTER_MASTER}-addr +MASTER_ZONE_PATH=$HOME/.hbase-${CLUSTER_MASTER}-zone # The Zookeeper EC2 group name. CLUSTER is set by calling scripts. -CLUSTER_ZOOKEEPER=$CLUSTER-zookeeper -ZOOKEEPER_QUORUM_PATH=~/.hbase-quorum-$CLUSTER_ZOOKEEPER +CLUSTER_ZOOKEEPER=$CLUSTER-zk +ZOOKEEPER_QUORUM_PATH=$HOME/.hbase-${CLUSTER_ZOOKEEPER}-quorum +ZOOKEEPER_ADDR_PATH=$HOME/.hbase-${CLUSTER_ZOOKEEPER}-addrs # The script to run on instance boot. USER_DATA_FILE=hbase-ec2-init-remote.sh @@ -103,6 +110,8 @@ USER_DATA_FILE=hbase-ec2-init-remote.sh # The version number of the installed JDK. JAVA_VERSION=1.6.0_17 +JAVA_URL=http://hbase.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-@arch@.bin + # SUPPORTED_ARCHITECTURES = ['i386', 'x86_64'] if [ "$SLAVE_INSTANCE_TYPE" = "m1.small" -o "$SLAVE_INSTANCE_TYPE" = "c1.medium" ]; then SLAVE_ARCH='i386' diff --git a/contrib/ec2/bin/image/create-hbase-image-remote b/contrib/ec2/bin/image/create-hbase-image-remote index fe8786dbcff..e4a9d7e911c 100755 --- a/contrib/ec2/bin/image/create-hbase-image-remote +++ b/contrib/ec2/bin/image/create-hbase-image-remote @@ -30,6 +30,9 @@ arch=$ARCH echo "Remote: INSTANCE_TYPE is $type" echo "Remote: ARCH is $arch" +# Perform any URL substitutions that must be done at this late stage +JAVA_URL=`echo $JAVA_URL | sed -e "s/@arch@/$arch/g"` + # Remove sensitive information rm -f "$bin"/hbase-ec2-env.sh rm -f "$bin"/credentials.sh @@ -37,7 +40,7 @@ rm -f "$bin"/credentials.sh # Install Java echo "Downloading and installing java binary." cd /usr/local -wget -nv -O java.bin http://hbase.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-${arch}.bin +wget -nv -O java.bin $JAVA_URL sh java.bin rm -f java.bin @@ -46,18 +49,20 @@ echo "Installing rpms." yum -y update yum -y install rsync lynx screen ganglia-gmetad ganglia-gmond ganglia-web httpd php lzo-devel xfsprogs yum -y clean all +chkconfig --levels 0123456 httpd off +chkconfig --levels 0123456 gmetad off +chkconfig --levels 0123456 gmond off # Install Hadoop echo "Installing Hadoop $HADOOP_VERSION." cd /usr/local -wget -nv http://archive.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz -[ ! -f hadoop-$HADOOP_VERSION.tar.gz ] && wget -nv http://www.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz +wget -nv $HADOOP_URL tar xzf hadoop-$HADOOP_VERSION.tar.gz rm -f hadoop-$HADOOP_VERSION.tar.gz # Configure Hadoop sed -i \ - -e "s|# export JAVA_HOME=.*|export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}|" \ + -e "s|# export JAVA_HOME=.*|export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}|"\ -e 's|# export HADOOP_LOG_DIR=.*|export HADOOP_LOG_DIR=/mnt/hadoop/logs|' \ -e 's|# export HADOOP_SLAVE_SLEEP=.*|export HADOOP_SLAVE_SLEEP=1|' \ -e 's|# export HADOOP_OPTS=.*|export HADOOP_OPTS=-server|' \ @@ -66,13 +71,13 @@ sed -i \ # Install HBase echo "Installing HBase $HBASE_VERSION." cd /usr/local -wget -nv http://hbase.s3.amazonaws.com/hbase/hbase-$HBASE_VERSION.tar.gz +wget -nv $HBASE_URL tar xzf hbase-$HBASE_VERSION.tar.gz rm -f hbase-$HBASE_VERSION.tar.gz # Configure HBase sed -i \ - -e "s|# export JAVA_HOME=.*|export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}|" \ + -e "s|# export JAVA_HOME=.*|export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}|"\ -e 's|# export HBASE_OPTS=.*|export HBASE_OPTS="$HBASE_OPTS -server -XX:+HeapDumpOnOutOfMemoryError"|' \ -e 's|# export HBASE_LOG_DIR=.*|export HBASE_LOG_DIR=/mnt/hbase/logs|' \ -e 's|# export HBASE_SLAVE_SLEEP=.*|export HBASE_SLAVE_SLEEP=1|' \ @@ -97,7 +102,7 @@ echo ' StrictHostKeyChecking no' >> /etc/ssh/ssh_config # Install LZO echo "Installing LZO codec support" -wget -nv -O /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz http://hbase.s3.amazonaws.com/hbase/lzo-linux-${HADOOP_VERSION}.tar.gz +wget -nv -O /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz $LZO_URL cd /usr/local/hadoop-${HADOOP_VERSION} && tar xzf /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz cd /usr/local/hbase-${HBASE_VERSION} && tar xzf /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz rm -f /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz diff --git a/contrib/ec2/bin/launch-hbase-master b/contrib/ec2/bin/launch-hbase-master index 0099cc6b04e..f16bf507379 100755 --- a/contrib/ec2/bin/launch-hbase-master +++ b/contrib/ec2/bin/launch-hbase-master @@ -47,14 +47,11 @@ arch=$MASTER_ARCH [ -z "$arch" ] && arch=$SLAVE_ARCH echo "Testing for existing master in group: $CLUSTER" -MASTER_EC2_HOST=`ec2-describe-instances $TOOL_OPTS | awk '"RESERVATION" == $1 && "'$CLUSTER_MASTER'" == $4, "RESERVATION" == $1 && "'$CLUSTER_MASTER'" != $4'` -MASTER_EC2_HOST=`echo "$MASTER_EC2_HOST" | awk '"INSTANCE" == $1 && "running" == $6 {print $4}'` +host=`ec2-describe-instances $TOOL_OPTS | awk '"RESERVATION" == $1 && "'$CLUSTER_MASTER'" == $4, "RESERVATION" == $1 && "'$CLUSTER_MASTER'" != $4'` +host=`echo "$host" | awk '"INSTANCE" == $1 && "running" == $6 {print $4}'` -if [ ! -z "$MASTER_EC2_HOST" ]; then - echo "Master already running on: $MASTER_EC2_HOST" - MASTER_HOST=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep INSTANCE | grep running | grep $MASTER_EC2_HOST | awk '{print $5}'` - echo $MASTER_HOST > $MASTER_PRIVATE_IP_PATH - echo $MASTER_EC2_HOST > $MASTER_IP_PATH +if [ ! -z "$host" ]; then + echo "Master already running on: $host" exit 0 fi @@ -64,41 +61,45 @@ fi # Start a master echo "Starting master with AMI $AMI_IMAGE (arch $arch)" # Substituting zookeeper quorum -ZOOKEEPER_QUORUM=`cat $ZOOKEEPER_QUORUM_PATH` -sed -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \ +quorum=`cat $ZOOKEEPER_QUORUM_PATH` +sed -e "s|%ZOOKEEPER_QUORUM%|$quorum|" \ -e "s|%NUM_SLAVES%|$NUM_SLAVES|" \ -e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \ "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.master -INSTANCE=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print $2}'` -echo -n "Waiting for instance $INSTANCE to start" + +inst=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print $2}'` +addr=`ec2-allocate-address $TOOL_OPTS | awk '{print $2}'` +ec2-associate-address $TOOL_OPTS $addr -i $inst +echo -n "Waiting for instance $inst to start" while true; do printf "." # get private dns - MASTER_HOST=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep running | awk '{print $5}'` - if [ ! -z $MASTER_HOST ]; then - echo " Started as $MASTER_HOST" + host=`ec2-describe-instances $TOOL_OPTS $inst | grep running | awk '{print $5}'` + if [ ! -z $host ]; then + echo " Started as $host" break; fi sleep 1 done rm -f "$bin"/$USER_DATA_FILE.master -MASTER_EC2_HOST=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep INSTANCE | grep running | grep $MASTER_HOST | awk '{print $4}'` -echo $MASTER_HOST > $MASTER_PRIVATE_IP_PATH -echo $MASTER_EC2_HOST > $MASTER_IP_PATH -MASTER_EC2_ZONE=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep INSTANCE | grep running | grep $MASTER_HOST | awk '{print $11}'` -echo $MASTER_EC2_ZONE > $MASTER_ZONE_PATH +# get public (elastic) hostname +host=`ec2-describe-instances $TOOL_OPTS $inst | grep INSTANCE | grep running | grep $host | awk '{print $4}'` +echo $host > $MASTER_ADDR_PATH +echo $addr > $MASTER_IP_PATH +# get zone +zone=`ec2-describe-instances $TOOL_OPTS $inst | grep INSTANCE | grep running | grep $host | awk '{print $11}'` +echo $zone > $MASTER_ZONE_PATH while true; do - REPLY=`ssh $SSH_OPTS "root@$MASTER_EC2_HOST" 'echo "hello"'` + REPLY=`ssh $SSH_OPTS "root@$host" 'echo "hello"'` if [ ! -z $REPLY ]; then break; fi sleep 5 done -scp $SSH_OPTS $EC2_ROOT_SSH_KEY "root@$MASTER_EC2_HOST:/root/.ssh/id_rsa" -ssh $SSH_OPTS "root@$MASTER_EC2_HOST" "chmod 600 /root/.ssh/id_rsa" +scp $SSH_OPTS $EC2_ROOT_SSH_KEY "root@$host:/root/.ssh/id_rsa" +ssh $SSH_OPTS "root@$host" "chmod 600 /root/.ssh/id_rsa" -MASTER_IP=`dig +short $MASTER_EC2_HOST` -echo "Master is $MASTER_EC2_HOST, ip is $MASTER_IP, zone is $MASTER_EC2_ZONE." +echo "Master is $host ($addr) in zone $zone" diff --git a/contrib/ec2/bin/launch-hbase-slaves b/contrib/ec2/bin/launch-hbase-slaves index 4e70a8d5503..1884f1a2be5 100755 --- a/contrib/ec2/bin/launch-hbase-slaves +++ b/contrib/ec2/bin/launch-hbase-slaves @@ -36,26 +36,26 @@ bin=`dirname "$0"` bin=`cd "$bin"; pwd` . "$bin"/hbase-ec2-env.sh -if [ ! -f $MASTER_IP_PATH ]; then +if [ ! -f $MASTER_ADDR_PATH ]; then echo "Must start Cluster Master first!" exit 1 fi -[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$SLAVE_ARCH |grep available | awk '{print $2}'` +[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$SLAVE_ARCH | grep available | awk '{print $2}'` -MASTER_HOST=`cat $MASTER_PRIVATE_IP_PATH` -MASTER_ZONE=`cat $MASTER_ZONE_PATH` -ZOOKEEPER_QUORUM=`cat $ZOOKEEPER_QUORUM_PATH` +master=`cat $MASTER_ADDR_PATH` +zone=`cat $MASTER_ZONE_PATH` +quorum=`cat $ZOOKEEPER_QUORUM_PATH` # Substituting master hostname and zookeeper quorum -sed -e "s|%MASTER_HOST%|$MASTER_HOST|" \ +sed -e "s|%MASTER_HOST%|$master|" \ -e "s|%NUM_SLAVES%|$NUM_SLAVES|" \ - -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \ + -e "s|%ZOOKEEPER_QUORUM%|$quorum|" \ -e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \ "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.slave # Start slaves -echo "Starting $NUM_SLAVES AMI(s) with ID $AMI_IMAGE (arch $SLAVE_ARCH) in group $CLUSTER in zone $MASTER_ZONE" -ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NUM_SLAVES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}' +echo "Starting $NUM_SLAVES AMI(s) with ID $AMI_IMAGE (arch $SLAVE_ARCH) in group $CLUSTER in zone $zone" +ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NUM_SLAVES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z "$zone" | grep INSTANCE | awk '{print $2}' rm "$bin"/$USER_DATA_FILE.slave diff --git a/contrib/ec2/bin/launch-hbase-zookeeper b/contrib/ec2/bin/launch-hbase-zookeeper index d7fe961ced6..10f033104f2 100755 --- a/contrib/ec2/bin/launch-hbase-zookeeper +++ b/contrib/ec2/bin/launch-hbase-zookeeper @@ -41,46 +41,48 @@ arch=$ZOO_ARCH [ -z "$arch" ] && arch=$SLAVE_ARCH # Finding HBase image -[ -z "$ZOO_AMI_IMAGE" ] && ZOO_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$arch |grep available | awk '{print $2}'` +[ -z "$ZOO_AMI_IMAGE" ] && ZOO_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$arch | grep available | awk '{print $2}'` # Start Zookeeper instances echo "Starting ZooKeeper quorum ensemble." peers="" -public_names="" -inst=0 -while [ $inst -lt $NO_INSTANCES ] ; do +peer_addrs="" +i=0 +while [ $i -lt $NO_INSTANCES ] ; do echo "Starting an AMI with ID $ZOO_AMI_IMAGE (arch $arch) in group $CLUSTER_ZOOKEEPER" - INSTANCE=`ec2-run-instances $ZOO_AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_ZOOKEEPER -k root -t $type | grep INSTANCE | awk '{print $2}'` - echo -n "Waiting for instance $INSTANCE to start: " + inst=`ec2-run-instances $ZOO_AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_ZOOKEEPER -k root -t $type | grep INSTANCE | awk '{print $2}'` + addr=`ec2-allocate-address $TOOL_OPTS | awk '{print $2}'` + ec2-associate-address $TOOL_OPTS $addr -i $inst + echo -n "Waiting for instance $inst to start: " while true; do printf "." - # get private dns - priv=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep running | awk '{print $5}'` + priv=`ec2-describe-instances $TOOL_OPTS $inst | grep running | awk '{print $5}'` if [ ! -z $priv ]; then - echo " Started ZooKeeper instance $INSTANCE as ${priv}" + echo " Started ZooKeeper instance $inst as $priv" break fi sleep 1 done - peers="$peers $priv" - public=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep running | awk '{print $4}'` - echo " Public DNS name is $public." - public_names="$public_names $public" - inst=$(($inst + 1)) + host=`ec2-describe-instances $TOOL_OPTS $inst | grep INSTANCE | awk '{print $4}'` + echo "Associated elastic IP $host ($addr) with instance $INSTANCE" + peers="$peers $host" + peer_addrs="$peer_addrs $addr" + i=$(($i + 1)) done -ZOOKEEPER_QUORUM=`echo $peers | sed -e 's/ /,/g'` -echo $ZOOKEEPER_QUORUM > $ZOOKEEPER_QUORUM_PATH -echo "ZooKeeper quorum is $ZOOKEEPER_QUORUM." +quorum=`echo $peers | sed -e 's/ /,/g'` +echo $quorum > $ZOOKEEPER_QUORUM_PATH +echo $peer_addrs > $ZOOKEEPER_ADDR_PATH +echo "ZooKeeper quorum is $quorum" # Start Zookeeper quorum sleep 10 -echo "Initializing the ZooKeeper quorum ensemble." +echo "Initializing the ZooKeeper quorum ensemble" -for host in $public_names ; do +for host in $peers ; do echo " $host" scp $SSH_OPTS "$bin"/hbase-ec2-init-zookeeper-remote.sh "root@${host}:/var/tmp" ssh $SSH_OPTS "root@${host}" "sh -c \"ZOOKEEPER_QUORUM=\"$ZOOKEEPER_QUORUM\" sh /var/tmp/hbase-ec2-init-zookeeper-remote.sh\"" diff --git a/contrib/ec2/bin/terminate-hbase-cluster b/contrib/ec2/bin/terminate-hbase-cluster index 58ab54bc8ab..f5d481a11c8 100755 --- a/contrib/ec2/bin/terminate-hbase-cluster +++ b/contrib/ec2/bin/terminate-hbase-cluster @@ -44,3 +44,17 @@ if [ "$answer" != "yes" ]; then fi ec2-terminate-instances $TOOL_OPTS `echo "$HBASE_INSTANCES" | awk '{print $2}'` + +# clean up elastic IPs + +# master +ec2-release-address $TOOL_OPTS `cat $MASTER_IP_PATH` +# zookeeper quorum ensemble +for addr in `cat $ZOOKEEPER_ADDR_PATH` ; do + ec2-release-address $TOOL_OPTS $addr +done + +# clean up state files + +rm -f $ZOOKEEPER_ADDR_PATH $ZOOKEEPER_QUORUM_PATH +rm -f $MASTER_IP_PATH $MASTER_ADDR_PATH $MASTER_ZONE_PATH