HBASE-2327 [EC2] Allocate elastic IP addresses for ZK and master nodes

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@923517 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -462,6 +462,7 @@ Release 0.21.0 - Unreleased
    HBASE-2274  [stargate] filter support: JSON descriptors
    HBASE-2316  Need an ability to run shell tests w/o invoking junit
                (Alexey Kovyrin via Stack)
+   HBASE-2327  [EC2] Allocate elastic IP addresses for ZK and master nodes
 
   OPTIMIZATIONS
    HBASE-410   [testing] Speed up the test suite

contrib/ec2/bin/create-hbase-image

@@ -76,7 +76,7 @@ scp $SSH_OPTS $EC2_PRIVATE_KEY "root@$HOSTNAME:/mnt"
 scp $SSH_OPTS $EC2_CERT "root@$HOSTNAME:/mnt"
 
 # Connect to it
-ssh $SSH_OPTS "root@$HOSTNAME" "sh -c \"INSTANCE_TYPE=$type ARCH=$arch /mnt/create-hbase-image-remote\""
+ssh $SSH_OPTS "root@$HOSTNAME" "sh -c \"INSTANCE_TYPE=$type ARCH=$arch HBASE_URL=$HBASE_URL HADOOP_URL=$HADOOP_URL LZO_URL=$LZO_URL JAVA_URL=$JAVA_URL /mnt/create-hbase-image-remote\""
 
 # Register image
 ec2-register $TOOL_OPTS $S3_BUCKET/hbase-$HBASE_VERSION-$arch.manifest.xml

contrib/ec2/bin/hbase-ec2-env.sh

@@ -36,8 +36,14 @@ EC2_ROOT_SSH_KEY=
 # The version of HBase to use.
 HBASE_VERSION=@HBASE_VERSION@
 
+HBASE_URL=http://hbase.s3.amazonaws.com/hbase/hbase-$HBASE_VERSION.tar.gz
+
 # The version of Hadoop to use.
-HADOOP_VERSION=0.20.1
+HADOOP_VERSION=0.20.2
+
+HADOOP_URL=http://hbase.s3.amazonaws.com/hadoop/hadoop-$HADOOP_VERSION.tar.gz
+
+LZO_URL=http://hbase.s3.amazonaws.com/hadoop/lzo-linux-$HADOOP_VERSION.tar.gz
 
 # The Amazon S3 bucket where the HBase AMI is stored.
 # Change this value only if you are creating your own (private) AMI
@@ -53,14 +59,14 @@ ENABLE_WEB_PORTS=false
 # The repository descriptor will be fetched into /etc/yum/repos.d.
 EXTRA_PACKAGES=
 
-# Use only c1.xlarge unless you know what you are doing
+# Use only m1.large or c1.xlarge unless you know what you are doing
-MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-c1.xlarge}
+MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-m1.large}
 
-# Use only c1.xlarge unless you know what you are doing
+# Use only m1.large or c1.xlarge unless you know what you are doing
-SLAVE_INSTANCE_TYPE=${SLAVE_INSTANCE_TYPE:-c1.xlarge}
+SLAVE_INSTANCE_TYPE=${SLAVE_INSTANCE_TYPE:-m1.large}
 
-# Use only c1.medium unless you know what you are doing
+# Use only m1.small or c1.medium unless you know what you are doing
-ZOO_INSTANCE_TYPE=${ZOO_INSTANCE_TYPE:-c1.medium}
+ZOO_INSTANCE_TYPE=${ZOO_INSTANCE_TYPE:-m1.small}
 
 ############################################################################
 
@@ -89,13 +95,14 @@ TOOL_OPTS=`echo -K "$EC2_PRIVATE_KEY" -C "$EC2_CERT" --request-timeout $REQUEST_
 CLUSTER_MASTER=$CLUSTER-master
 
 # Cached values for a given cluster
-MASTER_PRIVATE_IP_PATH=~/.hbase-private-$CLUSTER_MASTER
+MASTER_IP_PATH=$HOME/.hbase-${CLUSTER_MASTER}-ip
-MASTER_IP_PATH=~/.hbase-$CLUSTER_MASTER
+MASTER_ADDR_PATH=$HOME/.hbase-${CLUSTER_MASTER}-addr
-MASTER_ZONE_PATH=~/.hbase-zone-$CLUSTER_MASTER
+MASTER_ZONE_PATH=$HOME/.hbase-${CLUSTER_MASTER}-zone
 
 # The Zookeeper EC2 group name. CLUSTER is set by calling scripts.
-CLUSTER_ZOOKEEPER=$CLUSTER-zookeeper
+CLUSTER_ZOOKEEPER=$CLUSTER-zk
-ZOOKEEPER_QUORUM_PATH=~/.hbase-quorum-$CLUSTER_ZOOKEEPER
+ZOOKEEPER_QUORUM_PATH=$HOME/.hbase-${CLUSTER_ZOOKEEPER}-quorum
+ZOOKEEPER_ADDR_PATH=$HOME/.hbase-${CLUSTER_ZOOKEEPER}-addrs
 
 # The script to run on instance boot.
 USER_DATA_FILE=hbase-ec2-init-remote.sh
@@ -103,6 +110,8 @@ USER_DATA_FILE=hbase-ec2-init-remote.sh
 # The version number of the installed JDK.
 JAVA_VERSION=1.6.0_17
 
+JAVA_URL=http://hbase.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-@arch@.bin
+
 # SUPPORTED_ARCHITECTURES = ['i386', 'x86_64']
 if [ "$SLAVE_INSTANCE_TYPE" = "m1.small" -o "$SLAVE_INSTANCE_TYPE" = "c1.medium" ]; then
   SLAVE_ARCH='i386'

contrib/ec2/bin/image/create-hbase-image-remote

@@ -30,6 +30,9 @@ arch=$ARCH
 echo "Remote: INSTANCE_TYPE is $type"
 echo "Remote: ARCH is $arch"
 
+# Perform any URL substitutions that must be done at this late stage
+JAVA_URL=`echo $JAVA_URL | sed -e "s/@arch@/$arch/g"`
+
 # Remove sensitive information
 rm -f "$bin"/hbase-ec2-env.sh
 rm -f "$bin"/credentials.sh
@@ -37,7 +40,7 @@ rm -f "$bin"/credentials.sh
 # Install Java
 echo "Downloading and installing java binary."
 cd /usr/local
-wget -nv -O java.bin http://hbase.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-${arch}.bin
+wget -nv -O java.bin $JAVA_URL
 sh java.bin
 rm -f java.bin
 
@@ -46,18 +49,20 @@ echo "Installing rpms."
 yum -y update
 yum -y install rsync lynx screen ganglia-gmetad ganglia-gmond ganglia-web httpd php lzo-devel xfsprogs
 yum -y clean all
+chkconfig --levels 0123456 httpd off
+chkconfig --levels 0123456 gmetad off
+chkconfig --levels 0123456 gmond off
 
 # Install Hadoop
 echo "Installing Hadoop $HADOOP_VERSION."
 cd /usr/local
-wget -nv http://archive.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
+wget -nv $HADOOP_URL
-[ ! -f hadoop-$HADOOP_VERSION.tar.gz ] && wget -nv http://www.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
 tar xzf hadoop-$HADOOP_VERSION.tar.gz
 rm -f hadoop-$HADOOP_VERSION.tar.gz
 
 # Configure Hadoop
 sed -i \
-  -e "s|# export JAVA_HOME=.*|export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}|" \
+  -e "s|# export JAVA_HOME=.*|export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}|"\
   -e 's|# export HADOOP_LOG_DIR=.*|export HADOOP_LOG_DIR=/mnt/hadoop/logs|' \
   -e 's|# export HADOOP_SLAVE_SLEEP=.*|export HADOOP_SLAVE_SLEEP=1|' \
   -e 's|# export HADOOP_OPTS=.*|export HADOOP_OPTS=-server|' \
@@ -66,13 +71,13 @@ sed -i \
 # Install HBase
 echo "Installing HBase $HBASE_VERSION."
 cd /usr/local
-wget -nv http://hbase.s3.amazonaws.com/hbase/hbase-$HBASE_VERSION.tar.gz
+wget -nv $HBASE_URL
 tar xzf hbase-$HBASE_VERSION.tar.gz
 rm -f hbase-$HBASE_VERSION.tar.gz
 
 # Configure HBase
 sed -i \
-  -e "s|# export JAVA_HOME=.*|export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}|" \
+  -e "s|# export JAVA_HOME=.*|export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}|"\
   -e 's|# export HBASE_OPTS=.*|export HBASE_OPTS="$HBASE_OPTS -server -XX:+HeapDumpOnOutOfMemoryError"|' \
   -e 's|# export HBASE_LOG_DIR=.*|export HBASE_LOG_DIR=/mnt/hbase/logs|' \
   -e 's|# export HBASE_SLAVE_SLEEP=.*|export HBASE_SLAVE_SLEEP=1|' \
@@ -97,7 +102,7 @@ echo '    StrictHostKeyChecking no' >> /etc/ssh/ssh_config
 
 # Install LZO
 echo "Installing LZO codec support"
-wget -nv -O /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz http://hbase.s3.amazonaws.com/hbase/lzo-linux-${HADOOP_VERSION}.tar.gz
+wget -nv -O /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz $LZO_URL
 cd /usr/local/hadoop-${HADOOP_VERSION} && tar xzf /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz
 cd /usr/local/hbase-${HBASE_VERSION} && tar xzf /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz
 rm -f /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz

contrib/ec2/bin/launch-hbase-master

@@ -47,14 +47,11 @@ arch=$MASTER_ARCH
 [ -z "$arch" ] && arch=$SLAVE_ARCH
 
 echo "Testing for existing master in group: $CLUSTER"
-MASTER_EC2_HOST=`ec2-describe-instances $TOOL_OPTS | awk '"RESERVATION" == $1 && "'$CLUSTER_MASTER'" == $4, "RESERVATION" == $1 && "'$CLUSTER_MASTER'" != $4'`
+host=`ec2-describe-instances $TOOL_OPTS | awk '"RESERVATION" == $1 && "'$CLUSTER_MASTER'" == $4, "RESERVATION" == $1 && "'$CLUSTER_MASTER'" != $4'`
-MASTER_EC2_HOST=`echo "$MASTER_EC2_HOST" | awk '"INSTANCE" == $1 && "running" == $6 {print $4}'`
+host=`echo "$host" | awk '"INSTANCE" == $1 && "running" == $6 {print $4}'`
 
-if [ ! -z "$MASTER_EC2_HOST" ]; then
+if [ ! -z "$host" ]; then
-  echo "Master already running on: $MASTER_EC2_HOST"
+  echo "Master already running on: $host"
-  MASTER_HOST=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep INSTANCE | grep running | grep $MASTER_EC2_HOST | awk '{print $5}'`
-  echo $MASTER_HOST > $MASTER_PRIVATE_IP_PATH
-  echo $MASTER_EC2_HOST > $MASTER_IP_PATH
   exit 0
 fi
 
@@ -64,41 +61,45 @@ fi
 # Start a master
 echo "Starting master with AMI $AMI_IMAGE (arch $arch)"
 # Substituting zookeeper quorum
-ZOOKEEPER_QUORUM=`cat $ZOOKEEPER_QUORUM_PATH`
+quorum=`cat $ZOOKEEPER_QUORUM_PATH`
-sed -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \
+sed -e "s|%ZOOKEEPER_QUORUM%|$quorum|" \
     -e "s|%NUM_SLAVES%|$NUM_SLAVES|" \
     -e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \
     "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.master
-INSTANCE=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print $2}'`
+
-echo -n "Waiting for instance $INSTANCE to start"
+inst=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print $2}'`
+addr=`ec2-allocate-address $TOOL_OPTS | awk '{print $2}'`
+ec2-associate-address $TOOL_OPTS $addr -i $inst
+echo -n "Waiting for instance $inst to start"
 while true; do
   printf "."
   # get private dns
-  MASTER_HOST=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep running | awk '{print $5}'`
+  host=`ec2-describe-instances $TOOL_OPTS $inst | grep running | awk '{print $5}'`
-  if [ ! -z $MASTER_HOST ]; then
+  if [ ! -z $host ]; then
-    echo " Started as $MASTER_HOST"
+    echo " Started as $host"
     break;
   fi
   sleep 1
 done
 rm -f "$bin"/$USER_DATA_FILE.master
 
-MASTER_EC2_HOST=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep INSTANCE | grep running | grep $MASTER_HOST | awk '{print $4}'`
+# get public (elastic) hostname
-echo $MASTER_HOST > $MASTER_PRIVATE_IP_PATH
+host=`ec2-describe-instances $TOOL_OPTS $inst | grep INSTANCE | grep running | grep $host | awk '{print $4}'`
-echo $MASTER_EC2_HOST > $MASTER_IP_PATH
+echo $host > $MASTER_ADDR_PATH
-MASTER_EC2_ZONE=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep INSTANCE | grep running | grep $MASTER_HOST | awk '{print $11}'`
+echo $addr > $MASTER_IP_PATH
-echo $MASTER_EC2_ZONE > $MASTER_ZONE_PATH
+# get zone
+zone=`ec2-describe-instances $TOOL_OPTS $inst | grep INSTANCE | grep running | grep $host | awk '{print $11}'`
+echo $zone > $MASTER_ZONE_PATH
 
 while true; do
-  REPLY=`ssh $SSH_OPTS "root@$MASTER_EC2_HOST" 'echo "hello"'`
+  REPLY=`ssh $SSH_OPTS "root@$host" 'echo "hello"'`
   if [ ! -z $REPLY ]; then
    break;
   fi
   sleep 5
 done
 
-scp $SSH_OPTS $EC2_ROOT_SSH_KEY "root@$MASTER_EC2_HOST:/root/.ssh/id_rsa"
+scp $SSH_OPTS $EC2_ROOT_SSH_KEY "root@$host:/root/.ssh/id_rsa"
-ssh $SSH_OPTS "root@$MASTER_EC2_HOST" "chmod 600 /root/.ssh/id_rsa"
+ssh $SSH_OPTS "root@$host" "chmod 600 /root/.ssh/id_rsa"
 
-MASTER_IP=`dig +short $MASTER_EC2_HOST`
+echo "Master is $host ($addr) in zone $zone"
-echo "Master is $MASTER_EC2_HOST, ip is $MASTER_IP, zone is $MASTER_EC2_ZONE."

contrib/ec2/bin/launch-hbase-slaves

@@ -36,26 +36,26 @@ bin=`dirname "$0"`
 bin=`cd "$bin"; pwd`
 . "$bin"/hbase-ec2-env.sh
 
-if [ ! -f $MASTER_IP_PATH ]; then
+if [ ! -f $MASTER_ADDR_PATH ]; then
   echo "Must start Cluster Master first!"
   exit 1
 fi
 
-[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$SLAVE_ARCH |grep available | awk '{print $2}'`
+[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$SLAVE_ARCH | grep available | awk '{print $2}'`
 
-MASTER_HOST=`cat $MASTER_PRIVATE_IP_PATH`
+master=`cat $MASTER_ADDR_PATH`
-MASTER_ZONE=`cat $MASTER_ZONE_PATH`
+zone=`cat $MASTER_ZONE_PATH`
-ZOOKEEPER_QUORUM=`cat $ZOOKEEPER_QUORUM_PATH`
+quorum=`cat $ZOOKEEPER_QUORUM_PATH`
 
 # Substituting master hostname and zookeeper quorum
-sed -e "s|%MASTER_HOST%|$MASTER_HOST|" \
+sed -e "s|%MASTER_HOST%|$master|" \
     -e "s|%NUM_SLAVES%|$NUM_SLAVES|" \
-    -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \
+    -e "s|%ZOOKEEPER_QUORUM%|$quorum|" \
     -e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \
     "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.slave
 
 # Start slaves
-echo "Starting $NUM_SLAVES AMI(s) with ID $AMI_IMAGE (arch $SLAVE_ARCH) in group $CLUSTER in zone $MASTER_ZONE"
+echo "Starting $NUM_SLAVES AMI(s) with ID $AMI_IMAGE (arch $SLAVE_ARCH) in group $CLUSTER in zone $zone"
-ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NUM_SLAVES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
+ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NUM_SLAVES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z "$zone" | grep INSTANCE | awk '{print $2}'
 
 rm "$bin"/$USER_DATA_FILE.slave

contrib/ec2/bin/launch-hbase-zookeeper

@@ -41,46 +41,48 @@ arch=$ZOO_ARCH
 [ -z "$arch" ] && arch=$SLAVE_ARCH
 
 # Finding HBase image
-[ -z "$ZOO_AMI_IMAGE" ] && ZOO_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$arch |grep available | awk '{print $2}'`
+[ -z "$ZOO_AMI_IMAGE" ] && ZOO_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$arch | grep available | awk '{print $2}'`
 
 # Start Zookeeper instances
 
 echo "Starting ZooKeeper quorum ensemble."
 
 peers=""
-public_names=""
+peer_addrs=""
-inst=0
+i=0
-while [ $inst -lt $NO_INSTANCES ] ; do
+while [ $i -lt $NO_INSTANCES ] ; do
   echo "Starting an AMI with ID $ZOO_AMI_IMAGE (arch $arch) in group $CLUSTER_ZOOKEEPER"
-  INSTANCE=`ec2-run-instances $ZOO_AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_ZOOKEEPER -k root -t $type | grep INSTANCE | awk '{print $2}'`
+  inst=`ec2-run-instances $ZOO_AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_ZOOKEEPER -k root -t $type | grep INSTANCE | awk '{print $2}'`
-  echo -n "Waiting for instance $INSTANCE to start: "
+  addr=`ec2-allocate-address $TOOL_OPTS | awk '{print $2}'`
+  ec2-associate-address $TOOL_OPTS $addr -i $inst
+  echo -n "Waiting for instance $inst to start: "
   while true; do
     printf "."
-    # get private dns
+    priv=`ec2-describe-instances $TOOL_OPTS $inst | grep running | awk '{print $5}'`
-    priv=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep running | awk '{print $5}'`
     if [ ! -z $priv ]; then
-      echo " Started ZooKeeper instance $INSTANCE as ${priv}"
+      echo " Started ZooKeeper instance $inst as $priv"
       break
     fi
     sleep 1
   done
-  peers="$peers $priv"
+  host=`ec2-describe-instances $TOOL_OPTS $inst | grep INSTANCE | awk '{print $4}'`
-  public=`ec2-describe-instances $TOOL_OPTS $INSTANCE | grep running | awk '{print $4}'`
+  echo "Associated elastic IP $host ($addr) with instance $INSTANCE"
-  echo "    Public DNS name is $public."
+  peers="$peers $host"
-  public_names="$public_names $public"
+  peer_addrs="$peer_addrs $addr"
-  inst=$(($inst + 1))
+  i=$(($i + 1))
 done
 
-ZOOKEEPER_QUORUM=`echo $peers | sed -e 's/ /,/g'`
+quorum=`echo $peers | sed -e 's/ /,/g'`
-echo $ZOOKEEPER_QUORUM  > $ZOOKEEPER_QUORUM_PATH
+echo $quorum  > $ZOOKEEPER_QUORUM_PATH
-echo "ZooKeeper quorum is $ZOOKEEPER_QUORUM."
+echo $peer_addrs  > $ZOOKEEPER_ADDR_PATH
+echo "ZooKeeper quorum is $quorum"
 
 # Start Zookeeper quorum
 
 sleep 10
-echo "Initializing the ZooKeeper quorum ensemble."
+echo "Initializing the ZooKeeper quorum ensemble"
 
-for host in $public_names ; do 
+for host in $peers ; do 
   echo "    $host"
   scp $SSH_OPTS "$bin"/hbase-ec2-init-zookeeper-remote.sh "root@${host}:/var/tmp"
   ssh $SSH_OPTS "root@${host}" "sh -c \"ZOOKEEPER_QUORUM=\"$ZOOKEEPER_QUORUM\" sh /var/tmp/hbase-ec2-init-zookeeper-remote.sh\""

contrib/ec2/bin/terminate-hbase-cluster

@@ -44,3 +44,17 @@ if [ "$answer" != "yes" ]; then
 fi
 
 ec2-terminate-instances $TOOL_OPTS `echo "$HBASE_INSTANCES" | awk '{print $2}'`
+
+# clean up elastic IPs
+
+# master
+ec2-release-address $TOOL_OPTS `cat $MASTER_IP_PATH`
+# zookeeper quorum ensemble
+for addr in `cat $ZOOKEEPER_ADDR_PATH` ; do
+  ec2-release-address $TOOL_OPTS $addr
+done
+
+# clean up state files
+
+rm -f $ZOOKEEPER_ADDR_PATH $ZOOKEEPER_QUORUM_PATH
+rm -f $MASTER_IP_PATH $MASTER_ADDR_PATH $MASTER_ZONE_PATH