HBASE-2533 [EC2] ec2-describe-instances returning account number instead of bucket name

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@943303 13f79535-47bb-0310-9956-ffa450edef68

contrib/ec2/bin/create-hbase-image

@@ -79,6 +79,6 @@ scp $SSH_OPTS $EC2_CERT "root@$HOSTNAME:/mnt"
 ssh $SSH_OPTS "root@$HOSTNAME" "sh -c \"INSTANCE_TYPE=$type ARCH=$arch HBASE_URL=$HBASE_URL HADOOP_URL=$HADOOP_URL LZO_URL=$LZO_URL JAVA_URL=$JAVA_URL /mnt/create-hbase-image-remote\""
 
 # Register image
-ec2-register $TOOL_OPTS $S3_BUCKET/hbase-$HBASE_VERSION-$arch.manifest.xml
+ec2-register $TOOL_OPTS -n hbase-$REGION-$HBASE_VERSION-$arch $S3_BUCKET/hbase-$HBASE_VERSION-$arch.manifest.xml
 
 echo "Terminate with: ec2-terminate-instances $BOOTING_INSTANCE"

contrib/ec2/bin/hbase-ec2-env.sh

@@ -41,19 +41,21 @@ HBASE_URL=http://hbase.s3.amazonaws.com/hbase/hbase-$HBASE_VERSION.tar.gz
 # The version of Hadoop to use.
 HADOOP_VERSION=0.20.2
 
+# The Amazon S3 bucket where the HBase AMI is stored.
+REGION=us-east-1
+#REGION=us-west-1
+#REGION=eu-west-1
+#REGION=ap-southeast-1
+S3_BUCKET=apache-hbase-images-$REGION
+# Account for bucket
+# We need this because EC2 is returning account identifiers instead of bucket
+# names.
+S3_ACCOUNT=720040977164
+
 HADOOP_URL=http://hbase.s3.amazonaws.com/hadoop/hadoop-$HADOOP_VERSION.tar.gz
 
 LZO_URL=http://hbase.s3.amazonaws.com/hadoop/lzo-linux-$HADOOP_VERSION.tar.gz
 
-# The Amazon S3 bucket where the HBase AMI is stored.
-# Change this value if you are launching instances in regions other than 
-# us-east-1 (the default).
-# Change this value otherwise only if you are creating your own (private) AMI
-# so you can store it in a bucket you own.
-S3_BUCKET=apache-hbase-images            # us-east-1
-#S3_BUCKET=apache-hbase-images-eu         # eu-west-1
-#S3_BUCKET=apache-hbase-images-us-west-1  # us-west-1
-
 # Enable public access web interfaces
 ENABLE_WEB_PORTS=false
 
@@ -89,6 +91,8 @@ getCredentialSetting 'EC2_PRIVATE_KEY'
 getCredentialSetting 'EC2_CERT'
 getCredentialSetting 'EC2_ROOT_SSH_KEY'
 
+export EC2_URL=https://$REGION.ec2.amazonaws.com
+
 # SSH options used when connecting to EC2 instances.
 SSH_OPTS=`echo -q -i "$EC2_ROOT_SSH_KEY" -o StrictHostKeyChecking=no -o ServerAliveInterval=30`
 
@@ -118,7 +122,7 @@ INSTANCES_PATH=$HOME/.hbase-${CLUSTER}-instances
 USER_DATA_FILE=hbase-ec2-init-remote.sh
 
 # The version number of the installed JDK.
-JAVA_VERSION=1.6.0_17
+JAVA_VERSION=1.6.0_20
 
 JAVA_URL=http://hbase.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-@arch@.bin
 

contrib/ec2/bin/hbase-ec2-init-remote.sh

@@ -40,17 +40,17 @@ HBASE_VERSION=`echo $HBASE_HOME | cut -d '-' -f 2`
 export USER="root"
 
 # up file-max
-sysctl -w fs.file-max=32768
+sysctl -w fs.file-max=65535
 
 # up ulimits
-echo "root soft nofile 32768" >> /etc/security/limits.conf
+echo "root soft nofile 65535" >> /etc/security/limits.conf
-echo "root hard nofile 32768" >> /etc/security/limits.conf
+echo "root hard nofile 65535" >> /etc/security/limits.conf
 
 # up epoll limits; ok if this fails, only valid for kernels 2.6.27+
-sysctl -w fs.epoll.max_user_instances=32768 > /dev/null 2>&1
+sysctl -w fs.epoll.max_user_instances=65535 > /dev/null 2>&1
 
 # up conntrack_max
-sysctl -w net.ipv4.netfilter.ip_conntrack_max=65536 > /dev/null 2>&1
+sysctl -w net.ipv4.netfilter.ip_conntrack_max=65535 > /dev/null 2>&1
 
 [ ! -f /etc/hosts ] &&  echo "127.0.0.1 localhost" > /etc/hosts
 
@@ -252,10 +252,6 @@ cat > $HBASE_HOME/conf/hbase-site.xml <<EOF
   <name>dfs.client.block.write.retries</name>
   <value>100</value>
 </property>
-<property>
-  <name>dfs.datanode.socket.write.timeout</name>
-  <value>0</value>
-</property>
 <property>
   <name>zookeeper.session.timeout</name>
   <value>60000</value>
@@ -270,8 +266,8 @@ EOF
 cp $HADOOP_HOME/conf/mapred-site.xml $HBASE_HOME/conf/mapred-site.xml
 # Override JVM options
 cat >> $HBASE_HOME/conf/hbase-env.sh <<EOF
-export HBASE_MASTER_OPTS="-Xmx1000m -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
+export HBASE_MASTER_OPTS="-Xmx1000m -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
-export HBASE_REGIONSERVER_OPTS="-Xmx2000m -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:NewSize=128m -XX:MaxNewSize=128m -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
+export HBASE_REGIONSERVER_OPTS="-Xmx2000m -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:NewSize=128m -XX:MaxNewSize=128m -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
 EOF
 # Configure log4j
 sed -i -e 's/hadoop.hbase=DEBUG/hadoop.hbase=INFO/g' $HBASE_HOME/conf/log4j.properties

contrib/ec2/bin/launch-hbase-master

@@ -56,7 +56,7 @@ if [ ! -z "$host" ]; then
 fi
 
 # Finding HBase image
-[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$arch | grep available | awk '{print $2}'`
+[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep -e $S3_BUCKET -e $S3_ACCOUNT | grep hbase | grep $HBASE_VERSION | grep $arch | grep available | awk '{print $2}'`
 
 # Start a master
 echo "Starting master with AMI $AMI_IMAGE (arch $arch)"

contrib/ec2/bin/launch-hbase-slaves

@@ -41,7 +41,7 @@ if [ ! -f $MASTER_ADDR_PATH ]; then
   exit 1
 fi
 
-[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$SLAVE_ARCH | grep available | awk '{print $2}'`
+[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep -e $S3_BUCKET -e $S3_ACCOUNT | grep hbase | grep $HBASE_VERSION | grep $SLAVE_ARCH | grep available | awk '{print $2}'`
 
 master=`cat $MASTER_ADDR_PATH`
 zone=`cat $MASTER_ZONE_PATH`

contrib/ec2/bin/launch-hbase-zookeeper

@@ -41,7 +41,7 @@ arch=$ZOO_ARCH
 [ -z "$arch" ] && arch=$SLAVE_ARCH
 
 # Finding HBase image
-[ -z "$ZOO_AMI_IMAGE" ] && ZOO_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$arch | grep available | awk '{print $2}'`
+[ -z "$ZOO_AMI_IMAGE" ] && ZOO_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep -e $S3_BUCKET -e $S3_ACCOUNT | grep hbase | grep $HBASE_VERSION | grep $arch | grep available | awk '{print $2}'`
 
 # Start Zookeeper instances