HBASE-2533 [EC2] ec2-describe-instances returning account number instead of bucket name
git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@943303 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Kyle Purtell
parent
026227e87f
commit
449b5c10b4
|
|
@ -79,6 +79,6 @@ scp $SSH_OPTS $EC2_CERT "root@$HOSTNAME:/mnt"
|
|||
ssh $SSH_OPTS "root@$HOSTNAME" "sh -c \"INSTANCE_TYPE=$type ARCH=$arch HBASE_URL=$HBASE_URL HADOOP_URL=$HADOOP_URL LZO_URL=$LZO_URL JAVA_URL=$JAVA_URL /mnt/create-hbase-image-remote\""
|
||||
|
||||
# Register image
|
||||
ec2-register $TOOL_OPTS $S3_BUCKET/hbase-$HBASE_VERSION-$arch.manifest.xml
|
||||
ec2-register $TOOL_OPTS -n hbase-$REGION-$HBASE_VERSION-$arch $S3_BUCKET/hbase-$HBASE_VERSION-$arch.manifest.xml
|
||||
|
||||
echo "Terminate with: ec2-terminate-instances $BOOTING_INSTANCE"
|
||||
|
|
|
|||
|
|
@ -41,19 +41,21 @@ HBASE_URL=http://hbase.s3.amazonaws.com/hbase/hbase-$HBASE_VERSION.tar.gz
|
|||
# The version of Hadoop to use.
|
||||
HADOOP_VERSION=0.20.2
|
||||
|
||||
# The Amazon S3 bucket where the HBase AMI is stored.
|
||||
REGION=us-east-1
|
||||
#REGION=us-west-1
|
||||
#REGION=eu-west-1
|
||||
#REGION=ap-southeast-1
|
||||
S3_BUCKET=apache-hbase-images-$REGION
|
||||
# Account for bucket
|
||||
# We need this because EC2 is returning account identifiers instead of bucket
|
||||
# names.
|
||||
S3_ACCOUNT=720040977164
|
||||
|
||||
HADOOP_URL=http://hbase.s3.amazonaws.com/hadoop/hadoop-$HADOOP_VERSION.tar.gz
|
||||
|
||||
LZO_URL=http://hbase.s3.amazonaws.com/hadoop/lzo-linux-$HADOOP_VERSION.tar.gz
|
||||
|
||||
# The Amazon S3 bucket where the HBase AMI is stored.
|
||||
# Change this value if you are launching instances in regions other than
|
||||
# us-east-1 (the default).
|
||||
# Change this value otherwise only if you are creating your own (private) AMI
|
||||
# so you can store it in a bucket you own.
|
||||
S3_BUCKET=apache-hbase-images # us-east-1
|
||||
#S3_BUCKET=apache-hbase-images-eu # eu-west-1
|
||||
#S3_BUCKET=apache-hbase-images-us-west-1 # us-west-1
|
||||
|
||||
# Enable public access web interfaces
|
||||
ENABLE_WEB_PORTS=false
|
||||
|
||||
|
|
@ -89,6 +91,8 @@ getCredentialSetting 'EC2_PRIVATE_KEY'
|
|||
getCredentialSetting 'EC2_CERT'
|
||||
getCredentialSetting 'EC2_ROOT_SSH_KEY'
|
||||
|
||||
export EC2_URL=https://$REGION.ec2.amazonaws.com
|
||||
|
||||
# SSH options used when connecting to EC2 instances.
|
||||
SSH_OPTS=`echo -q -i "$EC2_ROOT_SSH_KEY" -o StrictHostKeyChecking=no -o ServerAliveInterval=30`
|
||||
|
||||
|
|
@ -118,7 +122,7 @@ INSTANCES_PATH=$HOME/.hbase-${CLUSTER}-instances
|
|||
USER_DATA_FILE=hbase-ec2-init-remote.sh
|
||||
|
||||
# The version number of the installed JDK.
|
||||
JAVA_VERSION=1.6.0_17
|
||||
JAVA_VERSION=1.6.0_20
|
||||
|
||||
JAVA_URL=http://hbase.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-@arch@.bin
|
||||
|
||||
|
|
|
|||
|
|
@ -40,17 +40,17 @@ HBASE_VERSION=`echo $HBASE_HOME | cut -d '-' -f 2`
|
|||
export USER="root"
|
||||
|
||||
# up file-max
|
||||
sysctl -w fs.file-max=32768
|
||||
sysctl -w fs.file-max=65535
|
||||
|
||||
# up ulimits
|
||||
echo "root soft nofile 32768" >> /etc/security/limits.conf
|
||||
echo "root hard nofile 32768" >> /etc/security/limits.conf
|
||||
echo "root soft nofile 65535" >> /etc/security/limits.conf
|
||||
echo "root hard nofile 65535" >> /etc/security/limits.conf
|
||||
|
||||
# up epoll limits; ok if this fails, only valid for kernels 2.6.27+
|
||||
sysctl -w fs.epoll.max_user_instances=32768 > /dev/null 2>&1
|
||||
sysctl -w fs.epoll.max_user_instances=65535 > /dev/null 2>&1
|
||||
|
||||
# up conntrack_max
|
||||
sysctl -w net.ipv4.netfilter.ip_conntrack_max=65536 > /dev/null 2>&1
|
||||
sysctl -w net.ipv4.netfilter.ip_conntrack_max=65535 > /dev/null 2>&1
|
||||
|
||||
[ ! -f /etc/hosts ] && echo "127.0.0.1 localhost" > /etc/hosts
|
||||
|
||||
|
|
@ -252,10 +252,6 @@ cat > $HBASE_HOME/conf/hbase-site.xml <<EOF
|
|||
<name>dfs.client.block.write.retries</name>
|
||||
<value>100</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.datanode.socket.write.timeout</name>
|
||||
<value>0</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>zookeeper.session.timeout</name>
|
||||
<value>60000</value>
|
||||
|
|
@ -270,8 +266,8 @@ EOF
|
|||
cp $HADOOP_HOME/conf/mapred-site.xml $HBASE_HOME/conf/mapred-site.xml
|
||||
# Override JVM options
|
||||
cat >> $HBASE_HOME/conf/hbase-env.sh <<EOF
|
||||
export HBASE_MASTER_OPTS="-Xmx1000m -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
|
||||
export HBASE_REGIONSERVER_OPTS="-Xmx2000m -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:NewSize=128m -XX:MaxNewSize=128m -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
|
||||
export HBASE_MASTER_OPTS="-Xmx1000m -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
|
||||
export HBASE_REGIONSERVER_OPTS="-Xmx2000m -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:NewSize=128m -XX:MaxNewSize=128m -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
|
||||
EOF
|
||||
# Configure log4j
|
||||
sed -i -e 's/hadoop.hbase=DEBUG/hadoop.hbase=INFO/g' $HBASE_HOME/conf/log4j.properties
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ if [ ! -z "$host" ]; then
|
|||
fi
|
||||
|
||||
# Finding HBase image
|
||||
[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$arch | grep available | awk '{print $2}'`
|
||||
[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep -e $S3_BUCKET -e $S3_ACCOUNT | grep hbase | grep $HBASE_VERSION | grep $arch | grep available | awk '{print $2}'`
|
||||
|
||||
# Start a master
|
||||
echo "Starting master with AMI $AMI_IMAGE (arch $arch)"
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ if [ ! -f $MASTER_ADDR_PATH ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$SLAVE_ARCH | grep available | awk '{print $2}'`
|
||||
[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep -e $S3_BUCKET -e $S3_ACCOUNT | grep hbase | grep $HBASE_VERSION | grep $SLAVE_ARCH | grep available | awk '{print $2}'`
|
||||
|
||||
master=`cat $MASTER_ADDR_PATH`
|
||||
zone=`cat $MASTER_ZONE_PATH`
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ arch=$ZOO_ARCH
|
|||
[ -z "$arch" ] && arch=$SLAVE_ARCH
|
||||
|
||||
# Finding HBase image
|
||||
[ -z "$ZOO_AMI_IMAGE" ] && ZOO_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$arch | grep available | awk '{print $2}'`
|
||||
[ -z "$ZOO_AMI_IMAGE" ] && ZOO_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep -e $S3_BUCKET -e $S3_ACCOUNT | grep hbase | grep $HBASE_VERSION | grep $arch | grep available | awk '{print $2}'`
|
||||
|
||||
# Start Zookeeper instances
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue