HBASE-2012 [EC2] LZO support
git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@884448 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Kyle Purtell
parent
0fa38355cf
commit
e505eb385c
|
|
@ -206,6 +206,7 @@ Release 0.21.0 - Unreleased
|
|||
HBASE-1958 Remove "# TODO: PUT BACK !!! "${HADOOP_HOME}"/bin/hadoop
|
||||
dfsadmin -safemode wait"
|
||||
HBASE-2009 [EC2] Support mapreduce
|
||||
HBASE-2012 [EC2] LZO support
|
||||
|
||||
NEW FEATURES
|
||||
HBASE-1901 "General" partitioner for "hbase-48" bulk (behind the api, write
|
||||
|
|
|
|||
|
|
@ -18,23 +18,26 @@
|
|||
# Create a HBase AMI.
|
||||
# Inspired by Jonathan Siegel's EC2 script (http://blogsiegel.blogspot.com/2006/08/sandboxing-amazon-ec2.html)
|
||||
|
||||
# allow override of INSTANCE_TYPE from the command line
|
||||
[ ! -z $1 ] && INSTANCE_TYPE=$1
|
||||
# allow override of SLAVE_INSTANCE_TYPE from the command line
|
||||
[ ! -z $1 ] && SLAVE_INSTANCE_TYPE=$1
|
||||
|
||||
# Import variables
|
||||
bin=`dirname "$0"`
|
||||
bin=`cd "$bin"; pwd`
|
||||
. "$bin"/hbase-ec2-env.sh
|
||||
|
||||
echo "INSTANCE_TYPE is $INSTANCE_TYPE."
|
||||
echo "ARCH is $ARCH."
|
||||
type=$SLAVE_INSTANCE_TYPE
|
||||
arch=$SLAVE_ARCH
|
||||
|
||||
AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION | grep $ARCH | grep available | awk '{print $2}'`
|
||||
echo "INSTANCE_TYPE is $type"
|
||||
echo "ARCH is $arch"
|
||||
|
||||
AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION | grep $arch | grep available | awk '{print $2}'`
|
||||
|
||||
[ ! -z $AMI_IMAGE ] && echo "AMI already registered, use: ec2-deregister $AMI_IMAGE" && exit 1
|
||||
|
||||
echo "Starting a AMI with ID $BASE_AMI_IMAGE."
|
||||
OUTPUT=`ec2-run-instances $BASE_AMI_IMAGE $TOOL_OPTS -k root -t $INSTANCE_TYPE`
|
||||
OUTPUT=`ec2-run-instances $BASE_AMI_IMAGE $TOOL_OPTS -k root -t $type`
|
||||
BOOTING_INSTANCE=`echo $OUTPUT | awk '{print $6}'`
|
||||
|
||||
echo "Instance is $BOOTING_INSTANCE."
|
||||
|
|
@ -69,9 +72,9 @@ scp $SSH_OPTS $EC2_PRIVATE_KEY "root@$HOSTNAME:/mnt"
|
|||
scp $SSH_OPTS $EC2_CERT "root@$HOSTNAME:/mnt"
|
||||
|
||||
# Connect to it
|
||||
ssh $SSH_OPTS "root@$HOSTNAME" "sh -c \"INSTANCE_TYPE=$INSTANCE_TYPE /mnt/create-hbase-image-remote\""
|
||||
ssh $SSH_OPTS "root@$HOSTNAME" "sh -c \"INSTANCE_TYPE=$type ARCH=$arch /mnt/create-hbase-image-remote\""
|
||||
|
||||
# Register image
|
||||
ec2-register $TOOL_OPTS $S3_BUCKET/hbase-$HBASE_VERSION-$ARCH.manifest.xml
|
||||
ec2-register $TOOL_OPTS $S3_BUCKET/hbase-$HBASE_VERSION-$arch.manifest.xml
|
||||
|
||||
echo "Terminate with: ec2-terminate-instances $BOOTING_INSTANCE"
|
||||
|
|
|
|||
|
|
@ -45,10 +45,10 @@ SSH_OPTS=`echo -q -i "$PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o ServerAl
|
|||
TOOL_OPTS=`echo -K "$EC2_PRIVATE_KEY" -C "$EC2_CERT"`
|
||||
|
||||
# The version of HBase to use.
|
||||
HBASE_VERSION=0.20.1
|
||||
HBASE_VERSION=0.20.2
|
||||
|
||||
# The version of Hadoop to use.
|
||||
HADOOP_VERSION=$HBASE_VERSION
|
||||
HADOOP_VERSION=0.20.1
|
||||
|
||||
# The Amazon S3 bucket where the HBase AMI is stored.
|
||||
# The default value is for public images, so can be left if you are using running a public image.
|
||||
|
|
@ -64,7 +64,10 @@ ENABLE_WEB_PORTS=false
|
|||
USER_DATA_FILE=hbase-ec2-init-remote.sh
|
||||
|
||||
# Use only c1.xlarge unless you know what you are doing
|
||||
INSTANCE_TYPE=${INSTANCE_TYPE:-c1.xlarge}
|
||||
MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-c1.xlarge}
|
||||
|
||||
# Use only c1.xlarge unless you know what you are doing
|
||||
SLAVE_INSTANCE_TYPE=${SLAVE_INSTANCE_TYPE:-c1.xlarge}
|
||||
|
||||
# Use only c1.medium unless you know what you are doing
|
||||
ZOO_INSTANCE_TYPE=${ZOO_INSTANCE_TYPE:-c1.medium}
|
||||
|
|
@ -89,17 +92,20 @@ ZOOKEEPER_QUORUM_PATH=~/.hbase-quorum-$CLUSTER_ZOOKEEPER
|
|||
JAVA_VERSION=1.6.0_16
|
||||
|
||||
# SUPPORTED_ARCHITECTURES = ['i386', 'x86_64']
|
||||
# The download URL for the Sun JDK. Visit http://java.sun.com/javase/downloads/index.jsp and get the URL for the "Linux self-extracting file".
|
||||
if [ "$INSTANCE_TYPE" = "m1.small" -o "$INSTANCE_TYPE" = "c1.medium" ]; then
|
||||
ARCH='i386'
|
||||
if [ "$SLAVE_INSTANCE_TYPE" = "m1.small" -o "$SLAVE_INSTANCE_TYPE" = "c1.medium" ]; then
|
||||
SLAVE_ARCH='i386'
|
||||
BASE_AMI_IMAGE="ami-48aa4921" # ec2-public-images/fedora-8-i386-base-v1.10.manifest.xml
|
||||
AMI_IMAGE="ami-c644a7af"
|
||||
JAVA_BINARY_URL='http://iridiant.s3.amazonaws.com/jdk/jdk-6u16-linux-i586.bin'
|
||||
else
|
||||
ARCH='x86_64'
|
||||
SLAVE_ARCH='x86_64'
|
||||
BASE_AMI_IMAGE="ami-f61dfd9f" # ec2-public-images/fedora-8-x86_64-base-v1.10.manifest.xml
|
||||
AMI_IMAGE="ami-f244a79b"
|
||||
JAVA_BINARY_URL='http://iridiant.s3.amazonaws.com/jdk/jdk-6u16-linux-x64.bin'
|
||||
fi
|
||||
|
||||
if [ "$MASTER_INSTANCE_TYPE" = "m1.small" -o "$MASTER_INSTANCE_TYPE" = "c1.medium" ]; then
|
||||
MASTER_ARCH='i386'
|
||||
MASTER_AMI_IMAGE="ami-c644a7af"
|
||||
else
|
||||
MASTER_ARCH='x86_64'
|
||||
MASTER_AMI_IMAGE="ami-f244a79b"
|
||||
fi
|
||||
|
||||
if [ "$ZOO_INSTANCE_TYPE" = "m1.small" -o "$ZOO_INSTANCE_TYPE" = "c1.medium" ]; then
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# Create a Hbase AMI. Runs on the EC2 instance.
|
||||
|
||||
# Import variables
|
||||
|
|
@ -23,8 +22,13 @@ bin=`dirname "$0"`
|
|||
bin=`cd "$bin"; pwd`
|
||||
. "$bin"/hbase-ec2-env.sh
|
||||
|
||||
echo "Remote: INSTANCE_TYPE is $INSTANCE_TYPE."
|
||||
echo "Remote: ARCH is $ARCH."
|
||||
type=$INSTANCE_TYPE
|
||||
[ -z "$type" ] && type=$SLAVE_INSTANCE_TYPE
|
||||
arch=$ARCH
|
||||
[ -z "$arch" ] && arch=$SLAVE_ARCH
|
||||
|
||||
echo "Remote: INSTANCE_TYPE is $type"
|
||||
echo "Remote: ARCH is $arch"
|
||||
|
||||
# Remove environment script since it contains sensitive information
|
||||
rm -f "$bin"/hbase-ec2-env.sh
|
||||
|
|
@ -32,14 +36,14 @@ rm -f "$bin"/hbase-ec2-env.sh
|
|||
# Install Java
|
||||
echo "Downloading and installing java binary."
|
||||
cd /usr/local
|
||||
wget -nv -O java.bin $JAVA_BINARY_URL
|
||||
wget -nv -O java.bin http://iridiant.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-${arch}.bin
|
||||
sh java.bin
|
||||
rm -f java.bin
|
||||
|
||||
# Install tools
|
||||
echo "Installing rpms."
|
||||
yum -y update
|
||||
yum -y install rsync lynx screen ganglia-gmetad ganglia-gmond ganglia-web httpd php
|
||||
yum -y install rsync lynx screen ganglia-gmetad ganglia-gmond ganglia-web httpd php lzo-devel
|
||||
yum -y clean all
|
||||
|
||||
# Install Hadoop
|
||||
|
|
@ -89,13 +93,20 @@ rm -f /root/.ssh/authorized_keys
|
|||
# Ensure logging in to new hosts is seamless.
|
||||
echo ' StrictHostKeyChecking no' >> /etc/ssh/ssh_config
|
||||
|
||||
# Install LZO for HBase
|
||||
echo "Installing LZO codec support for HBase"
|
||||
cd /usr/local/hbase-${HBASE_VERSION}
|
||||
wget -nv http://iridiant.s3.amazonaws.com/hbase/lzo-linux-${HADOOP_VERSION}.tar.gz
|
||||
tar xzf /var/tmp/lzo-linux-${HADOOP_VERSION}.tar.gz
|
||||
rm /var/tmp/lzo-linux-${HADOOP_VERSION}.tar.gz
|
||||
|
||||
# Bundle and upload image
|
||||
cd ~root
|
||||
# Don't need to delete .bash_history since it isn't written until exit.
|
||||
df -h
|
||||
ec2-bundle-vol -d /mnt -k /mnt/pk*.pem -c /mnt/cert*.pem -u $AWS_ACCOUNT_ID -s 3072 -p hbase-$HBASE_VERSION-$ARCH -r $ARCH
|
||||
ec2-bundle-vol -d /mnt -k /mnt/pk*.pem -c /mnt/cert*.pem -u $AWS_ACCOUNT_ID -s 3072 -p hbase-$HBASE_VERSION-$arch -r $arch
|
||||
|
||||
ec2-upload-bundle -b $S3_BUCKET -m /mnt/hbase-$HBASE_VERSION-$ARCH.manifest.xml -a $AWS_ACCESS_KEY_ID -s $AWS_SECRET_ACCESS_KEY
|
||||
ec2-upload-bundle -b $S3_BUCKET -m /mnt/hbase-$HBASE_VERSION-$arch.manifest.xml -a $AWS_ACCESS_KEY_ID -s $AWS_SECRET_ACCESS_KEY
|
||||
|
||||
# End
|
||||
echo Done
|
||||
|
|
|
|||
|
|
@ -29,6 +29,11 @@ bin=`dirname "$0"`
|
|||
bin=`cd "$bin"; pwd`
|
||||
. "$bin"/hbase-ec2-env.sh
|
||||
|
||||
type=$MASTER_INSTANCE_TYPE
|
||||
[ -z "$type" ] && type=$SLAVE_INSTANCE_TYPE
|
||||
arch=$MASTER_ARCH
|
||||
[ -z "$arch" ] && arch=$SLAVE_ARCH
|
||||
|
||||
if [ -z $AWS_ACCOUNT_ID ]; then
|
||||
echo "Please set AWS_ACCOUNT_ID in $bin/hbase-ec2-env.sh."
|
||||
exit 1
|
||||
|
|
@ -47,15 +52,15 @@ if [ ! -z "$MASTER_EC2_HOST" ]; then
|
|||
fi
|
||||
|
||||
# Finding HBase image
|
||||
[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep $HBASE_VERSION | grep $ARCH | grep available | awk '{print $2}'`
|
||||
[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep $HBASE_VERSION | grep $arch | grep available | awk '{print $2}'`
|
||||
|
||||
# Start a master
|
||||
echo "Starting master with AMI $AMI_IMAGE (arch $ARCH)"
|
||||
echo "Starting master with AMI $AMI_IMAGE (arch $arch)"
|
||||
# Substituting zookeeper quorum
|
||||
ZOOKEEPER_QUORUM=`cat $ZOOKEEPER_QUORUM_PATH`
|
||||
sed -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \
|
||||
"$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.master
|
||||
INSTANCE=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $INSTANCE_TYPE | grep INSTANCE | awk '{print $2}'`
|
||||
INSTANCE=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print $2}'`
|
||||
echo -n "Waiting for instance $INSTANCE to start"
|
||||
while true; do
|
||||
printf "."
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ if [ ! -f $MASTER_IP_PATH ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep $HBASE_VERSION | grep $ARCH |grep available | awk '{print $2}'`
|
||||
[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep $HBASE_VERSION | grep $SLAVE_ARCH |grep available | awk '{print $2}'`
|
||||
|
||||
MASTER_HOST=`cat $MASTER_PRIVATE_IP_PATH`
|
||||
MASTER_ZONE=`cat $MASTER_ZONE_PATH`
|
||||
|
|
@ -53,7 +53,7 @@ sed -e "s|%MASTER_HOST%|$MASTER_HOST|" \
|
|||
"$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.slave
|
||||
|
||||
# Start slaves
|
||||
echo "Starting $NO_INSTANCES AMI(s) with ID $AMI_IMAGE (arch $ARCH) in group $CLUSTER in zone $MASTER_ZONE"
|
||||
ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NO_INSTANCES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
|
||||
echo "Starting $NO_INSTANCES AMI(s) with ID $AMI_IMAGE (arch $SLAVE_ARCH) in group $CLUSTER in zone $MASTER_ZONE"
|
||||
ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NO_INSTANCES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
|
||||
|
||||
rm "$bin"/$USER_DATA_FILE.slave
|
||||
|
|
|
|||
|
|
@ -36,9 +36,9 @@ bin=`cd "$bin"; pwd`
|
|||
. "$bin"/hbase-ec2-env.sh
|
||||
|
||||
type=$ZOO_INSTANCE_TYPE
|
||||
[ -z "$type" ] && type=$INSTANCE_TYPE
|
||||
[ -z "$type" ] && type=$SLAVE_INSTANCE_TYPE
|
||||
arch=$ZOO_ARCH
|
||||
[ -z "$arch" ] && arch=$ARCH
|
||||
[ -z "$arch" ] && arch=$SLAVE_ARCH
|
||||
|
||||
# Finding HBase image
|
||||
[ -z "$ZOO_AMI_IMAGE" ] && ZOO_AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION | grep $arch |grep available | awk '{print $2}'`
|
||||
|
|
|
|||
Loading…
Reference in New Issue