diff --git a/src/contrib/ec2/bin/hbase-ec2-env.sh b/src/contrib/ec2/bin/hbase-ec2-env.sh index 37234292c87..71329bc4a18 100644 --- a/src/contrib/ec2/bin/hbase-ec2-env.sh +++ b/src/contrib/ec2/bin/hbase-ec2-env.sh @@ -48,6 +48,12 @@ S3_BUCKET=iridiant-bundles # Enable public access web interfaces ENABLE_WEB_PORTS=false +# Extra packages +# Allows you to add a private Yum repo and pull packages from it as your +# instances boot up. Format is ... +# The repository descriptor will be fetched into /etc/yum/repos.d. +EXTRA_PACKAGES= + # Use only c1.xlarge unless you know what you are doing MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-c1.xlarge} diff --git a/src/contrib/ec2/bin/hbase-ec2-init-remote.sh b/src/contrib/ec2/bin/hbase-ec2-init-remote.sh index c66e1b4a38e..ae74616d401 100644 --- a/src/contrib/ec2/bin/hbase-ec2-init-remote.sh +++ b/src/contrib/ec2/bin/hbase-ec2-init-remote.sh @@ -1,12 +1,11 @@ #!/usr/bin/env bash -############################################################################### # Script that is run on each EC2 instance on boot. It is passed in the EC2 user # data, so should not exceed 16K in size. -############################################################################### -MASTER_HOST=%MASTER_HOST% -ZOOKEEPER_QUORUM=%ZOOKEEPER_QUORUM% +MASTER_HOST="%MASTER_HOST%" +ZOOKEEPER_QUORUM="%ZOOKEEPER_QUORUM%" +EXTRA_PACKAGES="%EXTRA_PACKAGES%" SECURITY_GROUPS=`wget -q -O - http://169.254.169.254/latest/meta-data/security-groups` IS_MASTER=`echo $SECURITY_GROUPS | awk '{ a = match ($0, "-master$"); if (a) print "true"; else print "false"; }'` if [ "$IS_MASTER" = "true" ]; then @@ -17,9 +16,7 @@ HADOOP_VERSION=`echo $HADOOP_HOME | cut -d '-' -f 2` HBASE_HOME=`ls -d /usr/local/hbase-*` HBASE_VERSION=`echo $HBASE_HOME | cut -d '-' -f 2` -############################################################################### # Hadoop configuration -############################################################################### cat > $HADOOP_HOME/conf/core-site.xml < @@ -76,7 +73,6 @@ HADOOP_CLASSPATH="$HBASE_HOME/hbase-${HBASE_VERSION}.jar:$HBASE_HOME/lib/AgileJS EOF # Configure Hadoop for Ganglia -# overwrite hadoop-metrics.properties cat > $HADOOP_HOME/conf/hadoop-metrics.properties < $HBASE_HOME/conf/hbase-site.xml < @@ -147,7 +141,6 @@ export HBASE_REGIONSERVER_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupan EOF # Configure HBase for Ganglia -# overwrite hadoop-metrics.properties cat > $HBASE_HOME/conf/hadoop-metrics.properties <> /etc/security/limits.conf echo "root hard nofile 32768" >> /etc/security/limits.conf -# up epoll limits -# ok if this fails, only valid for kernels 2.6.27+ +# up epoll limits; ok if this fails, only valid for kernels 2.6.27+ sysctl -w fs.epoll.max_user_instances=32768 > /dev/null 2>&1 -mkdir -p /mnt/hadoop/logs -mkdir -p /mnt/hbase/logs +mkdir -p /mnt/hadoop/logs /mnt/hbase/logs [ ! -f /etc/hosts ] && echo "127.0.0.1 localhost" > /etc/hosts -# not set on boot export USER="root" +if [ "$EXTRA_PACKAGES" != "" ] ; then + # format should be ... + # this will only work with bash + pkg=( $EXTRA_PACKAGES ) + wget -nv -O /etc/yum.repos.d/user.repo ${pkg[0]} + yum -y update yum + yum -y install ${pkg[@]:1} +fi + if [ "$IS_MASTER" = "true" ]; then # MASTER # Prep Ganglia @@ -199,13 +194,9 @@ if [ "$IS_MASTER" = "true" ]; then [ ! -e /mnt/hadoop/dfs ] && "$HADOOP_HOME"/bin/hadoop namenode -format "$HADOOP_HOME"/bin/hadoop-daemon.sh start namenode - "$HADOOP_HOME"/bin/hadoop-daemon.sh start datanode - "$HADOOP_HOME"/bin/hadoop-daemon.sh start jobtracker - sleep 10 - "$HBASE_HOME"/bin/hbase-daemon.sh start master else @@ -218,11 +209,8 @@ else -e "s|\(udp_send_channel {\)|\1\n host=$MASTER_HOST|" \ /etc/gmond.conf service gmond start - "$HADOOP_HOME"/bin/hadoop-daemon.sh start datanode - "$HBASE_HOME"/bin/hbase-daemon.sh start regionserver - "$HADOOP_HOME"/bin/hadoop-daemon.sh start tasktracker fi diff --git a/src/contrib/ec2/bin/image/create-hbase-image-remote b/src/contrib/ec2/bin/image/create-hbase-image-remote index 03cc259f5de..b545b4bea02 100755 --- a/src/contrib/ec2/bin/image/create-hbase-image-remote +++ b/src/contrib/ec2/bin/image/create-hbase-image-remote @@ -73,7 +73,7 @@ rm -f hbase-$HBASE_VERSION.tar.gz # Configure HBase sed -i \ -e "s|# export JAVA_HOME=.*|export JAVA_HOME=/usr/local/jdk${JAVA_VERSION}|" \ - -e 's|# export HBASE_OPTS=.*|export HBASE_OPTS="$HBASE_OPTS -server -XX:+UseConcMarkSweepGC -XX:+DoEscapeAnalysis -XX:+AggressiveOpts"|' \ + -e 's|# export HBASE_OPTS=.*|export HBASE_OPTS="$HBASE_OPTS -server -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=90 -XX:NewSize=64m -XX:MaxNewSize=64m -XX:+DoEscapeAnalysis -XX:+AggressiveOpts"|' \ -e 's|# export HBASE_LOG_DIR=.*|export HBASE_LOG_DIR=/mnt/hbase/logs|' \ -e 's|# export HBASE_SLAVE_SLEEP=.*|export HBASE_SLAVE_SLEEP=1|' \ /usr/local/hbase-$HBASE_VERSION/conf/hbase-env.sh @@ -94,12 +94,12 @@ rm -f /root/.ssh/authorized_keys # Ensure logging in to new hosts is seamless. echo ' StrictHostKeyChecking no' >> /etc/ssh/ssh_config -# Install LZO for HBase -echo "Installing LZO codec support for HBase" -cd /usr/local/hbase-${HBASE_VERSION} -wget -nv http://iridiant.s3.amazonaws.com/hbase/lzo-linux-${HADOOP_VERSION}.tar.gz -tar xzf lzo-linux-${HADOOP_VERSION}.tar.gz -rm lzo-linux-${HADOOP_VERSION}.tar.gz +# Install LZO +echo "Installing LZO codec support" +wget -nv -O /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz http://iridiant.s3.amazonaws.com/hbase/lzo-linux-${HADOOP_VERSION}.tar.gz +cd /usr/local/hadoop-${HADOOP_VERSION} && tar xzf /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz +cd /usr/local/hbase-${HBASE_VERSION} && tar xzf /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz +rm -f /tmp/lzo-linux-${HADOOP_VERSION}.tar.gz # Bundle and upload image cd ~root diff --git a/src/contrib/ec2/bin/launch-hbase-master b/src/contrib/ec2/bin/launch-hbase-master index 72e2be51a04..bfaaa49a6ce 100755 --- a/src/contrib/ec2/bin/launch-hbase-master +++ b/src/contrib/ec2/bin/launch-hbase-master @@ -59,6 +59,7 @@ echo "Starting master with AMI $AMI_IMAGE (arch $arch)" # Substituting zookeeper quorum ZOOKEEPER_QUORUM=`cat $ZOOKEEPER_QUORUM_PATH` sed -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \ + -e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \ "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.master INSTANCE=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print $2}'` echo -n "Waiting for instance $INSTANCE to start" diff --git a/src/contrib/ec2/bin/launch-hbase-slaves b/src/contrib/ec2/bin/launch-hbase-slaves index 1c89ebff9e5..d36e300664b 100755 --- a/src/contrib/ec2/bin/launch-hbase-slaves +++ b/src/contrib/ec2/bin/launch-hbase-slaves @@ -50,6 +50,7 @@ ZOOKEEPER_QUORUM=`cat $ZOOKEEPER_QUORUM_PATH` # Substituting master hostname and zookeeper quorum sed -e "s|%MASTER_HOST%|$MASTER_HOST|" \ -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \ + -e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \ "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.slave # Start slaves