Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-2009 [EC2] Support mapreduce 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@883960 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Kyle Purtell 2009-11-25 02:25:30 +00:00
parent 0440aab870
commit 0fa38355cf
3 changed files with 60 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -205,6 +205,7 @@ Release 0.21.0 - Unreleased
HBASE-1985 change HTable.delete(ArrayList) to HTable.delete(List)
HBASE-1958 Remove "# TODO: PUT BACK !!! "${HADOOP_HOME}"/bin/hadoop
dfsadmin -safemode wait"
HBASE-2009 [EC2] Support mapreduce
NEW FEATURES
HBASE-1901 "General" partitioner for "hbase-48" bulk (behind the api, write

79
src/contrib/ec2/bin/hbase-ec2-init-remote.sh
View File

@ -19,7 +19,7 @@ HBASE_HOME=`ls -d /usr/local/hbase-*`
# Hadoop configuration
###############################################################################
cat > $HADOOP_HOME/conf/hadoop-site.xml <<EOF
cat > $HADOOP_HOME/conf/core-site.xml <<EOF
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
@ -29,11 +29,38 @@ cat > $HADOOP_HOME/conf/hadoop-site.xml <<EOF
</property>
<property>
<name>fs.default.name</name>
<value>hdfs://$MASTER_HOST:50001</value>
<value>hdfs://$MASTER_HOST:8020</value>
</property>
</configuration>
EOF
cat > $HADOOP_HOME/conf/hdfs-site.xml <<EOF
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://$MASTER_HOST:8020</value>
</property>
</configuration>
EOF
cat > $HADOOP_HOME/conf/mapred-site.xml <<EOF
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>mapred.job.tracker</name>
<value>$MASTER_HOST:8021</value>
</property>
</configuration>
EOF
# Update classpath to include HBase jars and config
cat >> $HADOOP_HOME/conf/hadoop-env.sh <<EOF
HADOOP_CLASSPATH="$HBASE_HOME/hbase-${HBASE_VERSION}.jar:$HBASE_HOME/lib/AgileJSON-2009-03-30.jar:$HBASE_HOME/lib/json.jar:$HBASE_HOME/lib/zookeeper-3.2.1.jar:$HBASE_HOME/conf"
EOF
# Configure Hadoop for Ganglia
# overwrite hadoop-metrics.properties
cat > $HADOOP_HOME/conf/hadoop-metrics.properties <<EOF
@ -56,21 +83,9 @@ cat > $HBASE_HOME/conf/hbase-site.xml <<EOF
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://$MASTER_HOST:50001</value>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.client.block.write.retries</name>
<value>100</value>
</property>
<property>
<name>hbase.rootdir</name>
<value>hdfs://$MASTER_HOST:50001/hbase</value>
<value>hdfs://$MASTER_HOST:8020/hbase</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
@ -80,10 +95,6 @@ cat > $HBASE_HOME/conf/hbase-site.xml <<EOF
<name>hbase.zookeeper.quorum</name>
<value>$ZOOKEEPER_QUORUM</value>
</property>
<property>
<name>zookeeper.session.timeout</name>
<value>60000</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>100</value>
@ -96,9 +107,31 @@ cat > $HBASE_HOME/conf/hbase-site.xml <<EOF
<name>hbase.hstore.blockingStoreFiles</name>
<value>15</value>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.client.block.write.retries</name>
<value>100</value>
</property>
<property>
<name>zookeeper.session.timeout</name>
<value>60000</value>
</property>
<property>
<name>hbase.tmp.dir</name>
<value>/mnt/hbase</value>
</property>
</configuration>
EOF
# Override JVM options
cat >> $HBASE_HOME/conf/hbase-env.sh <<EOF
export HBASE_MASTER_OPTS="-XX:+UseConcMarkSweepGC -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
export HBASE_REGIONSERVER_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
EOF
# Configure HBase for Ganglia
# overwrite hadoop-metrics.properties
cat > $HBASE_HOME/conf/hadoop-metrics.properties <<EOF
@ -123,7 +156,7 @@ echo "root hard nofile 32768" >> /etc/security/limits.conf
# up epoll limits
# ok if this fails, only valid for kernels 2.6.27+
sysctl -w fs.epoll.max_user_instances=32768
sysctl -w fs.epoll.max_user_instances=32768 > /dev/null 2>&1
mkdir -p /mnt/hadoop/logs
mkdir -p /mnt/hbase/logs
@ -155,7 +188,9 @@ if [ "$IS_MASTER" = "true" ]; then
"$HADOOP_HOME"/bin/hadoop-daemon.sh start datanode
sleep 10
"$HADOOP_HOME"/bin/hadoop-daemon.sh start jobtracker
sleep 10
"$HBASE_HOME"/bin/hbase-daemon.sh start master
@ -174,6 +209,8 @@ else
"$HBASE_HOME"/bin/hbase-daemon.sh start regionserver
"$HADOOP_HOME"/bin/hadoop-daemon.sh start tasktracker
fi
# Run this script on next boot

2
src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh
View File

@ -40,7 +40,7 @@ echo "root hard nofile 32768" >> /etc/security/limits.conf
# up epoll limits
# ok if this fails, only valid for kernels 2.6.27+
sysctl -w fs.epoll.max_user_instance=32768
sysctl -w fs.epoll.max_user_instance=32768 > /dev/null 2>&1
mkdir -p /mnt/hbase/logs
mkdir -p /mnt/hbase/zk