From dec6fcdb007a78a71854d3dd76b65d53b9f39235 Mon Sep 17 00:00:00 2001 From: Owen O'Malley Date: Fri, 27 May 2011 16:35:02 +0000 Subject: [PATCH] HADOOP-6255. Create RPM and Debian packages for common. Changes deployment layout to be consistent across the binary tgz, rpm, and deb. Adds setup scripts for easy one node cluster configuration and user creation. (Eric Yang via omalley) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1128385 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 9 + bin/hadoop | 13 +- bin/hadoop-config.sh | 117 ++++---- bin/hadoop-daemon.sh | 12 +- bin/hadoop-daemons.sh | 4 +- bin/rcc | 26 +- bin/slaves.sh | 4 +- bin/start-all.sh | 2 +- bin/stop-all.sh | 2 +- build.xml | 206 +++++++++++-- conf/hadoop-env.sh.template | 25 +- ivy.xml | 10 + ivy/libraries.properties | 1 + .../content/xdocs/cluster_setup.xml | 4 +- .../content/xdocs/commands_manual.xml | 2 +- .../content/xdocs/hod_admin_guide.xml | 2 +- .../content/xdocs/mapred_tutorial.xml | 4 +- .../content/xdocs/quickstart.xml | 2 +- .../documentation/content/xdocs/streaming.xml | 24 +- .../content/xdocs/cluster_setup.xml | 8 +- .../content/xdocs/commands_manual.xml | 2 +- .../content/xdocs/deployment_layout.xml | 147 +++++++++ .../content/xdocs/single_node_setup.xml | 2 +- .../src/documentation/content/xdocs/site.xml | 1 + src/native/Makefile.am | 2 +- src/native/lib/Makefile.am | 2 +- src/native/packageNativeHadoop.sh | 22 +- .../hadoop/io/compress/zlib/Makefile.am | 4 +- src/packages/deb/hadoop.control/conffile | 15 + src/packages/deb/hadoop.control/control | 24 ++ src/packages/deb/hadoop.control/postinst | 24 ++ src/packages/deb/hadoop.control/postrm | 19 ++ src/packages/deb/hadoop.control/preinst | 18 ++ src/packages/deb/hadoop.control/prerm | 25 ++ src/packages/deb/init.d/hadoop-datanode | 142 +++++++++ src/packages/deb/init.d/hadoop-jobtracker | 142 +++++++++ src/packages/deb/init.d/hadoop-namenode | 154 ++++++++++ src/packages/deb/init.d/hadoop-tasktracker | 142 +++++++++ src/packages/hadoop-create-user.sh | 76 +++++ src/packages/hadoop-setup-conf.sh | 282 ++++++++++++++++++ src/packages/hadoop-setup-hdfs.sh | 96 ++++++ src/packages/hadoop-setup-single-node.sh | 212 +++++++++++++ src/packages/rpm/init.d/hadoop-datanode | 84 ++++++ src/packages/rpm/init.d/hadoop-jobtracker | 84 ++++++ src/packages/rpm/init.d/hadoop-namenode | 98 ++++++ src/packages/rpm/init.d/hadoop-tasktracker | 84 ++++++ src/packages/rpm/spec/hadoop.spec | 173 +++++++++++ src/packages/templates/conf/core-site.xml | 11 + src/packages/update-hadoop-env.sh | 168 +++++++++++ src/test/system/c++/runAs/configure | 4 +- src/test/system/c++/runAs/configure.ac | 4 +- src/test/system/c++/runAs/runAs.c | 2 +- src/test/system/c++/runAs/runAs.h.in | 4 +- .../process/HadoopDaemonRemoteCluster.java | 4 +- 54 files changed, 2593 insertions(+), 157 deletions(-) create mode 100644 src/docs/src/documentation/content/xdocs/deployment_layout.xml create mode 100644 src/packages/deb/hadoop.control/conffile create mode 100644 src/packages/deb/hadoop.control/control create mode 100644 src/packages/deb/hadoop.control/postinst create mode 100644 src/packages/deb/hadoop.control/postrm create mode 100644 src/packages/deb/hadoop.control/preinst create mode 100644 src/packages/deb/hadoop.control/prerm create mode 100644 src/packages/deb/init.d/hadoop-datanode create mode 100644 src/packages/deb/init.d/hadoop-jobtracker create mode 100644 src/packages/deb/init.d/hadoop-namenode create mode 100644 src/packages/deb/init.d/hadoop-tasktracker create mode 100644 src/packages/hadoop-create-user.sh create mode 100644 src/packages/hadoop-setup-conf.sh create mode 100644 src/packages/hadoop-setup-hdfs.sh create mode 100644 src/packages/hadoop-setup-single-node.sh create mode 100644 src/packages/rpm/init.d/hadoop-datanode create mode 100644 src/packages/rpm/init.d/hadoop-jobtracker create mode 100644 src/packages/rpm/init.d/hadoop-namenode create mode 100644 src/packages/rpm/init.d/hadoop-tasktracker create mode 100644 src/packages/rpm/spec/hadoop.spec create mode 100644 src/packages/templates/conf/core-site.xml create mode 100644 src/packages/update-hadoop-env.sh diff --git a/CHANGES.txt b/CHANGES.txt index 821657137f0..2b6a5038338 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2402,6 +2402,15 @@ Release 0.20.3 - Unreleased HADOOP-7072. Remove java5 dependencies from build. (cos) +Release 0.20.204.0 - Unreleased + + NEW FEATURES + + HADOOP-6255. Create RPM and Debian packages for common. Changes deployment + layout to be consistent across the binary tgz, rpm, and deb. Adds setup + scripts for easy one node cluster configuration and user creation. + (Eric Yang via omalley) + Release 0.20.203.0 - 2011-5-11 BUG FIXES diff --git a/bin/hadoop b/bin/hadoop index 10509037cf1..88c49eb3c29 100755 --- a/bin/hadoop +++ b/bin/hadoop @@ -17,10 +17,11 @@ # This script runs the hadoop core commands. -bin=`dirname "$0"` +bin=`which $0` +bin=`dirname ${bin}` bin=`cd "$bin"; pwd` -. "$bin"/hadoop-config.sh +. "$bin"/../libexec/hadoop-config.sh function print_usage(){ echo "Usage: hadoop [--config confdir] COMMAND" @@ -54,8 +55,8 @@ case $COMMAND in #try to locate hdfs and if present, delegate to it. if [ -f "${HADOOP_HDFS_HOME}"/bin/hdfs ]; then exec "${HADOOP_HDFS_HOME}"/bin/hdfs $* - elif [ -f "${HADOOP_HOME}"/bin/hdfs ]; then - exec "${HADOOP_HOME}"/bin/hdfs $* + elif [ -f "${HADOOP_PREFIX}"/bin/hdfs ]; then + exec "${HADOOP_PREFIX}"/bin/hdfs $* else echo "HDFS not found." exit @@ -70,8 +71,8 @@ case $COMMAND in #try to locate mapred and if present, delegate to it. if [ -f "${HADOOP_MAPRED_HOME}"/bin/mapred ]; then exec "${HADOOP_MAPRED_HOME}"/bin/mapred $* - elif [ -f "${HADOOP_HOME}"/bin/mapred ]; then - exec "${HADOOP_HOME}"/bin/mapred $* + elif [ -f "${HADOOP_PREFIX}"/bin/mapred ]; then + exec "${HADOOP_PREFIX}"/bin/mapred $* else echo "MAPRED not found." exit diff --git a/bin/hadoop-config.sh b/bin/hadoop-config.sh index 5b834872901..11d31572298 100644 --- a/bin/hadoop-config.sh +++ b/bin/hadoop-config.sh @@ -26,11 +26,8 @@ script="$(basename -- "$this")" this="$common_bin/$script" # the root of the Hadoop installation -#TODO: change the env variable when dir structure is changed -export HADOOP_HOME=`dirname "$this"`/.. -export HADOOP_COMMON_HOME="${HADOOP_HOME}" -#export HADOOP_HOME=`dirname "$this"`/../.. -#export HADOOP_COMMON_HOME="${HADOOP_COMMON_HOME:-`dirname "$this"`/..}" +# See HADOOP-6255 for directory structure layout +export HADOOP_PREFIX=`dirname "$this"`/.. #check to see if the conf dir is given as an optional argument if [ $# -gt 1 ] @@ -45,7 +42,13 @@ then fi # Allow alternate conf dir location. -export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_HOME/conf}" +if [ -e "${HADOOP_PREFIX}/conf/hadoop-env.sh" ]; then + DEFAULT_CONF_DIR="conf" +else + DEFAULT_CONF_DIR="etc/hadoop" +fi + +export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_PREFIX/$DEFAULT_CONF_DIR}" # User can specify hostnames or a file where the hostnames are (not both) if [[ ( "$HADOOP_SLAVES" != '' ) && ( "$HADOOP_SLAVE_NAMES" != '' ) ]] ; then @@ -130,54 +133,61 @@ CLASSPATH="${HADOOP_CONF_DIR}" CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar # for developers, add Hadoop classes to CLASSPATH -if [ -d "$HADOOP_COMMON_HOME/build/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/classes +if [ -d "$HADOOP_PREFIX/build/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build/classes fi -if [ -d "$HADOOP_COMMON_HOME/build/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build +if [ -d "$HADOOP_PREFIX/build/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build fi -if [ -d "$HADOOP_COMMON_HOME/build/test/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/test/classes +if [ -d "$HADOOP_PREFIX/build/test/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build/test/classes fi -if [ -d "$HADOOP_COMMON_HOME/build/test/core/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/test/core/classes +if [ -d "$HADOOP_PREFIX/build/test/core/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build/test/core/classes fi # so that filenames w/ spaces are handled correctly in loops below IFS= # for releases, add core hadoop jar & webapps to CLASSPATH -if [ -d "$HADOOP_COMMON_HOME/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME +if [ -d "$HADOOP_PREFIX/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX fi -for f in $HADOOP_COMMON_HOME/hadoop-*.jar; do + +if [ -d "$HADOOP_PREFIX/share/hadoop/common/lib" ]; then + for f in $HADOOP_PREFIX/share/hadoop/common/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; + done +fi + +for f in $HADOOP_PREFIX/share/hadoop/common/*.jar; do CLASSPATH=${CLASSPATH}:$f; done -# add libs to CLASSPATH -for f in $HADOOP_COMMON_HOME/lib/*.jar; do +# for developers, add libs to CLASSPATH +for f in $HADOOP_PREFIX/lib/*.jar; do CLASSPATH=${CLASSPATH}:$f; done -if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Common/common" ]; then -for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Common/common/*.jar; do +if [ -d "$HADOOP_PREFIX/build/ivy/lib/Hadoop-Common/common" ]; then +for f in $HADOOP_PREFIX/build/ivy/lib/Hadoop-Common/common/*.jar; do CLASSPATH=${CLASSPATH}:$f; done fi -if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Hdfs/common" ]; then -for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Hdfs/common/*.jar; do +if [ -d "$HADOOP_PREFIX/build/ivy/lib/hadoop-hdfs/hdfs" ]; then +for f in $HADOOP_PREFIX/build/ivy/lib/hadoop-hdfs/hdfs/*.jar; do CLASSPATH=${CLASSPATH}:$f; done fi -if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop/common" ]; then -for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop/common/*.jar; do +if [ -d "$HADOOP_PREFIX/build/ivy/lib/Hadoop/mapred" ]; then +for f in $HADOOP_PREFIX/build/ivy/lib/Hadoop/mapred/*.jar; do CLASSPATH=${CLASSPATH}:$f; done fi -for f in $HADOOP_COMMON_HOME/lib/jsp-2.1/*.jar; do +for f in $HADOOP_PREFIX/lib/jsp-2.1/*.jar; do CLASSPATH=${CLASSPATH}:$f; done @@ -188,7 +198,7 @@ fi # default log directory & file if [ "$HADOOP_LOG_DIR" = "" ]; then - HADOOP_LOG_DIR="$HADOOP_HOME/logs" + HADOOP_LOG_DIR="$HADOOP_PREFIX/logs" fi if [ "$HADOOP_LOGFILE" = "" ]; then HADOOP_LOGFILE='hadoop.log' @@ -204,33 +214,37 @@ unset IFS # cygwin path translation if $cygwin; then - HADOOP_COMMON_HOME=`cygpath -w "$HADOOP_COMMON_HOME"` + HADOOP_PREFIX=`cygpath -w "$HADOOP_PREFIX"` HADOOP_LOG_DIR=`cygpath -w "$HADOOP_LOG_DIR"` JAVA_LIBRARY_PATH=`cygpath -w "$JAVA_LIBRARY_PATH"` fi # setup 'java.library.path' for native-hadoop code if necessary -if [ -d "${HADOOP_COMMON_HOME}/build/native" -o -d "${HADOOP_COMMON_HOME}/lib/native" ]; then +if [ -d "${HADOOP_PREFIX}/build/native" -o -d "${HADOOP_PREFIX}/lib/native" ]; then JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} -Xmx32m ${HADOOP_JAVA_PLATFORM_OPTS} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` - if [ -d "$HADOOP_COMMON_HOME/build/native" ]; then + if [ -d "$HADOOP_PREFIX/build/native" ]; then if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_COMMON_HOME}/build/native/${JAVA_PLATFORM}/lib + JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_PREFIX}/build/native/${JAVA_PLATFORM}/lib else - JAVA_LIBRARY_PATH=${HADOOP_COMMON_HOME}/build/native/${JAVA_PLATFORM}/lib + JAVA_LIBRARY_PATH=${HADOOP_PREFIX}/build/native/${JAVA_PLATFORM}/lib fi fi - if [ -d "${HADOOP_COMMON_HOME}/lib/native" ]; then + if [ -d "${HADOOP_PREFIX}/lib/native" ]; then if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_COMMON_HOME}/lib/native/${JAVA_PLATFORM} + JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_PREFIX}/lib/native/${JAVA_PLATFORM} else - JAVA_LIBRARY_PATH=${HADOOP_COMMON_HOME}/lib/native/${JAVA_PLATFORM} + JAVA_LIBRARY_PATH=${HADOOP_PREFIX}/lib/native/${JAVA_PLATFORM} fi fi fi +if [ -e "${HADOOP_PREFIX}/lib/libhadoop.a" ]; then + JAVA_LIBRARY_PATH=${HADOOP_PREFIX}/lib +fi + # cygwin path translation if $cygwin; then JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` @@ -238,7 +252,7 @@ fi HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR" HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE" -HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_COMMON_HOME" +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX" HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING" HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}" HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,console}" @@ -252,8 +266,8 @@ HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true" # put hdfs in classpath if present if [ "$HADOOP_HDFS_HOME" = "" ]; then - if [ -d "${HADOOP_HOME}/hdfs" ]; then - HADOOP_HDFS_HOME=$HADOOP_HOME/hdfs + if [ -d "${HADOOP_PREFIX}/share/hadoop/hdfs" ]; then + HADOOP_HDFS_HOME=$HADOOP_PREFIX/share/hadoop/hdfs #echo Found HDFS installed at $HADOOP_HDFS_HOME fi fi @@ -273,9 +287,11 @@ if [ -d "${HADOOP_HDFS_HOME}" ]; then done # add libs to CLASSPATH - for f in $HADOOP_HDFS_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; - done + if [ -d "${HADOOP_HDFS_HOME}/lib" ]; then + for f in $HADOOP_HDFS_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; + done + fi if [ -d "$HADOOP_HDFS_HOME/build/classes" ]; then CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/classes @@ -289,9 +305,8 @@ fi # set mapred home if mapred is present if [ "$HADOOP_MAPRED_HOME" = "" ]; then - if [ -d "${HADOOP_HOME}/mapred" ]; then - HADOOP_MAPRED_HOME=$HADOOP_HOME/mapred - #echo Found MAPRED installed at $HADOOP_MAPRED_HOME + if [ -d "${HADOOP_PREFIX}/share/hadoop/mapreduce" ]; then + HADOOP_MAPRED_HOME=$HADOOP_PREFIX/share/hadoop/mapreduce fi fi @@ -305,13 +320,15 @@ if [ -d "${HADOOP_MAPRED_HOME}" ]; then CLASSPATH=${CLASSPATH}:${HADOOP_MAPRED_HOME}/conf fi - for f in $HADOOP_MAPRED_HOME/hadoop-mapred-*.jar; do + for f in $HADOOP_MAPRED_HOME/hadoop-mapreduce-*.jar; do CLASSPATH=${CLASSPATH}:$f done - for f in $HADOOP_MAPRED_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f - done + if [ -d "${HADOOP_MAPRED_HOME}/lib" ]; then + for f in $HADOOP_MAPRED_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f + done + fi if [ -d "$HADOOP_MAPRED_HOME/build/classes" ]; then CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/classes @@ -321,10 +338,10 @@ if [ -d "${HADOOP_MAPRED_HOME}" ]; then CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/tools fi - for f in $HADOOP_MAPRED_HOME/hadoop-mapred-tools-*.jar; do + for f in $HADOOP_MAPRED_HOME/hadoop-mapreduce-tools-*.jar; do TOOL_PATH=${TOOL_PATH}:$f; done - for f in $HADOOP_MAPRED_HOME/build/hadoop-mapred-tools-*.jar; do + for f in $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-tools-*.jar; do TOOL_PATH=${TOOL_PATH}:$f; done fi diff --git a/bin/hadoop-daemon.sh b/bin/hadoop-daemon.sh index dcc5910c06b..d5551259e34 100755 --- a/bin/hadoop-daemon.sh +++ b/bin/hadoop-daemon.sh @@ -20,7 +20,7 @@ # # Environment Variables # -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_PREFIX}/conf. # HADOOP_LOG_DIR Where log files are stored. PWD by default. # HADOOP_MASTER host:path where hadoop code should be rsync'd from # HADOOP_PID_DIR The pid files are stored. /tmp by default. @@ -39,12 +39,12 @@ fi bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -. "$bin"/hadoop-config.sh +. "$bin"/../libexec/hadoop-config.sh # get arguments #default value -hadoopScript="$HADOOP_HOME"/bin/hadoop +hadoopScript="$HADOOP_PREFIX"/bin/hadoop if [ "--script" = "$1" ] then shift @@ -91,7 +91,7 @@ fi # get log directory if [ "$HADOOP_LOG_DIR" = "" ]; then - export HADOOP_LOG_DIR="$HADOOP_HOME/logs" + export HADOOP_LOG_DIR="$HADOOP_PREFIX/logs" fi mkdir -p "$HADOOP_LOG_DIR" chown $HADOOP_IDENT_STRING $HADOOP_LOG_DIR @@ -127,12 +127,12 @@ case $startStop in if [ "$HADOOP_MASTER" != "" ]; then echo rsync from $HADOOP_MASTER - rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $HADOOP_MASTER/ "$HADOOP_HOME" + rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $HADOOP_MASTER/ "$HADOOP_PREFIX" fi hadoop_rotate_log $log echo starting $command, logging to $log - cd "$HADOOP_HOME" + cd "$HADOOP_PREFIX" nohup nice -n $HADOOP_NICENESS $hadoopScript --config $HADOOP_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null & echo $! > $pid sleep 1; head "$log" diff --git a/bin/hadoop-daemons.sh b/bin/hadoop-daemons.sh index 5cabd39067d..fd580649a82 100755 --- a/bin/hadoop-daemons.sh +++ b/bin/hadoop-daemons.sh @@ -29,6 +29,6 @@ fi bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -. $bin/hadoop-config.sh +. $bin/../libexec/hadoop-config.sh -exec "$bin/slaves.sh" --config $HADOOP_CONF_DIR cd "$HADOOP_HOME" \; "$bin/hadoop-daemon.sh" --config $HADOOP_CONF_DIR "$@" +exec "$bin/slaves.sh" --config $HADOOP_CONF_DIR cd "$HADOOP_PREFIX" \; "$bin/hadoop-daemon.sh" --config $HADOOP_CONF_DIR "$@" diff --git a/bin/rcc b/bin/rcc index cddfc142356..d5bcca54e13 100755 --- a/bin/rcc +++ b/bin/rcc @@ -24,13 +24,13 @@ # # HADOOP_OPTS Extra Java runtime options. # -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_PREFIX}/conf. # bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -. "$bin"/hadoop-config.sh +. "$bin"/../libexec/hadoop-config.sh if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then . "${HADOOP_CONF_DIR}/hadoop-env.sh" @@ -55,33 +55,33 @@ CLASSPATH="${HADOOP_CONF_DIR}" CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar # for developers, add Hadoop classes to CLASSPATH -if [ -d "$HADOOP_HOME/build/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/classes +if [ -d "$HADOOP_PREFIX/build/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build/classes fi -if [ -d "$HADOOP_HOME/build/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build +if [ -d "$HADOOP_PREFIX/build/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build fi -if [ -d "$HADOOP_HOME/build/test/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/test/classes +if [ -d "$HADOOP_PREFIX/build/test/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build/test/classes fi # so that filenames w/ spaces are handled correctly in loops below IFS= # for releases, add core hadoop jar & webapps to CLASSPATH -if [ -d "$HADOOP_HOME/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_HOME +if [ -d "$HADOOP_PREFIX/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX fi -for f in $HADOOP_HOME/hadoop-*.jar; do +for f in $HADOOP_PREFIX/hadoop-*.jar; do CLASSPATH=${CLASSPATH}:$f; done # add libs to CLASSPATH -for f in $HADOOP_HOME/lib/*.jar; do +for f in $HADOOP_PREFIX/lib/*.jar; do CLASSPATH=${CLASSPATH}:$f; done -for f in $HADOOP_HOME/lib/jetty-ext/*.jar; do +for f in $HADOOP_PREFIX/lib/jetty-ext/*.jar; do CLASSPATH=${CLASSPATH}:$f; done diff --git a/bin/slaves.sh b/bin/slaves.sh index b4bafe10972..153f4416f7d 100755 --- a/bin/slaves.sh +++ b/bin/slaves.sh @@ -22,7 +22,7 @@ # # HADOOP_SLAVES File naming remote hosts. # Default is ${HADOOP_CONF_DIR}/slaves. -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_PREFIX}/conf. # HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. # HADOOP_SSH_OPTS Options passed to ssh when running remote commands. ## @@ -38,7 +38,7 @@ fi bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -. "$bin"/hadoop-config.sh +. "$bin"/../libexec/hadoop-config.sh if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then . "${HADOOP_CONF_DIR}/hadoop-env.sh" diff --git a/bin/start-all.sh b/bin/start-all.sh index d4b8bec6d4e..646c9b78020 100755 --- a/bin/start-all.sh +++ b/bin/start-all.sh @@ -23,7 +23,7 @@ echo "This script is Deprecated. Instead use start-dfs.sh and start-mapred.sh" bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -. "$bin"/hadoop-config.sh +. "$bin"/../libexec/hadoop-config.sh # start hdfs daemons if hdfs is present if [ -f "${HADOOP_HDFS_HOME}"/bin/start-dfs.sh ]; then diff --git a/bin/stop-all.sh b/bin/stop-all.sh index 7c267f3403c..9adb5a4e031 100755 --- a/bin/stop-all.sh +++ b/bin/stop-all.sh @@ -23,7 +23,7 @@ echo "This script is Deprecated. Instead use stop-dfs.sh and stop-mapred.sh" bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -. "$bin"/hadoop-config.sh +. "$bin"/../libexec/hadoop-config.sh # stop hdfs daemons if hdfs is present if [ -f "${HADOOP_HDFS_HOME}"/bin/stop-dfs.sh ]; then diff --git a/build.xml b/build.xml index 168b34f9527..80f2afd549a 100644 --- a/build.xml +++ b/build.xml @@ -26,12 +26,15 @@ - - - + + + + + + @@ -196,7 +199,18 @@ - + + + + + + + + + + + + @@ -1073,6 +1087,7 @@ description="Build distribution"> + @@ -1122,6 +1137,12 @@ + + + + + + @@ -1160,17 +1181,24 @@ - + + + + + + + - + - + @@ -1179,38 +1207,57 @@ - + - - + + - + + + - + + + + + + + + + + + + + + + + + + + + + - - - - + + - + - - + @@ -1219,16 +1266,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1420,6 +1574,7 @@ + @@ -1653,6 +1808,11 @@ log="${ivyresolvelog}"/> + + + + @@ -1710,6 +1870,14 @@ + + + + + + + + + + + + 通常，集群里的一台机器被指定为 NameNode，另一台不同的机器被指定为JobTracker。这些机器是masters。余下的机器即作为DataNode也作为TaskTracker。这些机器是slaves。

-

我们用HADOOP_HOME指代安装的根路径。通常，集群里的所有机器的HADOOP_HOME路径相同。

+

我们用HADOOP_PREFIX指代安装的根路径。通常，集群里的所有机器的HADOOP_PREFIX路径相同。

@@ -335,7 +335,7 @@
-

一但全部必要的配置完成，将这些文件分发到所有机器的HADOOP_CONF_DIR路径下，通常是${HADOOP_HOME}/conf。

+

一但全部必要的配置完成，将这些文件分发到所有机器的HADOOP_CONF_DIR路径下，通常是${HADOOP_PREFIX}/conf。

diff --git a/src/docs/cn/src/documentation/content/xdocs/commands_manual.xml b/src/docs/cn/src/documentation/content/xdocs/commands_manual.xml index beb65b677d4..724e5ea217a 100644 --- a/src/docs/cn/src/documentation/content/xdocs/commands_manual.xml +++ b/src/docs/cn/src/documentation/content/xdocs/commands_manual.xml @@ -39,7 +39,7 @@ --config confdir - 覆盖缺省配置目录。缺省是${HADOOP_HOME}/conf。 + 覆盖缺省配置目录。缺省是${HADOOP_PREFIX}/conf。 GENERIC_OPTIONS diff --git a/src/docs/cn/src/documentation/content/xdocs/hod_admin_guide.xml b/src/docs/cn/src/documentation/content/xdocs/hod_admin_guide.xml index 6d1a0d3d8da..12f998e9468 100644 --- a/src/docs/cn/src/documentation/content/xdocs/hod_admin_guide.xml +++ b/src/docs/cn/src/documentation/content/xdocs/hod_admin_guide.xml @@ -128,7 +128,7 @@
  • ${JAVA_HOME}：Hadoop的Java的安装位置。Hadoop支持Sun JDK 1.5.x及以上版本。
  • ${CLUSTER_NAME}：集群名称，由'node property'指定，在资源管理器配置中曾提到过。
    • -
  • ${HADOOP_HOME}：Hadoop在计算节点和提交节点上的安装位置。
    • +
  • ${HADOOP_PREFIX}：Hadoop在计算节点和提交节点上的安装位置。
  • ${RM_QUEUE}：在资源管理器配置中设置的作业提交队列。
  • ${RM_HOME}：资源管理器在计算节点和提交节点的安装位置。
diff --git a/src/docs/cn/src/documentation/content/xdocs/mapred_tutorial.xml b/src/docs/cn/src/documentation/content/xdocs/mapred_tutorial.xml index d842f4d93a8..6d9227730ea 100644 --- a/src/docs/cn/src/documentation/content/xdocs/mapred_tutorial.xml +++ b/src/docs/cn/src/documentation/content/xdocs/mapred_tutorial.xml @@ -511,11 +511,11 @@
用法 -

假设环境变量HADOOP_HOME对应安装时的根目录，HADOOP_VERSION对应Hadoop的当前安装版本，编译WordCount.java来创建jar包，可如下操作：

+

假设环境变量HADOOP_PREFIX对应安装时的根目录，HADOOP_VERSION对应Hadoop的当前安装版本，编译WordCount.java来创建jar包，可如下操作：

$ mkdir wordcount_classes
- $ javac -classpath ${HADOOP_HOME}/hadoop-${HADOOP_VERSION}-core.jar + $ javac -classpath ${HADOOP_PREFIX}/hadoop-${HADOOP_VERSION}-core.jar -d wordcount_classes WordCount.java
$ jar -cvf /usr/joe/wordcount.jar -C wordcount_classes/ . diff --git a/src/docs/cn/src/documentation/content/xdocs/quickstart.xml b/src/docs/cn/src/documentation/content/xdocs/quickstart.xml index e13395563ef..63c09048958 100644 --- a/src/docs/cn/src/documentation/content/xdocs/quickstart.xml +++ b/src/docs/cn/src/documentation/content/xdocs/quickstart.xml @@ -192,7 +192,7 @@

Hadoop守护进程的日志写入到 ${HADOOP_LOG_DIR} 目录 (默认是 - ${HADOOP_HOME}/logs).

+ ${HADOOP_PREFIX}/logs).

浏览NameNode和JobTracker的网络接口，它们的地址默认为：

    diff --git a/src/docs/cn/src/documentation/content/xdocs/streaming.xml b/src/docs/cn/src/documentation/content/xdocs/streaming.xml index 221205e13f1..bf99d70a2f6 100644 --- a/src/docs/cn/src/documentation/content/xdocs/streaming.xml +++ b/src/docs/cn/src/documentation/content/xdocs/streaming.xml @@ -36,7 +36,7 @@ Hadoop streaming是Hadoop的一个工具， 这些特殊的map/reduce作业是由一些可执行文件或脚本文件充当mapper或者reducer。例如：

    -$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ +$HADOOP_PREFIX/bin/hadoop jar $HADOOP_PREFIX/hadoop-streaming.jar \ -input myInputDirs \ -output myOutputDir \ -mapper /bin/cat \ @@ -68,7 +68,7 @@ Reducer任务运行时，它把输入切分成行并把每一行提供给可执 用户也可以使用java类作为mapper或者reducer。上面的例子与这里的代码等价：

    -$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ +$HADOOP_PREFIX/bin/hadoop jar $HADOOP_PREFIX/hadoop-streaming.jar \ -input myInputDirs \ -output myOutputDir \ -mapper org.apache.hadoop.mapred.lib.IdentityMapper \ @@ -88,7 +88,7 @@ $HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ 如果在集群上还没有，则需要用-file选项让framework把可执行文件作为作业的一部分，一起打包提交。例如：

    -$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ +$HADOOP_PREFIX/bin/hadoop jar $HADOOP_PREFIX/hadoop-streaming.jar \ -input myInputDirs \ -output myOutputDir \ -mapper myPythonScript.py \ @@ -103,7 +103,7 @@ $HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ 除了可执行文件外，其他mapper或reducer需要用到的辅助文件（比如字典，配置文件等）也可以用这种方式打包上传。例如：

    -$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ +$HADOOP_PREFIX/bin/hadoop jar $HADOOP_PREFIX/hadoop-streaming.jar \ -input myInputDirs \ -output myOutputDir \ -mapper myPythonScript.py \ @@ -173,7 +173,7 @@ $HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ 下面是使用-cacheArchive选项的另一个例子。其中，input.txt文件有两行内容，分别是两个文件的名字：testlink/cache.txt和testlink/cache2.txt。“testlink”是指向档案目录（jar文件解压后的目录）的符号链接，这个目录下有“cache.txt”和“cache2.txt”两个文件。

    -$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ +$HADOOP_PREFIX/bin/hadoop jar $HADOOP_PREFIX/hadoop-streaming.jar \ -input "/user/me/samples/cachefile/input.txt" \ -mapper "xargs cat" \ -reducer "cat" \ @@ -220,7 +220,7 @@ This is just the second cache string 用户可以使用“-jobconf <n>=<v>”增加一些配置变量。例如：

    -$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ +$HADOOP_PREFIX/bin/hadoop jar $HADOOP_PREFIX/hadoop-streaming.jar \ -input myInputDirs \ -output myOutputDir \ -mapper org.apache.hadoop.mapred.lib.IdentityMapper\ @@ -254,7 +254,7 @@ Streaming 作业的其他选项如下表： -verbose 可选 详细输出

    -使用-cluster <name>实现“本地”Hadoop和一个或多个远程Hadoop集群间切换。默认情况下，使用hadoop-default.xml和hadoop-site.xml；当使用-cluster <name>选项时，会使用$HADOOP_HOME/conf/hadoop-<name>.xml。 +使用-cluster <name>实现“本地”Hadoop和一个或多个远程Hadoop集群间切换。默认情况下，使用hadoop-default.xml和hadoop-site.xml；当使用-cluster <name>选项时，会使用$HADOOP_PREFIX/conf/hadoop-<name>.xml。

    下面的选项改变temp目录： @@ -295,7 +295,7 @@ Streaming 作业的其他选项如下表：

    -$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ +$HADOOP_PREFIX/bin/hadoop jar $HADOOP_PREFIX/hadoop-streaming.jar \ -input myInputDirs \ -output myOutputDir \ -mapper org.apache.hadoop.mapred.lib.IdentityMapper \ @@ -320,7 +320,7 @@ Hadoop有一个工具类org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner， 切分是基于key值的前缀，而不是整个key。例如：

    -$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ +$HADOOP_PREFIX/bin/hadoop jar $HADOOP_PREFIX/hadoop-streaming.jar \ -input myInputDirs \ -output myOutputDir \ -mapper org.apache.hadoop.mapred.lib.IdentityMapper \ @@ -390,7 +390,7 @@ combiner/reducer利用适当的聚合器聚合这些可聚合项。

    要使用Aggregate，只需指定“-reducer aggregate”：

    -$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ +$HADOOP_PREFIX/bin/hadoop jar $HADOOP_PREFIX/hadoop-streaming.jar \ -input myInputDirs \ -output myOutputDir \ -mapper myAggregatorForKeyCount.py \ @@ -434,7 +434,7 @@ Hadoop的工具类org.apache.hadoop.mapred.lib.FieldSelectionMapReduce帮助用 同样，工具类中的reduce函数也把输入的key/value对看作字段的列表，用户可以选取任意一段作为reduce输出的key或value。例如：

    -$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ +$HADOOP_PREFIX/bin/hadoop jar $HADOOP_PREFIX/hadoop-streaming.jar \ -input myInputDirs \ -output myOutputDir \ -mapper org.apache.hadoop.mapred.lib.FieldSelectionMapReduce\ @@ -523,7 +523,7 @@ bruce 70 charlie 80 dan 75 -$ c2='cut -f2'; $HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/hadoop-streaming.jar \ +$ c2='cut -f2'; $HADOOP_PREFIX/bin/hadoop jar $HADOOP_PREFIX/hadoop-streaming.jar \ -input /user/me/samples/student_marks -mapper \"$c2\" -reducer 'cat' -output /user/me/samples/student_out diff --git a/src/docs/src/documentation/content/xdocs/cluster_setup.xml b/src/docs/src/documentation/content/xdocs/cluster_setup.xml index 4e06247f50a..1cc1128186a 100644 --- a/src/docs/src/documentation/content/xdocs/cluster_setup.xml +++ b/src/docs/src/documentation/content/xdocs/cluster_setup.xml @@ -64,8 +64,8 @@ TaskTracker. These are the slaves.

    The root of the distribution is referred to as - HADOOP_HOME. All machines in the cluster usually have the same - HADOOP_HOME path.

    + HADOOP_PREFIX. All machines in the cluster usually have the same + HADOOP_PREFIX path.

@@ -1084,7 +1084,7 @@ on the cluster nodes where a configuration file for the setuid executable would be located. The executable would be built to build.dir/dist.dir/bin and should be installed to - $HADOOP_HOME/bin. + $HADOOP_PREFIX/bin.

@@ -1274,7 +1274,7 @@

Once all the necessary configuration is complete, distribute the files to the HADOOP_CONF_DIR directory on all the machines, - typically ${HADOOP_HOME}/conf.

+ typically ${HADOOP_PREFIX}/conf.

Cluster Restartability diff --git a/src/docs/src/documentation/content/xdocs/commands_manual.xml b/src/docs/src/documentation/content/xdocs/commands_manual.xml index 06ee65b357d..b46b1285a17 100644 --- a/src/docs/src/documentation/content/xdocs/commands_manual.xml +++ b/src/docs/src/documentation/content/xdocs/commands_manual.xml @@ -40,7 +40,7 @@ --config confdir - Overwrites the default Configuration directory. Default is ${HADOOP_HOME}/conf. + Overwrites the default Configuration directory. Default is ${HADOOP_PREFIX}/conf. GENERIC_OPTIONS diff --git a/src/docs/src/documentation/content/xdocs/deployment_layout.xml b/src/docs/src/documentation/content/xdocs/deployment_layout.xml new file mode 100644 index 00000000000..ae8ecccca59 --- /dev/null +++ b/src/docs/src/documentation/content/xdocs/deployment_layout.xml @@ -0,0 +1,147 @@ + + + + + + + + +
+ + Hadoop Deployment Layout + +
+ + +
+ Introduction +

+ This document describes the standard deployment layout for Hadoop. With increased complexity and evolving Hadoop ecosystem, having standard deployment layout ensures better integration between Hadoop sub-projects. By making the installation process easier, we can lower the barrier to entry and increase Hadoop adoption. +

+
+ +
+ Packages +

+ We need to divide Hadoop up into packages that can be independently upgraded. The list of packages should include: +

+
    +
  • Hadoop Common - Common including the native code and required jar files.
    • +
  • HDFS Client - HDFS jars, scripts, and shared libraries.
    • +
  • HDFS Server - jsvc executable
    • +
  • Yarn Client - Yarn client jars and scripts
    • +
  • Yarn Server - Yarn server jars and scripts
    • +
  • MapReduce - MapReduce jars, scripts, and shared libraries
    • +
  • LZO - LZ0 codec from github.com/omally/hadoop-gpl-compression
    • +
  • Metrics - Plugins for Chukwa and Ganglia
    • +
+

Packages from other teams will include:

+
    +
  • Pig
    • +
  • Hive
    • +
  • Oozie client
    • +
  • Oozie server
    • +
  • Howl client
    • +
  • Howl server
    • +
+

These packages should be deployable with RPM on RedHat. We also need a package that depends on a version of each of these packages. In general, we can generate tarballs in the new deployment layout.

+

Note that some packages, like Pig, which are user facing, will have 2 versions installed in a given deployment. This will be accomplished by modifying the package name and the associated binaries to include the version number.

+

All of the following paths are based on a prefix directory that is the root of the installation. Our packages must support having multiple Hadoop stack installation on a computer at the same time. For RPMs, this means that the packages must be relocatable and honor the --prefix option.

+
+ + +
+ Deployment +

It is important to have a standard deployment that results from installing the packages regardless of the package manager. Here are the top level directories and a sample of what would be under each. Note that all of the packages are installed "flattened" into the prefix directory. For compatibility reasons, we should create "share/hadoop" that matches the old HADOOP_PREFIX and set the HADOOP_PREFIX variable to that.

+ + $PREFIX/ bin / hadoop + | | mapred + | | pig -> pig7 + | | pig6 + | + pig7 + | + + etc / hadoop / core-site.xml + | | hdfs-site.xml + | + mapred-site.xml + | + + include / hadoop / Pipes.hh + | | + TemplateFactory.hh + | + hdfs.h + | + + lib / jni / hadoop-common / libhadoop.so.0.20.0 + | | + | | libhdfs.so -> libhdfs.so.0.20.0 + | + libhdfs.so.0.20.0 + | + + libexec / task-controller + | + + man / man1 / hadoop.1 + | | mapred.1 + | | pig6.1 + | + pig7.1 + | + + share / hadoop-common + | | hadoop-hdfs + | | hadoop-mapreduce + | | pig6 + | + pig7 + | + + sbin / hdfs-admin + | | mapred-admin + | + + src / hadoop-common + | | hadoop-hdfs + | + hadoop-mapreduce + | + + var / lib / data-node + | + task-tracker + | + | log / hadoop-datanode + | + hadoop-tasktracker + | + + run / hadoop-datanode.pid + + hadoop-tasktracker.pid + +

Note that we must continue to honor HADOOP_CONF_DIR to override the configuration location, but that it should default to $prefix/etc. User facing binaries and scripts go into bin. Configuration files go into etc with multiple configuration files having a directory. JNI shared libraries go into lib/jni/$tool since Java does not allow to specify the version of the library to load. Libraries that aren't loaded via System.loadLibrary are placed directly under lib. 64 bit versions of the libraries for platforms that support them should be placed in lib64. All of the architecture-independent pieces, including the jars for each tool will be placed in share/$tool. The default location for all the run time information will be in var. The storage will be in var/lib, the logs in var/log and the pid files in var/run.

+
+ +
+ Path Configurations +

Path can be configured at compile phase or installation phase. For RPM, it takes advantage of the --relocate directive to allow path reconfiguration at install phase. For Debian package, path is configured at compile phase. +

+

Build phase parameter:

+
    +
  • package.prefix - Location of package prefix (Default /usr)
    • +
  • package.conf.dir - Location of configuration directory (Default /etc/hadoop)
    • +
  • package.log.dir - Location of log directory (Default /var/log/hadoop)
    • +
  • package.pid.dir - Location of pid directory (Default /var/run/hadoop)
    • +
+ +

Install phase parameter:

+ + rpm -i hadoop-[version]-[rev].[arch].rpm \ + --relocate /usr=/usr/local/hadoop \ + --relocate /etc/hadoop=/usr/local/etc/hadoop \ + --relocate /var/log/hadoop=/opt/logs/hadoop \ + --relocate /var/run/hadoop=/opt/run/hadoop + +
+ + + + diff --git a/src/docs/src/documentation/content/xdocs/single_node_setup.xml b/src/docs/src/documentation/content/xdocs/single_node_setup.xml index 3f6a441e6e5..8705273bf61 100644 --- a/src/docs/src/documentation/content/xdocs/single_node_setup.xml +++ b/src/docs/src/documentation/content/xdocs/single_node_setup.xml @@ -229,7 +229,7 @@

The hadoop daemon log output is written to the ${HADOOP_LOG_DIR} directory (defaults to - ${HADOOP_HOME}/logs).

+ ${HADOOP_PREFIX}/logs).

Browse the web interface for the NameNode and the JobTracker; by default they are available at:

diff --git a/src/docs/src/documentation/content/xdocs/site.xml b/src/docs/src/documentation/content/xdocs/site.xml index 58536166c45..34f3f210ef0 100644 --- a/src/docs/src/documentation/content/xdocs/site.xml +++ b/src/docs/src/documentation/content/xdocs/site.xml @@ -39,6 +39,7 @@ See http://forrest.apache.org/docs/linking.html for more info. + diff --git a/src/native/Makefile.am b/src/native/Makefile.am index fecc70ca703..39192707f00 100644 --- a/src/native/Makefile.am +++ b/src/native/Makefile.am @@ -18,7 +18,7 @@ # # Notes: -# 1. This makefile is designed to do the actual builds in $(HADOOP_HOME)/build/native/${os.name}-${os-arch}. +# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os-arch}. # 2. This makefile depends on the following environment variables to function correctly: # * HADOOP_NATIVE_SRCDIR # * JAVA_HOME diff --git a/src/native/lib/Makefile.am b/src/native/lib/Makefile.am index f33df0e9a85..99c03602d1d 100644 --- a/src/native/lib/Makefile.am +++ b/src/native/lib/Makefile.am @@ -22,7 +22,7 @@ # # Notes: -# 1. This makefile is designed to do the actual builds in $(HADOOP_HOME)/build/native/${os.name}-${os.arch}/lib +# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os.arch}/lib # 2. This makefile depends on the following environment variables to function correctly: # * HADOOP_NATIVE_SRCDIR # * JAVA_HOME diff --git a/src/native/packageNativeHadoop.sh b/src/native/packageNativeHadoop.sh index 4ce6f2e13bb..46c69bae592 100755 --- a/src/native/packageNativeHadoop.sh +++ b/src/native/packageNativeHadoop.sh @@ -35,14 +35,14 @@ if [ -d $BASE_NATIVE_LIB_DIR ] then for platform in `ls $BASE_NATIVE_LIB_DIR` do - if [ ! -d $DIST_LIB_DIR/$platform ] + if [ ! -d $DIST_LIB_DIR ] then - mkdir -p $DIST_LIB_DIR/$platform - echo "Created $DIST_LIB_DIR/$platform" + mkdir -p $DIST_LIB_DIR + echo "Created $DIST_LIB_DIR" fi - echo "Copying libraries in $BASE_NATIVE_LIB_DIR/$platform to $DIST_LIB_DIR/$platform/" - cd $BASE_NATIVE_LIB_DIR/$platform/ - $TAR . | (cd $DIST_LIB_DIR/$platform/; $UNTAR) + echo "Copying libraries in $BASE_NATIVE_LIB_DIR/$platform to $DIST_LIB_DIR/" + cd $BASE_NATIVE_LIB_DIR/ + $TAR . | (cd $DIST_LIB_DIR/; $UNTAR) done fi @@ -51,14 +51,14 @@ if [ -d $BUILD_NATIVE_DIR ] then for platform in `ls $BUILD_NATIVE_DIR` do - if [ ! -d $DIST_LIB_DIR/$platform ] + if [ ! -d $DIST_LIB_DIR ] then - mkdir -p $DIST_LIB_DIR/$platform - echo "Created $DIST_LIB_DIR/$platform" + mkdir -p $DIST_LIB_DIR + echo "Created $DIST_LIB_DIR" fi - echo "Copying libraries in $BUILD_NATIVE_DIR/$platform/lib to $DIST_LIB_DIR/$platform/" + echo "Copying libraries in $BUILD_NATIVE_DIR/$platform/lib to $DIST_LIB_DIR/" cd $BUILD_NATIVE_DIR/$platform/lib - $TAR . | (cd $DIST_LIB_DIR/$platform/; $UNTAR) + $TAR . | (cd $DIST_LIB_DIR/; $UNTAR) done fi diff --git a/src/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am b/src/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am index a2cb5fef7b1..7849ed77623 100644 --- a/src/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am +++ b/src/native/src/org/apache/hadoop/io/compress/zlib/Makefile.am @@ -22,7 +22,7 @@ # # Notes: -# 1. This makefile is designed to do the actual builds in $(HADOOP_HOME)/build/native/${os.name}-${os.arch}/$(subdir) . +# 1. This makefile is designed to do the actual builds in $(HADOOP_PREFIX)/build/native/${os.name}-${os.arch}/$(subdir) . # 2. This makefile depends on the following environment variables to function correctly: # * HADOOP_NATIVE_SRCDIR # * JAVA_HOME @@ -31,7 +31,7 @@ # * PLATFORM # All these are setup by build.xml and/or the top-level makefile. # 3. The creation of requisite jni headers/stubs are also done by build.xml and they are -# assumed to be in $(HADOOP_HOME)/build/native/src/org/apache/hadoop/io/compress/zlib. +# assumed to be in $(HADOOP_PREFIX)/build/native/src/org/apache/hadoop/io/compress/zlib. # # The 'vpath directive' to locate the actual source files diff --git a/src/packages/deb/hadoop.control/conffile b/src/packages/deb/hadoop.control/conffile new file mode 100644 index 00000000000..e52c79f59b7 --- /dev/null +++ b/src/packages/deb/hadoop.control/conffile @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +/etc/hadoop diff --git a/src/packages/deb/hadoop.control/control b/src/packages/deb/hadoop.control/control new file mode 100644 index 00000000000..9f7fd0fa47a --- /dev/null +++ b/src/packages/deb/hadoop.control/control @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Package: hadoop-common +Version: @version@ +Section: misc +Priority: optional +Provides: hadoop-common +Architecture: all +Depends: openjdk-6-jre-headless +Maintainer: Apache Software Foundation +Description: The Apache Hadoop project develops open-source software for reliable, scalable, distributed computing. +Distribution: development diff --git a/src/packages/deb/hadoop.control/postinst b/src/packages/deb/hadoop.control/postinst new file mode 100644 index 00000000000..463335da195 --- /dev/null +++ b/src/packages/deb/hadoop.control/postinst @@ -0,0 +1,24 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bash /usr/sbin/update-hadoop-env.sh \ + --prefix=/usr \ + --bin-dir=/usr/bin \ + --sbin-dir=/usr/sbin \ + --conf-dir=/etc/hadoop \ + --log-dir=/var/log/hadoop \ + --pid-dir=/var/run/hadoop diff --git a/src/packages/deb/hadoop.control/postrm b/src/packages/deb/hadoop.control/postrm new file mode 100644 index 00000000000..0521972c147 --- /dev/null +++ b/src/packages/deb/hadoop.control/postrm @@ -0,0 +1,19 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +/usr/sbin/groupdel hadoop 2> /dev/null >dev/null +exit 0 diff --git a/src/packages/deb/hadoop.control/preinst b/src/packages/deb/hadoop.control/preinst new file mode 100644 index 00000000000..23a33fc9c35 --- /dev/null +++ b/src/packages/deb/hadoop.control/preinst @@ -0,0 +1,18 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hadoop diff --git a/src/packages/deb/hadoop.control/prerm b/src/packages/deb/hadoop.control/prerm new file mode 100644 index 00000000000..176f9aa4efc --- /dev/null +++ b/src/packages/deb/hadoop.control/prerm @@ -0,0 +1,25 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bash /usr/sbin/update-hadoop-env.sh \ + --prefix=/usr \ + --bin-dir=/usr/bin \ + --sbin-dir=/usr/sbin \ + --conf-dir=/etc/hadoop \ + --log-dir=/var/log/hadoop \ + --pid-dir=/var/run/hadoop \ + --uninstal diff --git a/src/packages/deb/init.d/hadoop-datanode b/src/packages/deb/init.d/hadoop-datanode new file mode 100644 index 00000000000..77bc6a5f172 --- /dev/null +++ b/src/packages/deb/init.d/hadoop-datanode @@ -0,0 +1,142 @@ +#! /bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### BEGIN INIT INFO +# Provides: hadoop-datanode +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: +# Short-Description: Apache Hadoop Name Node server +### END INIT INFO + +set -e + +# /etc/init.d/hadoop-datanode: start and stop the Apache Hadoop Data Node daemon + +test -x /usr/bin/hadoop || exit 0 +( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0 + +umask 022 + +if test -f /etc/default/hadoop-env.sh; then + . /etc/default/hadoop-env.sh +fi + +. /lib/lsb/init-functions + +# Are we running from init? +run_by_init() { + ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ] +} + +check_for_no_start() { + # forget it if we're trying to start, and /etc/hadoop/hadoop-datanode_not_to_be_run exists + if [ -e /etc/hadoop/hadoop-datanode_not_to_be_run ]; then + if [ "$1" = log_end_msg ]; then + log_end_msg 0 + fi + if ! run_by_init; then + log_action_msg "Apache Hadoop Data Node server not in use (/etc/hadoop/hadoop-datanode_not_to_be_run)" + fi + exit 0 + fi +} + +check_privsep_dir() { + # Create the PrivSep empty dir if necessary + if [ ! -d ${HADOOP_PID_DIR} ]; then + mkdir -p ${HADOOP_PID_DIR} + chown root:hadoop ${HADOOP_PID_DIR} + chmod 0775 ${HADOOP_PID_DIR} + fi +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin" + +case "$1" in + start) + check_privsep_dir + check_for_no_start + log_daemon_msg "Starting Apache Hadoop Data Node server" "hadoop-datanode" + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + stop) + log_daemon_msg "Stopping Apache Hadoop Data Node server" "hadoop-datanode" + if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + restart) + check_privsep_dir + log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode" + start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + try-restart) + check_privsep_dir + log_daemon_msg "Restarting Apache Hadoop Data Node server" "hadoop-datanode" + set +e + start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid + RET="$?" + set -e + case $RET in + 0) + # old daemon stopped + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start datanode; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + 1) + # daemon not running + log_progress_msg "(not running)" + log_end_msg 0 + ;; + *) + # failed to stop + log_progress_msg "(failed to stop)" + log_end_msg 1 + ;; + esac + ;; + + status) + status_of_proc -p ${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid ${JAVA_HOME}/bin/java hadoop-datanode && exit 0 || exit $? + ;; + + *) + log_action_msg "Usage: /etc/init.d/hadoop-datanode {start|stop|restart|try-restart|status}" + exit 1 +esac + +exit 0 diff --git a/src/packages/deb/init.d/hadoop-jobtracker b/src/packages/deb/init.d/hadoop-jobtracker new file mode 100644 index 00000000000..61e47af342b --- /dev/null +++ b/src/packages/deb/init.d/hadoop-jobtracker @@ -0,0 +1,142 @@ +#! /bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### BEGIN INIT INFO +# Provides: hadoop-jobtracker +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: +# Short-Description: Apache Hadoop Job Tracker server +### END INIT INFO + +set -e + +# /etc/init.d/hadoop-jobtracker: start and stop the Apache Hadoop Job Tracker daemon + +test -x /usr/bin/hadoop || exit 0 +( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0 + +umask 022 + +if test -f /etc/default/hadoop-env.sh; then + . /etc/default/hadoop-env.sh +fi + +. /lib/lsb/init-functions + +# Are we running from init? +run_by_init() { + ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ] +} + +check_for_no_start() { + # forget it if we're trying to start, and /etc/hadoop/hadoop-jobtracker_not_to_be_run exists + if [ -e /etc/hadoop/hadoop-jobtracker_not_to_be_run ]; then + if [ "$1" = log_end_msg ]; then + log_end_msg 0 + fi + if ! run_by_init; then + log_action_msg "Apache Hadoop Job Tracker server not in use (/etc/hadoop/hadoop-jobtracker_not_to_be_run)" + fi + exit 0 + fi +} + +check_privsep_dir() { + # Create the PrivSep empty dir if necessary + if [ ! -d ${HADOOP_PID_DIR} ]; then + mkdir -p ${HADOOP_PID_DIR} + chown root:hadoop ${HADOOP_PID_DIR} + chmod 0775 ${HADOOP_PID_DIR} + fi +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin" + +case "$1" in + start) + check_privsep_dir + check_for_no_start + log_daemon_msg "Starting Apache Hadoop Job Tracker server" "hadoop-jobtracker" + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + stop) + log_daemon_msg "Stopping Apache Hadoop Job Tracker server" "hadoop-jobtracker" + if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + restart) + check_privsep_dir + log_daemon_msg "Restarting Apache Hadoop Job Tracker server" "hadoop-jobtracker" + start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + try-restart) + check_privsep_dir + log_daemon_msg "Restarting Apache Hadoop Job Tracker server" "hadoop-jobtracker" + set +e + start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid + RET="$?" + set -e + case $RET in + 0) + # old daemon stopped + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start jobtracker; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + 1) + # daemon not running + log_progress_msg "(not running)" + log_end_msg 0 + ;; + *) + # failed to stop + log_progress_msg "(failed to stop)" + log_end_msg 1 + ;; + esac + ;; + + status) + status_of_proc -p ${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid ${JAVA_HOME}/bin/java hadoop-jobtracker && exit 0 || exit $? + ;; + + *) + log_action_msg "Usage: /etc/init.d/hadoop-jobtracker {start|stop|restart|try-restart|status}" + exit 1 +esac + +exit 0 diff --git a/src/packages/deb/init.d/hadoop-namenode b/src/packages/deb/init.d/hadoop-namenode new file mode 100644 index 00000000000..520177facf8 --- /dev/null +++ b/src/packages/deb/init.d/hadoop-namenode @@ -0,0 +1,154 @@ +#! /bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### BEGIN INIT INFO +# Provides: hadoop-namenode +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: +# Short-Description: Apache Hadoop Name Node server +### END INIT INFO + +set -e + +# /etc/init.d/hadoop-namenode: start and stop the Apache Hadoop Name Node daemon + +test -x /usr/bin/hadoop || exit 0 +( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0 + +umask 022 + +if test -f /etc/default/hadoop-env.sh; then + . /etc/default/hadoop-env.sh +fi + +. /lib/lsb/init-functions + +# Are we running from init? +run_by_init() { + ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ] +} + +check_for_no_start() { + # forget it if we're trying to start, and /etc/hadoop/hadoop-namenode_not_to_be_run exists + if [ -e /etc/hadoop/hadoop-namenode_not_to_be_run ]; then + if [ "$1" = log_end_msg ]; then + log_end_msg 0 + fi + if ! run_by_init; then + log_action_msg "Apache Hadoop Name Node server not in use (/etc/hadoop/hadoop-namenode_not_to_be_run)" + fi + exit 0 + fi +} + +check_privsep_dir() { + # Create the PrivSep empty dir if necessary + if [ ! -d ${HADOOP_PID_DIR} ]; then + mkdir -p ${HADOOP_PID_DIR} + chown root:hadoop ${HADOOP_PID_DIR} + chmod 0775 ${HADOOP_PID_DIR} + fi +} + +format() { + su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format' hdfs +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin" + +case "$1" in + start) + check_privsep_dir + check_for_no_start + log_daemon_msg "Starting Apache Hadoop Name Node server" "hadoop-namenode" + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + stop) + log_daemon_msg "Stopping Apache Hadoop Name Node server" "hadoop-namenode" + if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + format) + log_daemon_msg "Formatting Apache Hadoop Name Node" "hadoop-namenode" + format + if [ $? -eq 0 ]; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + restart) + check_privsep_dir + log_daemon_msg "Restarting Apache Hadoop Name Node server" "hadoop-namenode" + start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + try-restart) + check_privsep_dir + log_daemon_msg "Restarting Apache Hadoop Name Node server" "hadoop-namenode" + set +e + start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid + RET="$?" + set -e + case $RET in + 0) + # old daemon stopped + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid -c hdfs -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start namenode; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + 1) + # daemon not running + log_progress_msg "(not running)" + log_end_msg 0 + ;; + *) + # failed to stop + log_progress_msg "(failed to stop)" + log_end_msg 1 + ;; + esac + ;; + + status) + status_of_proc -p ${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid ${JAVA_HOME}/bin/java hadoop-namenode && exit 0 || exit $? + ;; + + *) + log_action_msg "Usage: /etc/init.d/hadoop-namenode {start|stop|restart|try-restart|status}" + exit 1 +esac + +exit 0 diff --git a/src/packages/deb/init.d/hadoop-tasktracker b/src/packages/deb/init.d/hadoop-tasktracker new file mode 100644 index 00000000000..582875ce85a --- /dev/null +++ b/src/packages/deb/init.d/hadoop-tasktracker @@ -0,0 +1,142 @@ +#! /bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### BEGIN INIT INFO +# Provides: hadoop-tasktracker +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: +# Short-Description: Apache Hadoop Task Tracker server +### END INIT INFO + +set -e + +# /etc/init.d/hadoop-tasktracker: start and stop the Apache Hadoop Task Tracker daemon + +test -x /usr/bin/hadoop || exit 0 +( /usr/bin/hadoop 2>&1 | grep -q hadoop ) 2>/dev/null || exit 0 + +umask 022 + +if test -f /etc/default/hadoop-env.sh; then + . /etc/default/hadoop-env.sh +fi + +. /lib/lsb/init-functions + +# Are we running from init? +run_by_init() { + ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ] +} + +check_for_no_start() { + # forget it if we're trying to start, and /etc/hadoop/hadoop-tasktracker_not_to_be_run exists + if [ -e /etc/hadoop/hadoop-tasktracker_not_to_be_run ]; then + if [ "$1" = log_end_msg ]; then + log_end_msg 0 + fi + if ! run_by_init; then + log_action_msg "Apache Hadoop Task Tracker server not in use (/etc/hadoop/hadoop-tasktracker_not_to_be_run)" + fi + exit 0 + fi +} + +check_privsep_dir() { + # Create the PrivSep empty dir if necessary + if [ ! -d ${HADOOP_PID_DIR} ]; then + mkdir -p ${HADOOP_PID_DIR} + chown root:hadoop ${HADOOP_PID_DIR} + chmod 0775 ${HADOOP_PID_DIR} + fi +} + +export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin" + +case "$1" in + start) + check_privsep_dir + check_for_no_start + log_daemon_msg "Starting Apache Hadoop Task Tracker server" "hadoop-tasktracker" + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + stop) + log_daemon_msg "Stopping Apache Hadoop Task Tracker server" "hadoop-tasktracker" + if start-stop-daemon --stop --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + restart) + check_privsep_dir + log_daemon_msg "Restarting Apache Hadoop Task Tracker server" "hadoop-tasktracker" + start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + + try-restart) + check_privsep_dir + log_daemon_msg "Restarting Apache Hadoop Task Tracker server" "hadoop-tasktracker" + set +e + start-stop-daemon --stop --quiet --retry 30 --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid + RET="$?" + set -e + case $RET in + 0) + # old daemon stopped + check_for_no_start log_end_msg + if start-stop-daemon --start --quiet --oknodo --pidfile ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid -c mapred -x ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh -- --config ${HADOOP_CONF_DIR} start tasktracker; then + log_end_msg 0 + else + log_end_msg 1 + fi + ;; + 1) + # daemon not running + log_progress_msg "(not running)" + log_end_msg 0 + ;; + *) + # failed to stop + log_progress_msg "(failed to stop)" + log_end_msg 1 + ;; + esac + ;; + + status) + status_of_proc -p ${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid ${JAVA_HOME}/bin/java hadoop-tasktracker && exit 0 || exit $? + ;; + + *) + log_action_msg "Usage: /etc/init.d/hadoop-tasktracker {start|stop|restart|try-restart|status}" + exit 1 +esac + +exit 0 diff --git a/src/packages/hadoop-create-user.sh b/src/packages/hadoop-create-user.sh new file mode 100644 index 00000000000..2426d4dc8b5 --- /dev/null +++ b/src/packages/hadoop-create-user.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +if [ "$HADOOP_HOME" != "" ]; then + echo "Warning: \$HADOOP_HOME is deprecated." + echo +fi + +. "$bin"/../libexec/hadoop-config.sh + +usage() { + echo " +usage: $0 + Require parameter: + -u Create user on HDFS + Optional parameters: + -h Display this message + " + exit 1 +} + +# Parse script parameters +if [ $# != 2 ] ; then + usage + exit 1 +fi + +while getopts "hu:" OPTION +do + case $OPTION in + u) + SETUP_USER=$2; shift 2 + ;; + h) + usage + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +# Create user directory on HDFS +export SETUP_USER +export SETUP_PATH=/user/${SETUP_USER} +export HADOOP_PREFIX +export HADOOP_CONF_DIR + +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir ${SETUP_PATH}' hdfs +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown ${SETUP_USER}:${SETUP_USER} ${SETUP_PATH}' hdfs + +if [ "$?" == "0" ]; then + echo "User directory has been setup: ${SETUP_PATH}" +fi diff --git a/src/packages/hadoop-setup-conf.sh b/src/packages/hadoop-setup-conf.sh new file mode 100644 index 00000000000..eb72b86b2ec --- /dev/null +++ b/src/packages/hadoop-setup-conf.sh @@ -0,0 +1,282 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +if [ "$HADOOP_HOME" != "" ]; then + echo "Warning: \$HADOOP_HOME is deprecated." + echo +fi + +. "$bin"/../libexec/hadoop-config.sh + +usage() { + echo " +usage: $0 + + Optional parameters: + --auto Setup automatically + --default Generate default config + --conf-dir=/etc/hadoop Set config directory + --datanode-dir=/var/lib/hadoop/hdfs/datanode Set datanode directory + -h Display this message + --jobtracker-url=hostname:9001 Set jobtracker url + --log-dir=/var/log/hadoop Set log directory + --hdfs-dir=/var/lib/hadoop/hdfs Set hdfs directory + --mapred-dir=/var/lib/hadoop/mapred Set mapreduce directory + --namenode-dir=/var/lib/hadoop/hdfs/namenode Set namenode directory + --namenode-url=hdfs://hostname:9000/ Set namenode url + --replication=3 Set replication factor + --taskscheduler=org.apache.hadoop.mapred.JobQueueTaskScheduler Set task scheduler + " + exit 1 +} + +template_generator() { + REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})' + cat $1 | + while read line ; do + while [[ "$line" =~ $REGEX ]] ; do + LHS=${BASH_REMATCH[1]} + RHS="$(eval echo "\"$LHS\"")" + line=${line//$LHS/$RHS} + done + echo $line >> $2 + done +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'auto' \ + -l 'conf-dir:' \ + -l 'default' \ + -l 'hdfs-dir:' \ + -l 'namenode-dir:' \ + -l 'datanode-dir:' \ + -l 'mapred-dir:' \ + -l 'namenode-url:' \ + -l 'jobtracker-url:' \ + -l 'log-dir:' \ + -l 'replication:' \ + -l 'taskscheduler:' \ + -o 'h' \ + -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +# Make sure the HADOOP_LOG_DIR is not picked up from user environment. +unset HADOOP_LOG_DIR + +# Parse script parameters +eval set -- "${OPTS}" +while true ; do + case "$1" in + --auto) + AUTOSETUP=1 + AUTOMATED=1 + shift + ;; + --conf-dir) + HADOOP_CONF_DIR=$2; shift 2 + AUTOMATED=1 + ;; + --default) + AUTOMATED=1; shift + ;; + -h) + usage + ;; + --hdfs-dir) + HADOOP_HDFS_DIR=$2; shift 2 + AUTOMATED=1 + ;; + --namenode-dir) + HADOOP_NN_DIR=$2; shift 2 + AUTOMATED=1 + ;; + --datanode-dir) + HADOOP_DN_DIR=$2; shift 2 + AUTOMATED=1 + ;; + --mapred-dir) + HADOOP_MAPRED_DIR=$2; shift 2 + AUTOMATED=1 + ;; + --namenode-url) + HADOOP_NN_HOST=$2; shift 2 + AUTOMATED=1 + ;; + --jobtracker-url) + HADOOP_JT_HOST=$2; shift 2 + AUTOMATED=1 + ;; + --log-dir) + HADOOP_LOG_DIR=$2; shift 2 + AUTOMATED=1 + ;; + --replication) + HADOOP_REPLICATION=$2; shift 2 + AUTOMATED=1 + ;; + --taskscheduler) + HADOOP_TASK_SCHEDULER=$2; shift 2 + AUTOMATED=1 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +# Fill in default values, if parameters have not been defined. +AUTOSETUP=${AUTOSETUP:-1} +JAVA_HOME=${JAVA_HOME:-/usr/java/default} +HADOOP_NN_HOST=${HADOOP_NN_HOST:-hdfs://`hostname`:9000/} +HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode} +HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode} +HADOOP_JT_HOST=${HADOOP_JT_HOST:-`hostname`:9001} +HADOOP_HDFS_DIR=${HADOOP_HDFS_DIR:-/var/lib/hadoop/hdfs} +HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred} +HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-/var/log/hadoop} +HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop} +HADOOP_REPLICATION=${HADOOP_RELICATION:-3} +HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler} + +# Interactive setup wizard +if [ "${AUTOMATED}" != "1" ]; then + echo "Setup Hadoop Configuration" + echo + echo -n "Where would you like to put config directory? (${HADOOP_CONF_DIR}) " + read USER_HADOOP_CONF_DIR + echo -n "Where would you like to put log directory? (${HADOOP_LOG_DIR}) " + read USER_HADOOP_LOG_DIR + echo -n "What is the url of the namenode? (${HADOOP_NN_HOST}) " + read USER_HADOOP_NN_HOST + echo -n "Where would you like to put namenode data directory? (${HADOOP_NN_DIR}) " + read USER_HADOOP_NN_DIR + echo -n "Where would you like to put datanode data directory? (${HADOOP_DN_DIR}) " + read USER_HADOOP_DN_DIR + echo -n "What is the url of the jobtracker? (${HADOOP_JT_HOST}) " + read USER_HADOOP_JT_HOST + echo -n "Where would you like to put jobtracker/tasktracker data directory? (${HADOOP_MAPRED_DIR}) " + read USER_HADOOP_MAPRED_DIR + echo -n "Which taskscheduler would you like? (${HADOOP_TASK_SCHEDULER}) " + read USER_HADOOP_TASK_SCHEDULER + echo -n "Where is JAVA_HOME directory? (${JAVA_HOME}) " + read USER_JAVA_HOME + echo -n "Would you like to create directories/copy conf files to localhost? (Y/n) " + read USER_AUTOSETUP + echo + JAVA_HOME=${USER_USER_JAVA_HOME:-$JAVA_HOME} + HADOOP_NN_HOST=${USER_HADOOP_NN_HOST:-$HADOOP_NN_HOST} + HADOOP_NN_DIR=${USER_HADOOP_NN_DIR:-$HADOOP_NN_DIR} + HADOOP_DN_DIR=${USER_HADOOP_DN_DIR:-$HADOOP_DN_DIR} + HADOOP_JT_HOST=${USER_HADOOP_JT_HOST:-$HADOOP_JT_HOST} + HADOOP_HDFS_DIR=${USER_HADOOP_HDFS_DIR:-$HADOOP_HDFS_DIR} + HADOOP_MAPRED_DIR=${USER_HADOOP_MAPRED_DIR:-$HADOOP_MAPRED_DIR} + HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler} + HADOOP_LOG_DIR=${USER_HADOOP_LOG_DIR:-$HADOOP_LOG_DIR} + HADOOP_CONF_DIR=${USER_HADOOP_CONF_DIR:-$HADOOP_CONF_DIR} + AUTOSETUP=${USER_AUTOSETUP:-y} + echo "Review your choices:" + echo + echo "Config directory : ${HADOOP_CONF_DIR}" + echo "Log directory : ${HADOOP_LOG_DIR}" + echo "Namenode url : ${HADOOP_NN_HOST}" + echo "Namenode directory : ${HADOOP_NN_DIR}" + echo "Datanode directory : ${HADOOP_DN_DIR}" + echo "Jobtracker url : ${HADOOP_JT_HOST}" + echo "Mapreduce directory : ${HADOOP_MAPRED_DIR}" + echo "Task scheduler : ${HADOOP_TASK_SCHEDULER}" + echo "JAVA_HOME directory : ${JAVA_HOME}" + echo "Create dirs/copy conf files : ${AUTOSETUP}" + echo + echo -n "Proceed with generate configuration? (y/N) " + read CONFIRM + if [ "${CONFIRM}" != "y" ]; then + echo "User aborted setup, exiting..." + exit 1 + fi +fi + +if [ "${AUTOSETUP}" == "1" ]; then + # If user wants to setup local system automatically, + # set config file generation location to HADOOP_CONF_DIR. + DEST=${HADOOP_CONF_DIR} +else + # If user is only interested to generate config file locally, + # place config files in the current working directory. + DEST=`pwd` +fi + +# remove existing config file, they are existed in current directory. +rm -f ${DEST}/core-site.xml >/dev/null +rm -f ${DEST}/hdfs-site.xml >/dev/null +rm -f ${DEST}/mapred-site.xml >/dev/null +rm -f ${DEST}/hadoop-env.sh >/dev/null + +# Generate config file with specified parameters. +template_generator ${HADOOP_PREFIX}/share/hadoop/common/templates/core-site.xml ${DEST}/core-site.xml +template_generator ${HADOOP_PREFIX}/share/hadoop/hdfs/templates/hdfs-site.xml ${DEST}/hdfs-site.xml +template_generator ${HADOOP_PREFIX}/share/hadoop/mapreduce/templates/mapred-site.xml ${DEST}/mapred-site.xml +template_generator ${HADOOP_CONF_DIR}/hadoop-env.sh.template ${DEST}/hadoop-env.sh + +chown root:hadoop ${DEST}/hadoop-env.sh +chmod 755 ${DEST}/hadoop-env.sh + +# Setup directory path and copy config files, if AUTOSETUP is chosen. +if [ "${AUTOSETUP}" == "1" -o "${AUTOSETUP}" == "y" ]; then + mkdir -p ${HADOOP_HDFS_DIR} + mkdir -p ${HADOOP_NN_DIR} + mkdir -p ${HADOOP_DN_DIR} + mkdir -p ${HADOOP_MAPRED_DIR} + mkdir -p ${HADOOP_CONF_DIR} + mkdir -p ${HADOOP_LOG_DIR} + mkdir -p ${HADOOP_LOG_DIR}/hdfs + mkdir -p ${HADOOP_LOG_DIR}/mapred + chown hdfs:hadoop ${HADOOP_HDFS_DIR} + chown hdfs:hadoop ${HADOOP_NN_DIR} + chown hdfs:hadoop ${HADOOP_DN_DIR} + chown mapred:hadoop ${HADOOP_MAPRED_DIR} + chown root:hadoop ${HADOOP_LOG_DIR} + chmod 775 ${HADOOP_LOG_DIR} + chown hdfs:hadoop ${HADOOP_LOG_DIR}/hdfs + chown mapred:hadoop ${HADOOP_LOG_DIR}/mapred + echo "Configuration setup is completed." + if [[ "$HADOOP_NN_HOST" =~ "`hostname`" ]]; then + echo "Proceed to run hadoop-setup-hdfs.sh on namenode." + fi +else + echo + echo "Configuration file has been generated, please copy:" + echo + echo "core-site.xml" + echo "hdfs-site.xml" + echo "mapred-site.xml" + echo "hadoop-env.sh" + echo + echo " to ${HADOOP_CONF_DIR} on all nodes, and proceed to run hadoop-setup-hdfs.sh on namenode." +fi diff --git a/src/packages/hadoop-setup-hdfs.sh b/src/packages/hadoop-setup-hdfs.sh new file mode 100644 index 00000000000..8ac6f312c5a --- /dev/null +++ b/src/packages/hadoop-setup-hdfs.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +if [ "$HADOOP_HOME" != "" ]; then + echo "Warning: \$HADOOP_HOME is deprecated." + echo +fi + +. "$bin"/../libexec/hadoop-config.sh + +usage() { + echo " +usage: $0 + Require parameter: + -c Set cluster identifier for HDFS + Optional parameters: + -h Display this message + " + exit 1 +} + +if [ $# != 2 ] ; then + usage + exit 1 +fi + +while getopts "hc:" OPTION +do + case $OPTION in + c) + SETUP_CLUSTER=$2; shift 2 + ;; + h) + usage + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +export HADOOP_PREFIX +export HADOOP_CONF_DIR +export SETUP_CLUSTER + +# Start namenode and initialize file system structure +echo "Setup Hadoop Distributed File System" +echo +echo "Formatting namenode" +echo +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} namenode -format -clusterid ${SETUP_CLUSTER}' hdfs +echo +echo "Starting namenode process" +echo +/etc/init.d/hadoop-namenode start +echo +echo "Initialize HDFS file system: " +echo + +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /jobtracker' hdfs +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /jobtracker' hdfs +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /user/mapred' hdfs +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /user/mapred' hdfs +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /tmp' hdfs +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chmod 777 /tmp' hdfs + +if [ $? -eq 0 ]; then + echo "Completed." +else + echo "Unknown error occurred, check hadoop logs for details." +fi + +echo +echo "Please startup datanode processes: /etc/init.d/hadoop-datanode start" diff --git a/src/packages/hadoop-setup-single-node.sh b/src/packages/hadoop-setup-single-node.sh new file mode 100644 index 00000000000..1ab40b75fc6 --- /dev/null +++ b/src/packages/hadoop-setup-single-node.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script for setup HDFS file system for single node deployment + +bin=`which $0` +bin=`dirname ${bin}` +bin=`cd "$bin"; pwd` + +export HADOOP_PREFIX=${bin}/.. + +if [ -e /etc/hadoop/hadoop-env.sh ]; then + . /etc/hadoop/hadoop-env.sh +fi + +usage() { + echo " +usage: $0 + + Optional parameters: + --default Setup system as default + -h Display this message + " + exit 1 +} + +# Parse script parameters +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'default' \ + -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "${OPTS}" +while true ; do + case "$1" in + --default) + AUTOMATED=1; shift + ;; + -h) + usage + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +# Interactive setup wizard +if [ "${AUTOMATED}" != "1" ]; then + echo "Welcome to Hadoop single node setup wizard" + echo + echo -n "Would you like to use default single node configuration? (y/n) " + read SET_CONFIG + echo -n "Would you like to format name node? (y/n) " + read SET_FORMAT + echo -n "Would you like to setup default directory structure? (y/n) " + read SET_MKDIR + echo -n "Would you like to start up Hadoop? (y/n) " + read STARTUP + echo -n "Would you like to start up Hadoop on reboot? (y/n) " + read SET_REBOOT + echo + echo "Review your choices:" + echo + echo "Setup single node configuration : ${SET_CONFIG}" + echo "Format namenode : ${SET_FORMAT}" + echo "Setup default file system structure: ${SET_MKDIR}" + echo "Start up Hadoop : ${STARTUP}" + echo "Start up Hadoop on reboot : ${SET_REBOOT}" + echo + echo -n "Proceed with setup? (y/n) " + read CONFIRM + if [ "${CONFIRM}" != "y" ]; then + echo "User aborted setup, exiting..." + exit 1 + fi +else + SET_CONFIG="y" + SET_FORMAT="y" + SET_MKDIR="y" + STARTUP="y" + SET_REBOOT="y" +fi + +AUTOMATED=${AUTOMATED:-0} +SET_CONFIG=${SET_CONFIG:-y} +SET_FORMAT=${SET_FORMAT:-n} +SET_MKDIR=${SET_MKDIR:-y} +STARTUP=${STARTUP:-y} +SET_REBOOT=${SET_REBOOT:-y} + +# Make sure system is not already started +/etc/init.d/hadoop-namenode stop 2>/dev/null >/dev/null +/etc/init.d/hadoop-datanode stop 2>/dev/null >/dev/null +/etc/init.d/hadoop-jobtracker stop 2>/dev/null >/dev/null +/etc/init.d/hadoop-tasktracker stop 2>/dev/null >/dev/null + +# Default settings +JAVA_HOME=${JAVA_HOME:-/usr/java/default} +HADOOP_NN_HOST=${HADOOP_NN_HOST:-hdfs://localhost:9000/} +HADOOP_NN_DIR=${HADOOP_NN_DIR:-/var/lib/hadoop/hdfs/namenode} +HADOOP_DN_DIR=${HADOOP_DN_DIR:-/var/lib/hadoop/hdfs/datanode} +HADOOP_JT_HOST=${HADOOP_JT_HOST:-localhost:9001} +HADOOP_HDFS_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/hdfs} +HADOOP_MAPRED_DIR=${HADOOP_MAPRED_DIR:-/var/lib/hadoop/mapred} +HADOOP_LOG_DIR="/var/log/hadoop" +HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop} +HADOOP_REPLICATION=${HADOOP_RELICATION:-1} +HADOOP_TASK_SCHEDULER=${HADOOP_TASK_SCHEDULER:-org.apache.hadoop.mapred.JobQueueTaskScheduler} + +# Setup config files +if [ "${SET_CONFIG}" == "y" ]; then + ${HADOOP_PREFIX}/sbin/hadoop-setup-conf.sh --auto \ + --conf-dir=${HADOOP_CONF_DIR} \ + --datanode-dir=${HADOOP_DN_DIR} \ + --hdfs-dir=${HADOOP_HDFS_DIR} \ + --jobtracker-url=${HADOOP_JT_HOST} \ + --log-dir=${HADOOP_LOG_DIR} \ + --mapred-dir=${HADOOP_MAPRED_DIR} \ + --namenode-dir=${HADOOP_NN_DIR} \ + --namenode-url=${HADOOP_NN_HOST} \ + --replication=${HADOOP_REPLICATION} +fi + +export HADOOP_CONF_DIR + +# Format namenode +if [ ! -e ${HADOOP_NN_DIR} ]; then + rm -rf ${HADOOP_HDFS_DIR} 2>/dev/null >/dev/null + mkdir -p ${HADOOP_HDFS_DIR} + chmod 755 ${HADOOP_HDFS_DIR} + chown hdfs:hadoop ${HADOOP_HDFS_DIR} + su -c '${HADOOP_PREFIX}/bin/hdfs --config ${HADOOP_CONF_DIR} namenode -format -clusterid hadoop' hdfs +elif [ "${SET_FORMAT}" == "y" ]; then + rm -rf ${HADOOP_HDFS_DIR} 2>/dev/null >/dev/null + mkdir -p ${HADOOP_HDFS_DIR} + chmod 755 ${HADOOP_HDFS_DIR} + chown hdfs:hadoop ${HADOOP_HDFS_DIR} + rm -rf /var/lib/hadoop/hdfs/namenode + su -c '${HADOOP_PREFIX}/bin/hdfs --config ${HADOOP_CONF_DIR} namenode -format -clusterid hadoop' hdfs +fi + +# Start hdfs service +/etc/init.d/hadoop-namenode start +/etc/init.d/hadoop-datanode start + +# Initialize file system structure +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /user/mapred' hdfs +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /user/mapred' hdfs +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /tmp' hdfs +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chmod 777 /tmp' hdfs +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -mkdir /jobtracker' hdfs +su -c '${HADOOP_PREFIX}/bin/hadoop --config ${HADOOP_CONF_DIR} fs -chown mapred:mapred /jobtracker' hdfs + +# Start mapreduce service +/etc/init.d/hadoop-jobtracker start +/etc/init.d/hadoop-tasktracker start + +# Toggle service startup on reboot +if [ "${SET_REBOOT}" == "y" ]; then + if [ -e /etc/debian_version ]; then + ln -sf ../init.d/hadoop-namenode /etc/rc2.d/S90hadoop-namenode + ln -sf ../init.d/hadoop-datanode /etc/rc2.d/S91hadoop-datanode + ln -sf ../init.d/hadoop-jobtracker /etc/rc2.d/S92hadoop-jobtracker + ln -sf ../init.d/hadoop-tasktracker /etc/rc2.d/S93hadoop-tasktracker + ln -sf ../init.d/hadoop-namenode /etc/rc6.d/S10hadoop-namenode + ln -sf ../init.d/hadoop-datanode /etc/rc6.d/S11hadoop-datanode + ln -sf ../init.d/hadoop-jobtracker /etc/rc6.d/S12hadoop-jobtracker + ln -sf ../init.d/hadoop-tasktracker /etc/rc6.d/S13hadoop-tasktracker + elif [ -e /etc/redhat-release ]; then + /sbin/chkconfig hadoop-namenode --add + /sbin/chkconfig hadoop-datanode --add + /sbin/chkconfig hadoop-jobtracker --add + /sbin/chkconfig hadoop-tasktracker --add + /sbin/chkconfig hadoop-namenode on + /sbin/chkconfig hadoop-datanode on + /sbin/chkconfig hadoop-jobtracker on + /sbin/chkconfig hadoop-tasktracker on + fi +fi + +# Shutdown service, if user choose to stop services after setup +if [ "${STARTUP}" != "y" ]; then + /etc/init.d/hadoop-namenode stop + /etc/init.d/hadoop-datanode stop + /etc/init.d/hadoop-jobtracker stop + /etc/init.d/hadoop-tasktracker stop +fi diff --git a/src/packages/rpm/init.d/hadoop-datanode b/src/packages/rpm/init.d/hadoop-datanode new file mode 100644 index 00000000000..019051b1505 --- /dev/null +++ b/src/packages/rpm/init.d/hadoop-datanode @@ -0,0 +1,84 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Starts a Hadoop datanode +# +# chkconfig: 2345 90 10 +# description: Hadoop datanode + +source /etc/rc.d/init.d/functions +source /etc/default/hadoop-env.sh + +RETVAL=0 +PIDFILE="${HADOOP_PID_DIR}/hadoop-hdfs-datanode.pid" +desc="Hadoop datanode daemon" + +start() { + echo -n $"Starting $desc (hadoop-datanode): " + daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start datanode + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-datanode + return $RETVAL +} + +stop() { + echo -n $"Stopping $desc (hadoop-datanode): " + daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop datanode + RETVAL=$? + sleep 5 + echo + [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-datanode $PIDFILE +} + +restart() { + stop + start +} + +checkstatus(){ + status -p $PIDFILE ${JAVA_HOME}/bin/java + RETVAL=$? +} + +condrestart(){ + [ -e /var/lock/subsys/hadoop-datanode ] && restart || : +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + checkstatus + ;; + restart) + restart + ;; + condrestart) + condrestart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart}" + exit 1 +esac + +exit $RETVAL diff --git a/src/packages/rpm/init.d/hadoop-jobtracker b/src/packages/rpm/init.d/hadoop-jobtracker new file mode 100644 index 00000000000..7aa6e28c7b8 --- /dev/null +++ b/src/packages/rpm/init.d/hadoop-jobtracker @@ -0,0 +1,84 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Starts a Hadoop jobtracker +# +# chkconfig: 2345 90 10 +# description: Hadoop jobtracker + +source /etc/rc.d/init.d/functions +source /etc/default/hadoop-env.sh + +RETVAL=0 +PIDFILE="${HADOOP_PID_DIR}/hadoop-mapred-jobtracker.pid" +desc="Hadoop jobtracker daemon" + +start() { + echo -n $"Starting $desc (hadoop-jobtracker): " + daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start jobtracker + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-jobtracker + return $RETVAL +} + +stop() { + echo -n $"Stopping $desc (hadoop-jobtracker): " + daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop jobtracker + RETVAL=$? + sleep 5 + echo + [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-jobtracker $PIDFILE +} + +restart() { + stop + start +} + +checkstatus(){ + status -p $PIDFILE ${JAVA_HOME}/bin/java + RETVAL=$? +} + +condrestart(){ + [ -e /var/lock/subsys/hadoop-jobtracker ] && restart || : +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + checkstatus + ;; + restart) + restart + ;; + condrestart) + condrestart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart}" + exit 1 +esac + +exit $RETVAL diff --git a/src/packages/rpm/init.d/hadoop-namenode b/src/packages/rpm/init.d/hadoop-namenode new file mode 100644 index 00000000000..dd0c94bf0ff --- /dev/null +++ b/src/packages/rpm/init.d/hadoop-namenode @@ -0,0 +1,98 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Starts a Hadoop namenode +# +# chkconfig: 2345 90 10 +# description: Hadoop namenode + +source /etc/rc.d/init.d/functions +source /etc/default/hadoop-env.sh + +RETVAL=0 +PIDFILE="${HADOOP_PID_DIR}/hadoop-hdfs-namenode.pid" +desc="Hadoop namenode daemon" + +start() { + echo -n $"Starting $desc (hadoop-namenode): " + daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start namenode $1 + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-namenode + return $RETVAL +} + +upgrade() { + start -upgrade +} + +stop() { + echo -n $"Stopping $desc (hadoop-namenode): " + daemon --user hdfs ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop namenode + RETVAL=$? + sleep 5 + echo + [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-namenode $PIDFILE +} + +checkstatus(){ + status -p $PIDFILE ${JAVA_HOME}/bin/java + RETVAL=$? +} + +restart() { + stop + start +} + +condrestart(){ + [ -e /var/lock/subsys/hadoop-namenode ] && restart || : +} + +format() { + daemon --user hdfs ${HADOOP_PREFIX}/bin/hadoop namenode -format +} + +case "$1" in + start) + start + ;; + upgrade) + upgrade + ;; + format) + format + ;; + stop) + stop + ;; + status) + checkstatus + ;; + restart) + restart + ;; + condrestart|try-restart) + condrestart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|try-restart|upgrade}" + exit 1 +esac + +exit $RETVAL diff --git a/src/packages/rpm/init.d/hadoop-tasktracker b/src/packages/rpm/init.d/hadoop-tasktracker new file mode 100644 index 00000000000..303ed869fd6 --- /dev/null +++ b/src/packages/rpm/init.d/hadoop-tasktracker @@ -0,0 +1,84 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Starts a Hadoop tasktracker +# +# chkconfig: 2345 90 10 +# description: Hadoop tasktracker + +source /etc/rc.d/init.d/functions +source /etc/default/hadoop-env.sh + +RETVAL=0 +PIDFILE="${HADOOP_PID_DIR}/hadoop-mapred-tasktracker.pid" +desc="Hadoop tasktracker daemon" + +start() { + echo -n $"Starting $desc (hadoop-tasktracker): " + daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" start tasktracker + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch /var/lock/subsys/hadoop-tasktracker + return $RETVAL +} + +stop() { + echo -n $"Stopping $desc (hadoop-tasktracker): " + daemon --user mapred ${HADOOP_PREFIX}/sbin/hadoop-daemon.sh --config "${HADOOP_CONF_DIR}" stop tasktracker + RETVAL=$? + sleep 5 + echo + [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/hadoop-tasktracker $PIDFILE +} + +restart() { + stop + start +} + +checkstatus(){ + status -p $PIDFILE ${JAVA_HOME}/bin/java + RETVAL=$? +} + +condrestart(){ + [ -e /var/lock/subsys/hadoop-tasktracker ] && restart || : +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + checkstatus + ;; + restart) + restart + ;; + condrestart) + condrestart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart}" + exit 1 +esac + +exit $RETVAL diff --git a/src/packages/rpm/spec/hadoop.spec b/src/packages/rpm/spec/hadoop.spec new file mode 100644 index 00000000000..33cf8476f17 --- /dev/null +++ b/src/packages/rpm/spec/hadoop.spec @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# RPM Spec file for Hadoop version @version@ +# + +%define name hadoop-common +%define version @version@ +%define release @package.release@ + +# Installation Locations +%define _prefix @package.prefix@ +%define _bin_dir %{_prefix}/bin +%define _conf_dir @package.conf.dir@ +%define _lib_dir %{_prefix}/lib +%define _lib64_dir %{_prefix}/lib64 +%define _libexec_dir %{_prefix}/libexec +%define _log_dir @package.log.dir@ +%define _pid_dir @package.pid.dir@ +%define _sbin_dir %{_prefix}/sbin +%define _share_dir %{_prefix}/share +%define _var_dir @package.var.dir@ + +# Build time settings +%define _build_dir @package.build.dir@ +%define _final_name @final.name@ +%define debug_package %{nil} + +# Disable brp-java-repack-jars for aspect J +%define __os_install_post \ + /usr/lib/rpm/redhat/brp-compress \ + %{!?__debug_package:/usr/lib/rpm/redhat/brp-strip %{__strip}} \ + /usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \ + /usr/lib/rpm/redhat/brp-strip-comment-note %{__strip} %{__objdump} \ + /usr/lib/rpm/brp-python-bytecompile %{nil} + +# RPM searches perl files for dependancies and this breaks for non packaged perl lib +# like thrift so disable this +%define _use_internal_dependency_generator 0 + +%ifarch i386 +%global hadoop_arch Linux-i386-32 +%endif +%ifarch amd64 x86_64 +%global hadoop_arch Linux-amd64-64 +%endif +%ifarch noarch +%global hadoop_arch "" +%endif + +Summary: The Apache Hadoop project develops open-source software for reliable, scalable, distributed computing +License: Apache License, Version 2.0 +URL: http://hadoop.apache.org/core/ +Vendor: Apache Software Foundation +Group: Development/Libraries +Name: %{name} +Version: %{version} +Release: %{release} +Source0: %{_final_name}-bin.tar.gz +Prefix: %{_prefix} +Prefix: %{_conf_dir} +Prefix: %{_log_dir} +Prefix: %{_pid_dir} +Buildroot: %{_build_dir} +Requires: sh-utils, textutils, /usr/sbin/useradd, /usr/sbin/usermod, /sbin/chkconfig, /sbin/service, jdk >= 1.6 +AutoReqProv: no +Provides: hadoop + +%description +The Apache Hadoop project develops open-source software for reliable, scalable, +distributed computing. Hadoop includes these subprojects: + +Hadoop Common: The common utilities that support the other Hadoop subprojects. + +%prep +%setup -n %{_final_name} + +%build +if [ -d ${RPM_BUILD_DIR}%{_prefix} ]; then + rm -rf ${RPM_BUILD_DIR}%{_prefix} +fi + +if [ -d ${RPM_BUILD_DIR}%{_log_dir} ]; then + rm -rf ${RPM_BUILD_DIR}%{_log_dir} +fi + +if [ -d ${RPM_BUILD_DIR}%{_conf_dir} ]; then + rm -rf ${RPM_BUILD_DIR}%{_conf_dir} +fi + +if [ -d ${RPM_BUILD_DIR}%{_pid_dir} ]; then + rm -rf ${RPM_BUILD_DIR}%{_pid_dir} +fi + +mkdir -p ${RPM_BUILD_DIR}%{_prefix} +mkdir -p ${RPM_BUILD_DIR}%{_bin_dir} +mkdir -p ${RPM_BUILD_DIR}%{_lib_dir} +%ifarch amd64 x86_64 +mkdir -p ${RPM_BUILD_DIR}%{_lib64_dir} +%endif +mkdir -p ${RPM_BUILD_DIR}%{_libexec_dir} +mkdir -p ${RPM_BUILD_DIR}%{_log_dir} +mkdir -p ${RPM_BUILD_DIR}%{_conf_dir} +mkdir -p ${RPM_BUILD_DIR}%{_pid_dir} +mkdir -p ${RPM_BUILD_DIR}%{_sbin_dir} +mkdir -p ${RPM_BUILD_DIR}%{_share_dir} +mkdir -p ${RPM_BUILD_DIR}%{_var_dir} + +######################### +#### INSTALL SECTION #### +######################### +%install +mv ${RPM_BUILD_DIR}/%{_final_name}/bin/* ${RPM_BUILD_DIR}%{_bin_dir} +mv ${RPM_BUILD_DIR}/%{_final_name}/etc/hadoop/* ${RPM_BUILD_DIR}%{_conf_dir} +mv ${RPM_BUILD_DIR}/%{_final_name}/lib/* ${RPM_BUILD_DIR}%{_lib_dir} +mv ${RPM_BUILD_DIR}/%{_final_name}/libexec/* ${RPM_BUILD_DIR}%{_libexec_dir} +mv ${RPM_BUILD_DIR}/%{_final_name}/sbin/* ${RPM_BUILD_DIR}%{_sbin_dir} +mv ${RPM_BUILD_DIR}/%{_final_name}/share/* ${RPM_BUILD_DIR}%{_share_dir} +rm -rf ${RPM_BUILD_DIR}/%{_final_name}/etc + +%pre +getent group hadoop 2>/dev/null >/dev/null || /usr/sbin/groupadd -r hadoop + +%post +bash ${RPM_INSTALL_PREFIX0}/sbin/update-hadoop-env.sh \ + --prefix=${RPM_INSTALL_PREFIX0} \ + --bin-dir=${RPM_INSTALL_PREFIX0}/bin \ + --sbin-dir=${RPM_INSTALL_PREFIX0}/sbin \ + --conf-dir=${RPM_INSTALL_PREFIX1} \ + --log-dir=${RPM_INSTALL_PREFIX2} \ + --pid-dir=${RPM_INSTALL_PREFIX3} + +%preun +bash ${RPM_INSTALL_PREFIX0}/sbin/update-hadoop-env.sh \ + --prefix=${RPM_INSTALL_PREFIX0} \ + --bin-dir=${RPM_INSTALL_PREFIX0}/bin \ + --sbin-dir=${RPM_INSTALL_PREFIX0}/sbin \ + --conf-dir=${RPM_INSTALL_PREFIX1} \ + --log-dir=${RPM_INSTALL_PREFIX2} \ + --pid-dir=${RPM_INSTALL_PREFIX3} \ + --uninstall + +%files +%defattr(-,root,root) +%attr(0755,root,hadoop) %{_log_dir} +%attr(0775,root,hadoop) %{_pid_dir} +%config(noreplace) %{_conf_dir}/configuration.xsl +%config(noreplace) %{_conf_dir}/core-site.xml +%config(noreplace) %{_conf_dir}/hadoop-env.sh +%config(noreplace) %{_conf_dir}/hadoop-metrics.properties +%config(noreplace) %{_conf_dir}/hadoop-metrics2.properties +%config(noreplace) %{_conf_dir}/hadoop-policy.xml +%config(noreplace) %{_conf_dir}/log4j.properties +%config(noreplace) %{_conf_dir}/masters +%config(noreplace) %{_conf_dir}/slaves +%{_conf_dir}/hadoop-env.sh.template +%{_conf_dir}/ssl-client.xml.example +%{_conf_dir}/ssl-server.xml.example +%{_prefix} + diff --git a/src/packages/templates/conf/core-site.xml b/src/packages/templates/conf/core-site.xml new file mode 100644 index 00000000000..c4797503e52 --- /dev/null +++ b/src/packages/templates/conf/core-site.xml @@ -0,0 +1,11 @@ + + + + + + + + fs.default.name + ${HADOOP_NN_HOST} + + diff --git a/src/packages/update-hadoop-env.sh b/src/packages/update-hadoop-env.sh new file mode 100644 index 00000000000..eb17344dfca --- /dev/null +++ b/src/packages/update-hadoop-env.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script configures hadoop-env.sh and symlinkis directories for +# relocating RPM locations. + +usage() { + echo " +usage: $0 + Required parameters: + --prefix=PREFIX path to install into + + Optional parameters: + --arch=i386 OS Architecture + --bin-dir=PREFIX/bin Executable directory + --conf-dir=/etc/hadoop Configuration directory + --log-dir=/var/log/hadoop Log directory + --pid-dir=/var/run PID file location + --sbin-dir=PREFIX/sbin System executable directory + " + exit 1 +} + +template_generator() { + REGEX='(\$\{[a-zA-Z_][a-zA-Z_0-9]*\})' + cat $1 | + while read line ; do + while [[ "$line" =~ $REGEX ]] ; do + LHS=${BASH_REMATCH[1]} + RHS="$(eval echo "\"$LHS\"")" + line=${line//$LHS/$RHS} + done + echo $line >> $2 + done +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'arch:' \ + -l 'prefix:' \ + -l 'bin-dir:' \ + -l 'conf-dir:' \ + -l 'lib-dir:' \ + -l 'log-dir:' \ + -l 'pid-dir:' \ + -l 'sbin-dir:' \ + -l 'uninstall' \ + -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "${OPTS}" +while true ; do + case "$1" in + --arch) + ARCH=$2 ; shift 2 + ;; + --prefix) + PREFIX=$2 ; shift 2 + ;; + --bin-dir) + BIN_DIR=$2 ; shift 2 + ;; + --log-dir) + LOG_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --conf-dir) + CONF_DIR=$2 ; shift 2 + ;; + --pid-dir) + PID_DIR=$2 ; shift 2 + ;; + --sbin-dir) + SBIN_DIR=$2 ; shift 2 + ;; + --uninstall) + UNINSTALL=1; shift + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +ARCH=${ARCH:-i386} +HADOOP_PREFIX=$PREFIX +HADOOP_BIN_DIR=${BIN_DIR:-$PREFIX/bin} +HADOOP_CONF_DIR=${CONF_DIR:-$PREFIX/etc/hadoop} +HADOOP_LIB_DIR=${LIB_DIR:-$PREFIX/lib} +HADOOP_LOG_DIR=${LOG_DIR:-$PREFIX/var/log} +HADOOP_PID_DIR=${PID_DIR:-$PREFIX/var/run} +HADOOP_SBIN_DIR=${SBIN_DIR:-$PREFIX/sbin} +UNINSTALL=${UNINSTALL:-0} + +if [ "${ARCH}" != "i386" ]; then + HADOOP_LIB_DIR=${HADOOP_LIB_DIR}64 +fi + +if [ "${UNINSTALL}" -eq "1" ]; then + # Remove symlinks + if [ "${HADOOP_CONF_DIR}" != "${HADOOP_PREFIX}/etc/hadoop" ]; then + rm -rf ${HADOOP_PREFIX}/etc/hadoop + fi + rm -f /etc/default/hadoop-env.sh + rm -f /etc/profile.d/hadoop-env.sh +else + # Create symlinks + if [ "${HADOOP_CONF_DIR}" != "${HADOOP_PREFIX}/etc/hadoop" ]; then + mkdir -p ${HADOOP_PREFIX}/etc + ln -sf ${HADOOP_CONF_DIR} ${HADOOP_PREFIX}/etc/hadoop + fi + ln -sf ${HADOOP_CONF_DIR}/hadoop-env.sh /etc/default/hadoop-env.sh + ln -sf ${HADOOP_CONF_DIR}/hadoop-env.sh /etc/profile.d/hadoop-env.sh + + mkdir -p ${HADOOP_LOG_DIR} + chown root:hadoop ${HADOOP_LOG_DIR} + chmod 775 ${HADOOP_LOG_DIR} + + if [ ! -d ${HADOOP_PID_DIR} ]; then + mkdir -p ${HADOOP_PID_DIR} + chown root:hadoop ${HADOOP_PID_DIR} + chmod 775 ${HADOOP_PID_DIR} + fi + + TFILE="/tmp/$(basename $0).$$.tmp" + if [ -z "${JAVA_HOME}" ]; then + if [ -e /etc/debian_version ]; then + JAVA_HOME=`update-alternatives --config java | grep java | cut -f2 -d':' | cut -f2 -d' ' | sed -e 's/\/bin\/java//'` + else + JAVA_HOME=/usr/java/default + fi + fi + template_generator ${HADOOP_CONF_DIR}/hadoop-env.sh.template $TFILE + cp ${TFILE} ${CONF_DIR}/hadoop-env.sh + rm -f ${TFILE} +fi diff --git a/src/test/system/c++/runAs/configure b/src/test/system/c++/runAs/configure index 19c0a73a8ac..acd5bfa10d1 100644 --- a/src/test/system/c++/runAs/configure +++ b/src/test/system/c++/runAs/configure @@ -3349,11 +3349,11 @@ fi done -#check for HADOOP_HOME +#check for HADOOP_PREFIX if test "$with_home" != "" then cat >>confdefs.h <<_ACEOF -#define HADOOP_HOME "$with_home" +#define HADOOP_PREFIX "$with_home" _ACEOF fi diff --git a/src/test/system/c++/runAs/configure.ac b/src/test/system/c++/runAs/configure.ac index 6594c505600..ffaa4584762 100644 --- a/src/test/system/c++/runAs/configure.ac +++ b/src/test/system/c++/runAs/configure.ac @@ -40,10 +40,10 @@ AC_PROG_CC AC_HEADER_STDC AC_CHECK_HEADERS([stdlib.h string.h unistd.h fcntl.h]) -#check for HADOOP_HOME +#check for HADOOP_PREFIX if test "$with_home" != "" then -AC_DEFINE_UNQUOTED(HADOOP_HOME,"$with_home") +AC_DEFINE_UNQUOTED(HADOOP_PREFIX,"$with_home") fi # Checks for typedefs, structures, and compiler characteristics. diff --git a/src/test/system/c++/runAs/runAs.c b/src/test/system/c++/runAs/runAs.c index 359ece817f8..9b7803cbfbf 100644 --- a/src/test/system/c++/runAs/runAs.c +++ b/src/test/system/c++/runAs/runAs.c @@ -86,7 +86,7 @@ int process_cluster_command(char * user, char * node , char *command) { } len = STRLEN + strlen(command); finalcommandstr = (char *) malloc((len + 1) * sizeof(char)); - snprintf(finalcommandstr, len, SCRIPT_DIR_PATTERN, HADOOP_HOME, + snprintf(finalcommandstr, len, SCRIPT_DIR_PATTERN, HADOOP_PREFIX, command); finalcommandstr[len + 1] = '\0'; errorcode = switchuser(user); diff --git a/src/test/system/c++/runAs/runAs.h.in b/src/test/system/c++/runAs/runAs.h.in index 18bb86ac223..4cdab71e6c3 100644 --- a/src/test/system/c++/runAs/runAs.h.in +++ b/src/test/system/c++/runAs/runAs.h.in @@ -39,13 +39,13 @@ enum errorcodes { INVALID_COMMAND_PASSED, //6 }; -#undef HADOOP_HOME +#undef HADOOP_PREFIX #define SSH_COMMAND "ssh" #define SCRIPT_DIR_PATTERN "%s/bin/hadoop-daemon.sh %s" //%s to be substituded -#define STRLEN strlen(SCRIPT_DIR_PATTERN) + strlen(HADOOP_HOME) +#define STRLEN strlen(SCRIPT_DIR_PATTERN) + strlen(HADOOP_PREFIX) /* * Function to get the user details populated given a user name. diff --git a/src/test/system/java/org/apache/hadoop/test/system/process/HadoopDaemonRemoteCluster.java b/src/test/system/java/org/apache/hadoop/test/system/process/HadoopDaemonRemoteCluster.java index b6a5efc4aee..d3e5d630a37 100644 --- a/src/test/system/java/org/apache/hadoop/test/system/process/HadoopDaemonRemoteCluster.java +++ b/src/test/system/java/org/apache/hadoop/test/system/process/HadoopDaemonRemoteCluster.java @@ -59,7 +59,7 @@ public abstract class HadoopDaemonRemoteCluster public static final String CONF_HADOOPNEWCONFDIR = "test.system.hdrc.hadoopnewconfdir"; /** - * Key used to configure the HADOOP_HOME to be used by the + * Key used to configure the HADOOP_PREFIX to be used by the * HadoopDaemonRemoteCluster. */ public final static String CONF_HADOOPHOME = @@ -188,7 +188,7 @@ public abstract class HadoopDaemonRemoteCluster if (hadoopHome == null || hadoopConfDir == null || hadoopHome.isEmpty() || hadoopConfDir.isEmpty()) { LOG.error("No configuration " - + "for the HADOOP_HOME and HADOOP_CONF_DIR passed"); + + "for the HADOOP_PREFIX and HADOOP_CONF_DIR passed"); throw new IllegalArgumentException( "No Configuration passed for hadoop home " + "and hadoop conf directories");