diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml index 4868590ac46..57f3c66dad3 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml @@ -22,7 +22,6 @@ dir false - hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/native/target/usr/local/bin @@ -33,7 +32,7 @@ hadoop-yarn/bin bin - * + yarn 0755 @@ -41,17 +40,81 @@ bin bin - * + mapred + + 0755 + + + bin + libexec + + mapred-config.sh + + 0755 + + + hadoop-yarn/bin + libexec + + yarn-config.sh + + 0755 + + + hadoop-yarn/bin + sbin + + yarn-daemon.sh + yarn-daemons.sh + start-yarn.sh + stop-yarn.sh 0755 hadoop-yarn/conf - conf + etc/hadoop **/* + + ${basedir} + /share/doc/hadoop/${hadoop.component} + + *.txt + + + + ${project.build.directory}/webapps + /share/hadoop/${hadoop.component}/webapps + + + ${basedir}/src/main/conf + /share/hadoop/${hadoop.component}/templates + + *-site.xml + + + + ${basedir}/src/main/packages/templates/conf + /share/hadoop/${hadoop.component}/templates/conf + + * + + + + ${basedir}/dev-support/jdiff + /share/hadoop/${hadoop.component}/jdiff + + + ${project.build.directory}/site/jdiff/xml + /share/hadoop/${hadoop.component}/jdiff + + + ${project.build.directory}/site + /share/doc/hadoop/${hadoop.component} + @@ -59,7 +122,7 @@ org.apache.hadoop:hadoop-yarn-server-tests - modules + share/hadoop/${hadoop.component} false false @@ -68,7 +131,7 @@ false - /lib + /share/hadoop/${hadoop.component}/lib org.apache.hadoop:hadoop-common diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh index e53ec737f5a..71c94817146 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh @@ -231,6 +231,23 @@ fi CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs'/*' +# put yarn in classpath if present +if [ "$YARN_HOME" = "" ]; then + if [ -d "${HADOOP_PREFIX}/share/hadoop/mapreduce" ]; then + YARN_HOME=$HADOOP_PREFIX + fi +fi + +if [ -d "$YARN_HOME/share/hadoop/mapreduce/webapps" ]; then + CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce +fi + +if [ -d "$YARN_HOME/share/hadoop/mapreduce/lib" ]; then + CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce/lib'/*' +fi + +CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce'/*' + # cygwin path translation if $cygwin; then HADOOP_HDFS_HOME=`cygpath -w "$HADOOP_HDFS_HOME"` diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index ed6b729a933..93fe32be252 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -76,6 +76,9 @@ dist false + + tar|rpm|deb + @@ -114,15 +117,6 @@ run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* . run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/* . run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* . - COMMON_LIB=share/hadoop/common/lib - MODULES=../../../../modules - run ln -s $MODULES/hadoop-mapreduce-client-app-${project.version}.jar $COMMON_LIB - run ln -s $MODULES/hadoop-yarn-api-${project.version}.jar $COMMON_LIB - run ln -s $MODULES/hadoop-mapreduce-client-common-${project.version}.jar $COMMON_LIB - run ln -s $MODULES/hadoop-yarn-common-${project.version}.jar $COMMON_LIB - run ln -s $MODULES/hadoop-mapreduce-client-core-${project.version}.jar $COMMON_LIB - run ln -s $MODULES/hadoop-yarn-server-common-${project.version}.jar $COMMON_LIB - run ln -s $MODULES/hadoop-mapreduce-client-jobclient-${project.version}.jar $COMMON_LIB echo echo "Hadoop dist layout available at: ${project.build.directory}/hadoop-${project.version}" echo diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a851daf472a..95a98ff344f 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -323,6 +323,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3422. Counter display names are not being picked up. (Jonathan Eagles via sseth) + MAPREDUCE-3366. Mapreduce component should use consistent directory structure + layout as HDFS/common (Eric Yang via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/INSTALL b/hadoop-mapreduce-project/INSTALL index e6de8cb92e4..e75b2aff2f9 100644 --- a/hadoop-mapreduce-project/INSTALL +++ b/hadoop-mapreduce-project/INSTALL @@ -55,11 +55,11 @@ Step 8) Modify mapred-site.xml to use yarn framework Step 9) cd $YARN_HOME -Step 10) bin/yarn-daemon.sh start resourcemanager +Step 10) sbin/yarn-daemon.sh start resourcemanager -Step 11) bin/yarn-daemon.sh start nodemanager +Step 11) sbin/yarn-daemon.sh start nodemanager -Step 12) bin/yarn-daemon.sh start historyserver +Step 12) sbin/yarn-daemon.sh start historyserver Step 13) You are all set, an example on how to run a mapreduce job is: cd $HADOOP_MAPRED_HOME diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/slaves.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/slaves.sh index ee834779015..ee254603d6d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/bin/slaves.sh +++ b/hadoop-mapreduce-project/hadoop-yarn/bin/slaves.sh @@ -38,7 +38,7 @@ fi bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -DEFAULT_LIBEXEC_DIR="$bin" +DEFAULT_LIBEXEC_DIR="$bin"/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} . $HADOOP_LIBEXEC_DIR/yarn-config.sh diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/start-yarn.sh similarity index 97% rename from hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh rename to hadoop-mapreduce-project/hadoop-yarn/bin/start-yarn.sh index fa4fcf3d0d3..ccd63a44789 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh +++ b/hadoop-mapreduce-project/hadoop-yarn/bin/start-yarn.sh @@ -23,7 +23,7 @@ echo "starting yarn daemons" bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -DEFAULT_LIBEXEC_DIR="$bin" +DEFAULT_LIBEXEC_DIR="$bin"/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} . $HADOOP_LIBEXEC_DIR/yarn-config.sh diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/stop-yarn.sh similarity index 97% rename from hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh rename to hadoop-mapreduce-project/hadoop-yarn/bin/stop-yarn.sh index 546b67f5c96..c10d1ce7d18 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh +++ b/hadoop-mapreduce-project/hadoop-yarn/bin/stop-yarn.sh @@ -23,7 +23,7 @@ echo "stopping yarn daemons" bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -DEFAULT_LIBEXEC_DIR="$bin" +DEFAULT_LIBEXEC_DIR="$bin"/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} . $HADOOP_LIBEXEC_DIR/yarn-config.sh diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn index b8e23a97f5e..f5c8c1f8e84 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn +++ b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn @@ -44,7 +44,7 @@ bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -DEFAULT_LIBEXEC_DIR="$bin" +DEFAULT_LIBEXEC_DIR="$bin"/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} . $HADOOP_LIBEXEC_DIR/yarn-config.sh @@ -109,8 +109,7 @@ if [ ! -d "$HADOOP_CONF_DIR" ]; then exit 1 fi -CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}" -CLASSPATH=${CLASSPATH}:${YARN_CLASSPATH} +CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}:${CLASSPATH}" CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar # for developers, add Hadoop classes to CLASSPATH @@ -146,38 +145,6 @@ fi # so that filenames w/ spaces are handled correctly in loops below IFS= -# add hadoop-common libs to CLASSPATH -if [ ! -d "$HADOOP_COMMON_HOME" ]; then - if [ -d "$HADOOP_PREFIX" ]; then - export HADOOP_COMMON_HOME=$HADOOP_PREFIX - else - echo No HADOOP_COMMON_HOME set. - echo Please specify it either in yarn-env.sh or in the environment. - exit 1 - fi -fi - -CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/share/hadoop/common'/*' -CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/share/hadoop/common/lib'/*' - -# add hadoop-hdfs libs to CLASSPATH -if [ ! -d "$HADOOP_HDFS_HOME" ]; then - if [ -d "$HADOOP_PREFIX" ]; then - export HADOOP_HDFS_HOME=$HADOOP_PREFIX - else - echo No HADOOP_HDFS_HOME set. - echo Please specify it either in yarn-env.sh or in the environment. - exit 1 - fi -fi -CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs'/*' -CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib'/*' - -# add yarn libs to CLASSPATH - -CLASSPATH=${CLASSPATH}:$YARN_HOME/modules'/*' -CLASSPATH=${CLASSPATH}:$YARN_HOME/lib'/*' - # default log directory & file if [ "$YARN_LOG_DIR" = "" ]; then YARN_LOG_DIR="$YARN_HOME/logs" diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh index 4371484b866..27570442731 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh +++ b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh @@ -15,29 +15,24 @@ # included in all the hadoop scripts with source command # should not be executable directly -# also should not be passed any arguments, since we need original $* - -# resolve links - $0 may be a softlink - -this="$0" -while [ -h "$this" ]; do - ls=`ls -ld "$this"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '.*/.*' > /dev/null; then - this="$link" - else - this=`dirname "$this"`/"$link" - fi -done - -# convert relative path to absolute path -bin=`dirname "$this"` -script=`basename "$this"` +bin=`which "$0"` +bin=`dirname "${bin}"` bin=`cd "$bin"; pwd` -this="$bin/$script" -# the root of the Hadoop installation -export YARN_HOME=`dirname "$this"`/.. +export HADOOP_PREFIX="${HADOOP_PREFIX:-$bin/..}" + +DEFAULT_LIBEXEC_DIR="$bin"/../libexec +HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} +if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then + . ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh +elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then + . "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh +elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then + . "$HADOOP_HOME"/libexec/hadoop-config.sh +else + echo "Hadoop common not found." + exit +fi # Same glibc bug that discovered in Hadoop. # Without this you can see very large vmem settings on containers. @@ -56,7 +51,7 @@ then fi # Allow alternate conf dir location. -YARN_CONF_DIR="${YARN_CONF_DIR:-$YARN_HOME/conf}" +YARN_CONF_DIR="${HADOOP_CONF_DIR:-$YARN_HOME/conf}" #check to see it is specified whether to use the slaves or the # masters file diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh index 99fcb0a5501..6e41f791c3e 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh +++ b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh @@ -39,7 +39,7 @@ fi bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -DEFAULT_LIBEXEC_DIR="$bin" +DEFAULT_LIBEXEC_DIR="$bin"/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} . $HADOOP_LIBEXEC_DIR/yarn-config.sh diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemons.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemons.sh index e34e4ca8b1e..aafb42b9b17 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemons.sh +++ b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemons.sh @@ -30,7 +30,7 @@ fi bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` -DEFAULT_LIBEXEC_DIR="$bin" +DEFAULT_LIBEXEC_DIR="$bin"/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} . $HADOOP_LIBEXEC_DIR/yarn-config.sh diff --git a/hadoop-mapreduce-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-mapreduce-project/hadoop-yarn/conf/yarn-env.sh index b219eddf1a3..cfcb250b8e3 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/conf/yarn-env.sh +++ b/hadoop-mapreduce-project/hadoop-yarn/conf/yarn-env.sh @@ -41,10 +41,6 @@ if [ "$YARN_HEAPSIZE" != "" ]; then #echo $JAVA_HEAP_MAX fi -# CLASSPATH initially contains $YARN_CONF_DIR -CLASSPATH="${YARN_CONF_DIR}" -CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar - # so that filenames w/ spaces are handled correctly in loops below IFS= diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index c4d0d78ea5d..9439e21cfa0 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -95,8 +95,8 @@ public interface ApplicationConstants { "$HADOOP_COMMON_HOME/share/hadoop/common/lib/*", "$HADOOP_HDFS_HOME/share/hadoop/hdfs/*", "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*", - "$YARN_HOME/modules/*", - "$YARN_HOME/lib/*" + "$YARN_HOME/share/hadoop/mapreduce/*", + "$YARN_HOME/share/hadoop/mapreduce/lib/*" }; /** diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm index 3d34351708e..f4ea1fe69c7 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm @@ -171,20 +171,6 @@ Add the following configs to your <<>> +---+ -* Create Symlinks. - - You will have to create the following symlinks: - -+---+ -$ cd $HADOOP_COMMON_HOME/share/hadoop/common/lib/ -$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-app-*-SNAPSHOT.jar . -$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-jobclient-*-SNAPSHOT.jar . -$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-common-*-SNAPSHOT.jar . -$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-shuffle-*-SNAPSHOT.jar . -$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-core-*-SNAPSHOT.jar . -$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-yarn-common-*-SNAPSHOT.jar . -$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-yarn-api-*-SNAPSHOT.jar . -+---+ * Running daemons. Assuming that the environment variables <<$HADOOP_COMMON_HOME>>, <<$HADOOP_HDFS_HOME>>, <<$HADOO_MAPRED_HOME>>, @@ -195,8 +181,8 @@ $ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-yarn-api-*-SNAPSHOT.jar . +---+ $ cd $HADOOP_MAPRED_HOME -$ bin/yarn-daemon.sh start resourcemanager -$ bin/yarn-daemon.sh start nodemanager +$ sbin/yarn-daemon.sh start resourcemanager +$ sbin/yarn-daemon.sh start nodemanager +---+ You should be up and running. You can run randomwriter as: diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml index 74970dd5ee7..b9e64473cf6 100644 --- a/hadoop-mapreduce-project/pom.xml +++ b/hadoop-mapreduce-project/pom.xml @@ -34,6 +34,8 @@ 600000 once ${basedir} + mapreduce + true @@ -321,7 +323,10 @@ - release + dist + + false + @@ -336,16 +341,6 @@ - - - - - dist - - false - - - org.apache.maven.plugins maven-assembly-plugin @@ -367,7 +362,7 @@ - dist + package-mapreduce prepare-package single