diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh index ac69037a371..6b1dd2e71fc 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh @@ -20,6 +20,16 @@ # Resolve links ($0 may be a softlink) and convert a relative path # to an absolute path. NB: The -P option requires bash built-ins # or POSIX:2001 compliant cd and pwd. + +# HADOOP_CLASSPATH Extra Java CLASSPATH entries. +# +# HADOOP_USER_CLASSPATH_FIRST When defined, the HADOOP_CLASSPATH is +# added in the beginning of the global +# classpath. Can be defined, for example, +# by doing +# export HADOOP_USER_CLASSPATH_FIRST=true +# + this="${BASH_SOURCE-$0}" common_bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P) script="$(basename -- "$this")" @@ -153,6 +163,10 @@ fi # CLASSPATH initially contains $HADOOP_CONF_DIR CLASSPATH="${HADOOP_CONF_DIR}" +if [ "$HADOOP_USER_CLASSPATH_FIRST" != "" ] && [ "$HADOOP_CLASSPATH" != "" ] ; then + CLASSPATH=${CLASSPATH}:${HADOOP_CLASSPATH} +fi + # so that filenames w/ spaces are handled correctly in loops below IFS= @@ -174,7 +188,7 @@ fi CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/$HADOOP_COMMON_DIR'/*' # add user-specified CLASSPATH last -if [ "$HADOOP_CLASSPATH" != "" ]; then +if [ "$HADOOP_USER_CLASSPATH_FIRST" = "" ] && [ "$HADOOP_CLASSPATH" != "" ]; then CLASSPATH=${CLASSPATH}:${HADOOP_CLASSPATH} fi diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index bd5d7211e70..4bea421c408 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -597,6 +597,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3499. New MiniMR does not setup proxyuser configuration correctly, thus tests using doAs do not work. (johnvijoe via tucu) + MAPREDUCE-3696. MR job via oozie does not work on hadoop 23. + (John George via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 129996e8193..019f1de0308 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -227,15 +227,23 @@ public class MRApps extends Apps { public static void setClasspath(Map environment, Configuration conf) throws IOException { + boolean userClassesTakesPrecedence = + conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); + + if (!userClassesTakesPrecedence) { + MRApps.setMRFrameworkClasspath(environment, conf); + } Apps.addToEnvironment( - environment, - Environment.CLASSPATH.name(), + environment, + Environment.CLASSPATH.name(), MRJobConfig.JOB_JAR); Apps.addToEnvironment( - environment, + environment, Environment.CLASSPATH.name(), Environment.PWD.$() + Path.SEPARATOR + "*"); - MRApps.setMRFrameworkClasspath(environment, conf); + if (userClassesTakesPrecedence) { + MRApps.setMRFrameworkClasspath(environment, conf); + } } private static final String STAGING_CONSTANT = ".staging"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index fd9d1d23281..77299a05c79 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -130,13 +130,43 @@ public class TestMRApps { Job job = Job.getInstance(); Map environment = new HashMap(); MRApps.setClasspath(environment, job.getConfiguration()); - assertEquals("job.jar:$PWD/*:$HADOOP_CONF_DIR:" + + assertEquals("$HADOOP_CONF_DIR:" + "$HADOOP_COMMON_HOME/share/hadoop/common/*:" + "$HADOOP_COMMON_HOME/share/hadoop/common/lib/*:" + "$HADOOP_HDFS_HOME/share/hadoop/hdfs/*:" + "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*:" + "$YARN_HOME/share/hadoop/mapreduce/*:" + - "$YARN_HOME/share/hadoop/mapreduce/lib/*", + "$YARN_HOME/share/hadoop/mapreduce/lib/*:" + + "job.jar:$PWD/*", environment.get("CLASSPATH")); } + + @Test public void testSetClasspathWithUserPrecendence() { + Configuration conf = new Configuration(); + conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true); + Map env = new HashMap(); + try { + MRApps.setClasspath(env, conf); + } catch (Exception e) { + fail("Got exception while setting classpath"); + } + String env_str = env.get("CLASSPATH"); + assertSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!", + env_str.indexOf("job.jar"), 0); + } + + @Test public void testSetClasspathWithNoUserPrecendence() { + Configuration conf = new Configuration(); + conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); + Map env = new HashMap(); + try { + MRApps.setClasspath(env, conf); + } catch (Exception e) { + fail("Got exception while setting classpath"); + } + String env_str = env.get("CLASSPATH"); + assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!", + env_str.indexOf("job.jar"), 0); + } + } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index fd5f87bd40c..c60fe25c2de 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -118,6 +118,8 @@ public interface MRJobConfig { public static final String USER_LOG_RETAIN_HOURS = "mapreduce.job.userlog.retain.hours"; + public static final String MAPREDUCE_JOB_USER_CLASSPATH_FIRST = "mapreduce.job.user.classpath.first"; + public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor"; public static final String IO_SORT_MB = "mapreduce.task.io.sort.mb"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 3f02092d242..3b00ddf83c5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -408,7 +408,7 @@ public class YARNRunner implements ClientProtocol { + mergedCommand); // Setup the CLASSPATH in environment - // i.e. add { job jar, CWD, Hadoop jars} to classpath. + // i.e. add { Hadoop jars, job jar, CWD } to classpath. Map environment = new HashMap(); MRApps.setClasspath(environment, conf);