From 658905cd444bbcdac7a58e624d34463488ff754b Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Fri, 4 Oct 2013 05:42:58 +0000 Subject: [PATCH] MAPREDUCE-5442. Merging change r1529077 from trunk to branch-2. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1529079 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/mapreduce/v2/util/MRApps.java | 4 +++- .../apache/hadoop/mapreduce/v2/util/TestMRApps.java | 4 +++- .../org/apache/hadoop/mapreduce/MRJobConfig.java | 11 +++++++---- .../src/main/resources/mapred-default.xml | 13 +++++++++++-- 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 506b4c39037..ee31221d627 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -138,6 +138,9 @@ Release 2.1.2 - UNRELEASED MAPREDUCE-5489. MR jobs hangs as it does not use the node-blacklisting feature in RM requests (Zhijie Shen via bikas) + MAPREDUCE-5442. $HADOOP_MAPRED_HOME/$HADOOP_CONF_DIR setting not working on + Windows. (Yingda Chen via cnauroth) + Release 2.1.1-beta - 2013-09-23 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 977e0066bef..92fc26cae19 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -49,6 +49,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.ContainerLogAppender; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; @@ -181,7 +182,8 @@ public class MRApps extends Apps { boolean foundFrameworkInClasspath = (frameworkName == null); for (String c : conf.getStrings( MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, - MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) { + StringUtils.getStrings( + MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))){ Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c .trim()); if (!foundFrameworkInClasspath) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index ea4e1635483..0fc236b2479 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -192,7 +192,9 @@ public class TestMRApps { } assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath)); String mrAppClasspath = - job.getConfiguration().get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH); + job.getConfiguration().get( + MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, + MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH); if (mrAppClasspath != null) { mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", File.pathSeparator) .trim(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 93f7e50a132..980b808ec62 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -19,6 +19,7 @@ package org.apache.hadoop.mapreduce; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.util.Shell; @InterfaceAudience.Private @InterfaceStability.Evolving @@ -653,10 +654,12 @@ public interface MRJobConfig { /** * Default CLASSPATH for all YARN MapReduce applications. */ - public static final String[] DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH = { - "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*", - "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*", - }; + public final String + DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH = Shell.WINDOWS ? + "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\*," + + "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\lib\\*" : + "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*," + + "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*"; public static final String WORKFLOW_ID = "mapreduce.workflow.id"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 5cd2fd26ef5..9275332d58f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1460,9 +1460,18 @@ CLASSPATH for MR applications. A comma-separated list of CLASSPATH entries. If mapreduce.application.framework is set then this must specify the appropriate classpath for that archive, and the name of - the archive must be present in the classpath. + the archive must be present in the classpath. + When this value is empty, the following default CLASSPATH for MR + applications would be used. + For Linux: + $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*, + $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*. + For Windows: + %HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/*, + %HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/lib/*. + mapreduce.application.classpath - $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/* +