From cf3ac1a59eb199d9807f54b026feb2552edac9ff Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 9 Jan 2014 23:10:43 +0000 Subject: [PATCH] YARN-1138. Merging change r1556978 from trunk to branch-2. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1556979 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/mapreduce/v2/util/TestMRApps.java | 12 ++++++--- .../security/ssl/TestEncryptedShuffle.java | 7 +++-- hadoop-yarn-project/CHANGES.txt | 3 +++ .../src/main/resources/yarn-default.xml | 27 ++++++++++++++++--- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index 0fc236b2479..e9cecbee990 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -183,9 +183,10 @@ public class TestMRApps { MRApps.setClasspath(environment, job.getConfiguration()); assertTrue(environment.get("CLASSPATH").startsWith( ApplicationConstants.Environment.PWD.$() + File.pathSeparator)); - String yarnAppClasspath = - job.getConfiguration().get( - YarnConfiguration.YARN_APPLICATION_CLASSPATH); + String yarnAppClasspath = job.getConfiguration().get( + YarnConfiguration.YARN_APPLICATION_CLASSPATH, + StringUtils.join(",", + YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); if (yarnAppClasspath != null) { yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", File.pathSeparator) .trim(); @@ -218,7 +219,10 @@ public class TestMRApps { MRApps.setClasspath(environment, conf); assertTrue(environment.get("CLASSPATH").startsWith( ApplicationConstants.Environment.PWD.$() + File.pathSeparator)); - String confClasspath = job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH); + String confClasspath = job.getConfiguration().get( + YarnConfiguration.YARN_APPLICATION_CLASSPATH, + StringUtils.join(",", + YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); if (confClasspath != null) { confClasspath = confClasspath.replaceAll(",\\s*", File.pathSeparator) .trim(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java index 6f630aecb6b..10f871404be 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.After; import org.junit.AfterClass; @@ -86,8 +87,10 @@ public class TestEncryptedShuffle { conf.set("dfs.block.access.token.enable", "false"); conf.set("dfs.permissions", "true"); conf.set("hadoop.security.authentication", "simple"); - String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH) + - File.pathSeparator + classpathDir; + String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, + StringUtils.join(",", + YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) + + File.pathSeparator + classpathDir; conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp); dfsCluster = new MiniDFSCluster(conf, 1, true, null); FileSystem fileSystem = dfsCluster.getFileSystem(); diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 12fe6774ebc..e40fc4d3eef 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -380,6 +380,9 @@ Release 2.3.0 - UNRELEASED YARN-1438. Ensure container diagnostics includes exception from container launch. (stevel via acmurthy) + YARN-1138. yarn.application.classpath is set to point to $HADOOP_CONF_DIR + etc., which does not work on Windows. (Chuan Liu via cnauroth) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 6fe45ae9d23..ead11eb7123 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1016,10 +1016,29 @@ - CLASSPATH for YARN applications. A comma-separated list - of CLASSPATH entries - yarn.application.classpath - $HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/share/hadoop/common/*,$HADOOP_COMMON_HOME/share/hadoop/common/lib/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*,$HADOOP_YARN_HOME/share/hadoop/yarn/*,$HADOOP_YARN_HOME/share/hadoop/yarn/lib/* + + CLASSPATH for YARN applications. A comma-separated list + of CLASSPATH entries. When this value is empty, the following default + CLASSPATH for YARN applications would be used. + For Linux: + $HADOOP_CONF_DIR, + $HADOOP_COMMON_HOME/share/hadoop/common/*, + $HADOOP_COMMON_HOME/share/hadoop/common/lib/*, + $HADOOP_HDFS_HOME/share/hadoop/hdfs/*, + $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*, + $HADOOP_YARN_HOME/share/hadoop/yarn/*, + $HADOOP_YARN_HOME/share/hadoop/yarn/lib/* + For Windows: + %HADOOP_CONF_DIR%, + %HADOOP_COMMON_HOME%/share/hadoop/common/*, + %HADOOP_COMMON_HOME%/share/hadoop/common/lib/*, + %HADOOP_HDFS_HOME%/share/hadoop/hdfs/*, + %HADOOP_HDFS_HOME%/share/hadoop/hdfs/lib/*, + %HADOOP_YARN_HOME%/share/hadoop/yarn/*, + %HADOOP_YARN_HOME%/share/hadoop/yarn/lib/* + + yarn.application.classpath +