MAPREDUCE-4638. MR AM supplies MapReduce jars in classpath rather than rely on YARN. Contributed by Arun C Murthy.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1381868 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
e8c193bf24
commit
0056914773
|
@ -249,6 +249,9 @@ Release 2.1.0-alpha - Unreleased
|
|||
MAPREDUCE-4579. Split TestTaskAttempt into two so as to pass tests on
|
||||
jdk7. (Thomas Graves via vinodkv)
|
||||
|
||||
MAPREDUCE-4638. MR AM supplies MapReduce jars in classpath rather than
|
||||
rely on YARN. (acmurthy)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
@ -179,6 +179,12 @@ public class MRApps extends Apps {
|
|||
Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
|
||||
.trim());
|
||||
}
|
||||
for (String c : conf.getStrings(
|
||||
MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
|
||||
MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) {
|
||||
Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c
|
||||
.trim());
|
||||
}
|
||||
} finally {
|
||||
if (classpathFileStream != null) {
|
||||
classpathFileStream.close();
|
||||
|
|
|
@ -140,11 +140,19 @@ public class TestMRApps {
|
|||
Map<String, String> environment = new HashMap<String, String>();
|
||||
MRApps.setClasspath(environment, job.getConfiguration());
|
||||
assertTrue(environment.get("CLASSPATH").startsWith("$PWD:"));
|
||||
String confClasspath = job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
|
||||
if (confClasspath != null) {
|
||||
confClasspath = confClasspath.replaceAll(",\\s*", ":").trim();
|
||||
String yarnAppClasspath =
|
||||
job.getConfiguration().get(
|
||||
YarnConfiguration.YARN_APPLICATION_CLASSPATH);
|
||||
if (yarnAppClasspath != null) {
|
||||
yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", ":").trim();
|
||||
}
|
||||
assertTrue(environment.get("CLASSPATH").contains(confClasspath));
|
||||
assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
|
||||
String mrAppClasspath =
|
||||
job.getConfiguration().get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH);
|
||||
if (mrAppClasspath != null) {
|
||||
mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", ":").trim();
|
||||
}
|
||||
assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
|
||||
}
|
||||
|
||||
@Test public void testSetClasspathWithUserPrecendence() {
|
||||
|
|
|
@ -587,4 +587,18 @@ public interface MRJobConfig {
|
|||
MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT =
|
||||
"security.job.client.protocol.acl";
|
||||
|
||||
/**
|
||||
* CLASSPATH for all YARN MapReduce applications.
|
||||
*/
|
||||
public static final String MAPREDUCE_APPLICATION_CLASSPATH =
|
||||
"mapreduce.application.classpath";
|
||||
|
||||
/**
|
||||
* Default CLASSPATH for all YARN MapReduce applications.
|
||||
*/
|
||||
public static final String[] DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH = {
|
||||
"$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*",
|
||||
"$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*",
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
@ -1203,6 +1203,8 @@
|
|||
|
||||
<!-- end of node health script variables -->
|
||||
|
||||
<!-- MR YARN Application properties -->
|
||||
|
||||
<property>
|
||||
<name>mapreduce.job.counters.limit</name>
|
||||
<value>120</value>
|
||||
|
@ -1312,6 +1314,13 @@
|
|||
<description>The amount of memory the MR AppMaster needs.</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>CLASSPATH for MR applications. A comma-separated list
|
||||
of CLASSPATH entries</description>
|
||||
<name>mapreduce.application.classpath</name>
|
||||
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
|
||||
</property>
|
||||
|
||||
<!-- jobhistory properties -->
|
||||
|
||||
<property>
|
||||
|
|
Loading…
Reference in New Issue