From f73daf6af1c87c65dd97e5ec4608ba2742dc83ea Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Tue, 24 Jan 2012 18:21:27 +0000 Subject: [PATCH] MAPREDUCE-3505. yarn APPLICATION_CLASSPATH needs to be overridable. (ahmed via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1235391 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../v2/app/job/impl/TaskAttemptImpl.java | 6 +++--- .../hadoop-mapreduce-client-common/pom.xml | 4 ++++ .../hadoop/mapreduce/v2/util/MRApps.java | 15 +++++++++------ .../hadoop/mapreduce/v2/util/TestMRApps.java | 18 ++++++++++++++++++ .../org/apache/hadoop/mapred/YARNRunner.java | 2 +- .../hadoop/yarn/api/ApplicationConstants.java | 16 +--------------- .../hadoop/yarn/conf/YarnConfiguration.java | 4 ++++ .../src/main/resources/yarn-default.xml | 14 ++++++++++++++ 9 files changed, 57 insertions(+), 25 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 799ce495ec8..f74ebe519b5 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -540,6 +540,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3681. Fixed computation of queue's usedCapacity. (acmurthy) + MAPREDUCE-3505. yarn APPLICATION_CLASSPATH needs to be overridable. + (ahmed via tucu) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index b296d02d55f..7cca98031d5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -522,13 +522,13 @@ private static LocalResource createLocalResource(FileSystem fc, Path file, * a parent CLC and use it for all the containers, so this should go away * once the mr-generated-classpath stuff is gone. */ - private static String getInitialClasspath() throws IOException { + private static String getInitialClasspath(Configuration conf) throws IOException { synchronized (classpathLock) { if (initialClasspathFlag.get()) { return initialClasspath; } Map env = new HashMap(); - MRApps.setClasspath(env); + MRApps.setClasspath(env, conf); initialClasspath = env.get(Environment.CLASSPATH.name()); initialClasspathFlag.set(true); return initialClasspath; @@ -631,7 +631,7 @@ private static ContainerLaunchContext createCommonContainerLaunchContext( Apps.addToEnvironment( environment, Environment.CLASSPATH.name(), - getInitialClasspath()); + getInitialClasspath(conf)); } catch (IOException e) { throw new YarnException(e); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml index cb199ac70a9..e33e589c9e2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml @@ -38,6 +38,10 @@ org.apache.hadoop hadoop-mapreduce-client-core + + org.apache.hadoop + hadoop-yarn-server-common + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index cb802f1c5ad..129996e8193 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.BuilderUtils; @@ -171,7 +172,7 @@ public static TaskAttemptStateUI taskAttemptState(String attemptStateStr) { } private static void setMRFrameworkClasspath( - Map environment) throws IOException { + Map environment, Configuration conf) throws IOException { InputStream classpathFileStream = null; BufferedReader reader = null; try { @@ -208,8 +209,10 @@ private static void setMRFrameworkClasspath( } // Add standard Hadoop classes - for (String c : ApplicationConstants.APPLICATION_CLASSPATH) { - Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c); + for (String c : conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH) + .split(",")) { + Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c + .trim()); } } finally { if (classpathFileStream != null) { @@ -222,8 +225,8 @@ private static void setMRFrameworkClasspath( // TODO: Remove duplicates. } - public static void setClasspath(Map environment) - throws IOException { + public static void setClasspath(Map environment, + Configuration conf) throws IOException { Apps.addToEnvironment( environment, Environment.CLASSPATH.name(), @@ -232,7 +235,7 @@ public static void setClasspath(Map environment) environment, Environment.CLASSPATH.name(), Environment.PWD.$() + Path.SEPARATOR + "*"); - MRApps.setMRFrameworkClasspath(environment); + MRApps.setMRFrameworkClasspath(environment, conf); } private static final String STAGING_CONSTANT = ".staging"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index 11589980625..fd9d1d23281 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -18,7 +18,12 @@ package org.apache.hadoop.mapreduce.v2.util; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.v2.api.records.JobId; @@ -121,4 +126,17 @@ public class TestMRApps { "/my/path/to/staging/dummy-user/.staging/job_dummy-job_12345/job.xml", jobFile); } + @Test public void testSetClasspath() throws IOException { + Job job = Job.getInstance(); + Map environment = new HashMap(); + MRApps.setClasspath(environment, job.getConfiguration()); + assertEquals("job.jar:$PWD/*:$HADOOP_CONF_DIR:" + + "$HADOOP_COMMON_HOME/share/hadoop/common/*:" + + "$HADOOP_COMMON_HOME/share/hadoop/common/lib/*:" + + "$HADOOP_HDFS_HOME/share/hadoop/hdfs/*:" + + "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*:" + + "$YARN_HOME/share/hadoop/mapreduce/*:" + + "$YARN_HOME/share/hadoop/mapreduce/lib/*", + environment.get("CLASSPATH")); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 3513d52859a..da48e9c0249 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -406,7 +406,7 @@ public ApplicationSubmissionContext createApplicationSubmissionContext( // Setup the CLASSPATH in environment // i.e. add { job jar, CWD, Hadoop jars} to classpath. Map environment = new HashMap(); - MRApps.setClasspath(environment); + MRApps.setClasspath(environment, conf); // Parse distributed cache MRApps.setupDistributedCache(jobConf, localResources); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index 9439e21cfa0..53020472013 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -84,21 +84,7 @@ public interface ApplicationConstants { public static final String STDERR = "stderr"; public static final String STDOUT = "stdout"; - - /** - * Classpath for typical applications. - */ - public static final String[] APPLICATION_CLASSPATH = - new String[] { - "$HADOOP_CONF_DIR", - "$HADOOP_COMMON_HOME/share/hadoop/common/*", - "$HADOOP_COMMON_HOME/share/hadoop/common/lib/*", - "$HADOOP_HDFS_HOME/share/hadoop/hdfs/*", - "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*", - "$YARN_HOME/share/hadoop/mapreduce/*", - "$YARN_HOME/share/hadoop/mapreduce/lib/*" - }; - + /** * Environment for Applications. * diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index f8a4324302c..a0300797790 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -508,6 +508,10 @@ public class YarnConfiguration extends Configuration { public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS = 2000; + /** Standard Hadoop classes */ + public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX + + "application.classpath"; + public YarnConfiguration() { super(); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml index 089a23608b7..b9e5ea47e40 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml @@ -482,4 +482,18 @@ yarn.web-proxy.address + + + Classpath for typical applications. + yarn.application.classpath + + $HADOOP_CONF_DIR, + $HADOOP_COMMON_HOME/share/hadoop/common/*, + $HADOOP_COMMON_HOME/share/hadoop/common/lib/*, + $HADOOP_HDFS_HOME/share/hadoop/hdfs/*, + $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*, + $YARN_HOME/share/hadoop/mapreduce/*, + $YARN_HOME/share/hadoop/mapreduce/lib/* + +