YARN-3626. On Windows localized resources are not moved to the front of the classpath when they should be. Contributed by Craig Welch.
(cherry picked from commit 7bc33b63c78c6ee9a04c0c1511ef16e63f7f3074) (cherry picked from commit 6ed8989a6f897437ab53c85c42fa8b58fe556359)
This commit is contained in:
parent
fb57a1aac8
commit
02ab202a2b
@ -817,10 +817,16 @@ static ContainerLaunchContext createContainerLaunchContext(
|
||||
// Fill in the fields needed per-container that are missing in the common
|
||||
// spec.
|
||||
|
||||
boolean userClassesTakesPrecedence =
|
||||
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
|
||||
|
||||
// Setup environment by cloning from common env.
|
||||
Map<String, String> env = commonContainerSpec.getEnvironment();
|
||||
Map<String, String> myEnv = new HashMap<String, String>(env.size());
|
||||
myEnv.putAll(env);
|
||||
if (userClassesTakesPrecedence) {
|
||||
myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
|
||||
}
|
||||
MapReduceChildJVM.setVMEnv(myEnv, remoteTask);
|
||||
|
||||
// Set up the launch command
|
||||
|
@ -241,11 +241,6 @@ public static void setClasspath(Map<String, String> environment,
|
||||
boolean userClassesTakesPrecedence =
|
||||
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
|
||||
|
||||
if (userClassesTakesPrecedence) {
|
||||
conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
|
||||
"true");
|
||||
}
|
||||
|
||||
String classpathEnvVar =
|
||||
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)
|
||||
? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name();
|
||||
|
@ -19,6 +19,7 @@
|
||||
package org.apache.hadoop.yarn.api;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Evolving;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
@ -190,6 +191,13 @@ public enum Environment {
|
||||
*/
|
||||
HADOOP_YARN_HOME("HADOOP_YARN_HOME"),
|
||||
|
||||
/**
|
||||
* $CLASSPATH_PREPEND_DISTCACHE
|
||||
* Private, Windows specific
|
||||
*/
|
||||
@Private
|
||||
CLASSPATH_PREPEND_DISTCACHE("CLASSPATH_PREPEND_DISTCACHE"),
|
||||
|
||||
/**
|
||||
* $CONTAINER_ID
|
||||
* Final, exported by NodeManager and non-modifiable by users.
|
||||
|
@ -611,6 +611,7 @@ private static void addDeprecatedKeys() {
|
||||
ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(),
|
||||
ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(),
|
||||
ApplicationConstants.Environment.HADOOP_CONF_DIR.key(),
|
||||
ApplicationConstants.Environment.CLASSPATH_PREPEND_DISTCACHE.key(),
|
||||
ApplicationConstants.Environment.HADOOP_YARN_HOME.key()));
|
||||
|
||||
/** address of node manager IPC.*/
|
||||
@ -1160,16 +1161,6 @@ private static void addDeprecatedKeys() {
|
||||
public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX
|
||||
+ "application.classpath";
|
||||
|
||||
/**
|
||||
* Whether or not entries from the distributed cache should be preferred over
|
||||
* the rest of the YARN CLASSPATH
|
||||
*/
|
||||
public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE =
|
||||
YARN_PREFIX + "application.classpath.prepend.distcache";
|
||||
|
||||
public static final boolean
|
||||
DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = false;
|
||||
|
||||
/**
|
||||
* Default platform-agnostic CLASSPATH for YARN applications. A
|
||||
* comma-separated list of CLASSPATH entries. The parameter expansion marker
|
||||
|
@ -735,11 +735,12 @@ public void sanitizeEnv(Map<String, String> environment, Path pwd,
|
||||
//jar is created and so they "are lost" and have to be explicitly
|
||||
//added to the classpath instead. This also means that their position
|
||||
//is lost relative to other non-distcache classpath entries which will
|
||||
//break things like mapreduce.job.user.classpath.first.
|
||||
//break things like mapreduce.job.user.classpath.first. An environment
|
||||
//variable can be set to indicate that distcache entries should come
|
||||
//first
|
||||
|
||||
boolean preferLocalizedJars = conf.getBoolean(
|
||||
YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
|
||||
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE
|
||||
boolean preferLocalizedJars = Boolean.valueOf(
|
||||
environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name())
|
||||
);
|
||||
|
||||
boolean needsSeparator = false;
|
||||
|
@ -417,7 +417,7 @@ public void testPrependDistcache() throws Exception {
|
||||
userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
|
||||
userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
|
||||
userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
|
||||
userSetEnv.put(Environment.CLASSPATH.name(), "SYSTEM_CLPATH");
|
||||
userSetEnv.put(Environment.CLASSPATH.name(), "APATH");
|
||||
containerLaunchContext.setEnvironment(userSetEnv);
|
||||
Container container = mock(Container.class);
|
||||
when(container.getContainerId()).thenReturn(cId);
|
||||
@ -463,14 +463,12 @@ public void handle(Event event) {
|
||||
Assert.assertTrue(
|
||||
result.get(result.size() - 1).endsWith("userjarlink.jar"));
|
||||
|
||||
//Now move userjar to the front
|
||||
//Then, with user classpath first
|
||||
userSetEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
|
||||
|
||||
cId = ContainerId.newContainerId(appAttemptId, 1);
|
||||
when(container.getContainerId()).thenReturn(cId);
|
||||
|
||||
conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
|
||||
"true");
|
||||
|
||||
launch = new ContainerLaunch(distContext, conf,
|
||||
dispatcher, exec, null, container, dirsHandler, containerManager);
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user