YARN-3626. On Windows localized resources are not moved to the front of the classpath when they should be. Contributed by Craig Welch.

This commit is contained in:
cnauroth 2015-05-27 14:21:05 -07:00
parent 3164e7d838
commit 4102e5882e
6 changed files with 23 additions and 24 deletions

View File

@ -931,10 +931,16 @@ public abstract class TaskAttemptImpl implements
// Fill in the fields needed per-container that are missing in the common
// spec.
boolean userClassesTakesPrecedence =
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
// Setup environment by cloning from common env.
Map<String, String> env = commonContainerSpec.getEnvironment();
Map<String, String> myEnv = new HashMap<String, String>(env.size());
myEnv.putAll(env);
if (userClassesTakesPrecedence) {
myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
}
MapReduceChildJVM.setVMEnv(myEnv, remoteTask);
// Set up the launch command

View File

@ -241,11 +241,6 @@ public class MRApps extends Apps {
boolean userClassesTakesPrecedence =
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
if (userClassesTakesPrecedence) {
conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
"true");
}
String classpathEnvVar =
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)
? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name();

View File

@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.api;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
@ -190,6 +191,13 @@ public interface ApplicationConstants {
*/
HADOOP_YARN_HOME("HADOOP_YARN_HOME"),
/**
* $CLASSPATH_PREPEND_DISTCACHE
* Private, Windows specific
*/
@Private
CLASSPATH_PREPEND_DISTCACHE("CLASSPATH_PREPEND_DISTCACHE"),
/**
* $CONTAINER_ID
* Final, exported by NodeManager and non-modifiable by users.

View File

@ -611,6 +611,7 @@ public class YarnConfiguration extends Configuration {
ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(),
ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(),
ApplicationConstants.Environment.HADOOP_CONF_DIR.key(),
ApplicationConstants.Environment.CLASSPATH_PREPEND_DISTCACHE.key(),
ApplicationConstants.Environment.HADOOP_YARN_HOME.key()));
/** address of node manager IPC.*/
@ -1279,16 +1280,6 @@ public class YarnConfiguration extends Configuration {
public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX
+ "application.classpath";
/**
* Whether or not entries from the distributed cache should be preferred over
* the rest of the YARN CLASSPATH
*/
public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE =
YARN_PREFIX + "application.classpath.prepend.distcache";
public static final boolean
DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = false;
/**
* Default platform-agnostic CLASSPATH for YARN applications. A
* comma-separated list of CLASSPATH entries. The parameter expansion marker

View File

@ -749,11 +749,12 @@ public class ContainerLaunch implements Callable<Integer> {
//jar is created and so they "are lost" and have to be explicitly
//added to the classpath instead. This also means that their position
//is lost relative to other non-distcache classpath entries which will
//break things like mapreduce.job.user.classpath.first.
//break things like mapreduce.job.user.classpath.first. An environment
//variable can be set to indicate that distcache entries should come
//first
boolean preferLocalizedJars = conf.getBoolean(
YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE
boolean preferLocalizedJars = Boolean.valueOf(
environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name())
);
boolean needsSeparator = false;

View File

@ -417,7 +417,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
userSetEnv.put(Environment.CLASSPATH.name(), "SYSTEM_CLPATH");
userSetEnv.put(Environment.CLASSPATH.name(), "APATH");
containerLaunchContext.setEnvironment(userSetEnv);
Container container = mock(Container.class);
when(container.getContainerId()).thenReturn(cId);
@ -463,14 +463,12 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Assert.assertTrue(
result.get(result.size() - 1).endsWith("userjarlink.jar"));
//Now move userjar to the front
//Then, with user classpath first
userSetEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
cId = ContainerId.newContainerId(appAttemptId, 1);
when(container.getContainerId()).thenReturn(cId);
conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
"true");
launch = new ContainerLaunch(distContext, conf,
dispatcher, exec, null, container, dirsHandler, containerManager);