YARN-3626. On Windows localized resources are not moved to the front of the classpath when they should be. Contributed by Craig Welch.
This commit is contained in:
parent
3164e7d838
commit
4102e5882e
|
@ -931,10 +931,16 @@ public abstract class TaskAttemptImpl implements
|
|||
// Fill in the fields needed per-container that are missing in the common
|
||||
// spec.
|
||||
|
||||
boolean userClassesTakesPrecedence =
|
||||
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
|
||||
|
||||
// Setup environment by cloning from common env.
|
||||
Map<String, String> env = commonContainerSpec.getEnvironment();
|
||||
Map<String, String> myEnv = new HashMap<String, String>(env.size());
|
||||
myEnv.putAll(env);
|
||||
if (userClassesTakesPrecedence) {
|
||||
myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
|
||||
}
|
||||
MapReduceChildJVM.setVMEnv(myEnv, remoteTask);
|
||||
|
||||
// Set up the launch command
|
||||
|
|
|
@ -241,11 +241,6 @@ public class MRApps extends Apps {
|
|||
boolean userClassesTakesPrecedence =
|
||||
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
|
||||
|
||||
if (userClassesTakesPrecedence) {
|
||||
conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
|
||||
"true");
|
||||
}
|
||||
|
||||
String classpathEnvVar =
|
||||
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)
|
||||
? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name();
|
||||
|
|
|
@ -19,6 +19,7 @@
|
|||
package org.apache.hadoop.yarn.api;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Evolving;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
|
@ -190,6 +191,13 @@ public interface ApplicationConstants {
|
|||
*/
|
||||
HADOOP_YARN_HOME("HADOOP_YARN_HOME"),
|
||||
|
||||
/**
|
||||
* $CLASSPATH_PREPEND_DISTCACHE
|
||||
* Private, Windows specific
|
||||
*/
|
||||
@Private
|
||||
CLASSPATH_PREPEND_DISTCACHE("CLASSPATH_PREPEND_DISTCACHE"),
|
||||
|
||||
/**
|
||||
* $CONTAINER_ID
|
||||
* Final, exported by NodeManager and non-modifiable by users.
|
||||
|
|
|
@ -611,6 +611,7 @@ public class YarnConfiguration extends Configuration {
|
|||
ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(),
|
||||
ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(),
|
||||
ApplicationConstants.Environment.HADOOP_CONF_DIR.key(),
|
||||
ApplicationConstants.Environment.CLASSPATH_PREPEND_DISTCACHE.key(),
|
||||
ApplicationConstants.Environment.HADOOP_YARN_HOME.key()));
|
||||
|
||||
/** address of node manager IPC.*/
|
||||
|
@ -1279,16 +1280,6 @@ public class YarnConfiguration extends Configuration {
|
|||
public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX
|
||||
+ "application.classpath";
|
||||
|
||||
/**
|
||||
* Whether or not entries from the distributed cache should be preferred over
|
||||
* the rest of the YARN CLASSPATH
|
||||
*/
|
||||
public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE =
|
||||
YARN_PREFIX + "application.classpath.prepend.distcache";
|
||||
|
||||
public static final boolean
|
||||
DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = false;
|
||||
|
||||
/**
|
||||
* Default platform-agnostic CLASSPATH for YARN applications. A
|
||||
* comma-separated list of CLASSPATH entries. The parameter expansion marker
|
||||
|
|
|
@ -749,11 +749,12 @@ public class ContainerLaunch implements Callable<Integer> {
|
|||
//jar is created and so they "are lost" and have to be explicitly
|
||||
//added to the classpath instead. This also means that their position
|
||||
//is lost relative to other non-distcache classpath entries which will
|
||||
//break things like mapreduce.job.user.classpath.first.
|
||||
//break things like mapreduce.job.user.classpath.first. An environment
|
||||
//variable can be set to indicate that distcache entries should come
|
||||
//first
|
||||
|
||||
boolean preferLocalizedJars = conf.getBoolean(
|
||||
YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
|
||||
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE
|
||||
boolean preferLocalizedJars = Boolean.valueOf(
|
||||
environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name())
|
||||
);
|
||||
|
||||
boolean needsSeparator = false;
|
||||
|
|
|
@ -417,7 +417,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
|
|||
userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
|
||||
userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
|
||||
userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
|
||||
userSetEnv.put(Environment.CLASSPATH.name(), "SYSTEM_CLPATH");
|
||||
userSetEnv.put(Environment.CLASSPATH.name(), "APATH");
|
||||
containerLaunchContext.setEnvironment(userSetEnv);
|
||||
Container container = mock(Container.class);
|
||||
when(container.getContainerId()).thenReturn(cId);
|
||||
|
@ -463,14 +463,12 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
|
|||
Assert.assertTrue(
|
||||
result.get(result.size() - 1).endsWith("userjarlink.jar"));
|
||||
|
||||
//Now move userjar to the front
|
||||
//Then, with user classpath first
|
||||
userSetEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
|
||||
|
||||
cId = ContainerId.newContainerId(appAttemptId, 1);
|
||||
when(container.getContainerId()).thenReturn(cId);
|
||||
|
||||
conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
|
||||
"true");
|
||||
|
||||
launch = new ContainerLaunch(distContext, conf,
|
||||
dispatcher, exec, null, container, dirsHandler, containerManager);
|
||||
|
||||
|
|
Loading…
Reference in New Issue