YARN-3626. On Windows localized resources are not moved to the front of the classpath when they should be. Contributed by Craig Welch.

(cherry picked from commit 7bc33b63c78c6ee9a04c0c1511ef16e63f7f3074)
(cherry picked from commit 6ed8989a6f)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java

@@ -817,10 +817,16 @@ public abstract class TaskAttemptImpl implements
     // Fill in the fields needed per-container that are missing in the common
     // spec.
 
+    boolean userClassesTakesPrecedence =
+      conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
+
     // Setup environment by cloning from common env.
     Map<String, String> env = commonContainerSpec.getEnvironment();
     Map<String, String> myEnv = new HashMap<String, String>(env.size());
     myEnv.putAll(env);
+    if (userClassesTakesPrecedence) {
+      myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
+    }
     MapReduceChildJVM.setVMEnv(myEnv, remoteTask);
 
     // Set up the launch command

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java

@@ -241,11 +241,6 @@ public class MRApps extends Apps {
     boolean userClassesTakesPrecedence = 
       conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
 
-    if (userClassesTakesPrecedence) {
-      conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
-        "true");
-    }
-
     String classpathEnvVar =
       conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)
         ? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.api;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -190,6 +191,13 @@ public interface ApplicationConstants {
      */
     HADOOP_YARN_HOME("HADOOP_YARN_HOME"),
 
+    /**
+     * $CLASSPATH_PREPEND_DISTCACHE
+     * Private, Windows specific
+     */
+    @Private
+    CLASSPATH_PREPEND_DISTCACHE("CLASSPATH_PREPEND_DISTCACHE"),
+
     /**
      * $CONTAINER_ID
      * Final, exported by NodeManager and non-modifiable by users.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -611,6 +611,7 @@ public class YarnConfiguration extends Configuration {
       ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(),
       ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(),
       ApplicationConstants.Environment.HADOOP_CONF_DIR.key(),
+      ApplicationConstants.Environment.CLASSPATH_PREPEND_DISTCACHE.key(),
       ApplicationConstants.Environment.HADOOP_YARN_HOME.key()));
   
   /** address of node manager IPC.*/
@@ -1160,16 +1161,6 @@ public class YarnConfiguration extends Configuration {
   public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX
       + "application.classpath";
 
-  /**
-   * Whether or not entries from the distributed cache should be preferred over
-   * the rest of the YARN CLASSPATH
-   */
-  public static final String YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE =
-    YARN_PREFIX + "application.classpath.prepend.distcache";
-
-  public static final boolean
-    DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE = false;
-
   /**
    * Default platform-agnostic CLASSPATH for YARN applications. A
    * comma-separated list of CLASSPATH entries. The parameter expansion marker

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java

@@ -735,11 +735,12 @@ public class ContainerLaunch implements Callable<Integer> {
         //jar is created and so they "are lost" and have to be explicitly
         //added to the classpath instead.  This also means that their position
         //is lost relative to other non-distcache classpath entries which will
-        //break things like mapreduce.job.user.classpath.first.
+        //break things like mapreduce.job.user.classpath.first.  An environment
+        //variable can be set to indicate that distcache entries should come
+        //first
 
-        boolean preferLocalizedJars = conf.getBoolean(
-          YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
-          YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE
+        boolean preferLocalizedJars = Boolean.valueOf(
+          environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name())
           );
 
         boolean needsSeparator = false;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java

@@ -417,7 +417,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
     userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
     userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
     userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
-    userSetEnv.put(Environment.CLASSPATH.name(), "SYSTEM_CLPATH");
+    userSetEnv.put(Environment.CLASSPATH.name(), "APATH");
     containerLaunchContext.setEnvironment(userSetEnv);
     Container container = mock(Container.class);
     when(container.getContainerId()).thenReturn(cId);
@@ -463,14 +463,12 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
     Assert.assertTrue(
       result.get(result.size() - 1).endsWith("userjarlink.jar"));
 
-    //Now move userjar to the front
+    //Then, with user classpath first
+    userSetEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
 
     cId = ContainerId.newContainerId(appAttemptId, 1);
     when(container.getContainerId()).thenReturn(cId);
 
-    conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH_PREPEND_DISTCACHE,
-      "true");
-
     launch = new ContainerLaunch(distContext, conf,
         dispatcher, exec, null, container, dirsHandler, containerManager);