MAPREDUCE-6577. MR AM unable to load native library without MR_AM_ADMIN_USER_ENV set (sjlee)

hadoop-mapreduce-project/CHANGES.txt

@@ -315,6 +315,9 @@ Release 2.9.0 - UNRELEASED
 
   BUG FIXES
 
+    MAPREDUCE-6577. MR AM unable to load native library without
+    MR_AM_ADMIN_USER_ENV set (sjlee)
+
 Release 2.8.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

@@ -740,6 +740,16 @@ public interface MRJobConfig {
   public static final String MR_AM_ADMIN_USER_ENV =
       MR_AM_PREFIX + "admin.user.env";
 
+  // although the AM admin user env default should be the same as the task user
+  // env default, there are problems in making it work on Windows currently
+  // MAPREDUCE-6588 should address the issue and set it to a proper non-empty
+  // value
+  public static final String DEFAULT_MR_AM_ADMIN_USER_ENV =
+      Shell.WINDOWS ?
+          "" :
+          "LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
+              "/lib/native";
+
   public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile";
   public static final boolean DEFAULT_MR_AM_PROFILE = false;
   public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX
@@ -763,10 +773,13 @@ public interface MRJobConfig {
   public static final String MAPRED_ADMIN_USER_ENV =
       "mapreduce.admin.user.env";
 
-  public final String DEFAULT_MAPRED_ADMIN_USER_ENV = 
+  // the "%...%" macros can be expanded prematurely and are probably not OK
+  // this should be addressed by MAPREDUCE-6588
+  public static final String DEFAULT_MAPRED_ADMIN_USER_ENV =
       Shell.WINDOWS ?
-          "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin":
+          "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin" :
-          "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native";
+          "LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
+              "/lib/native";
 
   public static final String WORKDIR = "work";
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java

@@ -472,13 +472,14 @@ public class YARNRunner implements ClientProtocol {
         conf.get(MRJobConfig.MAPRED_ADMIN_USER_SHELL,
             MRJobConfig.DEFAULT_SHELL));
 
-    // Add the container working directory at the front of LD_LIBRARY_PATH
+    // Add the container working directory in front of LD_LIBRARY_PATH
     MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
         MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
 
     // Setup the environment variables for Admin first
     MRApps.setEnvFromInputString(environment, 
-        conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), conf);
+        conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV,
+            MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV), conf);
     // Setup the environment variables (LD_LIBRARY_PATH, etc)
     MRApps.setEnvFromInputString(environment, 
         conf.get(MRJobConfig.MR_AM_ENV), conf);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java

@@ -395,39 +395,39 @@ public class TestMiniMRChildTask {
   /**
    * To test OS dependent setting of default execution path for a MapRed task.
    * Mainly that we can use MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV to set -
-   * for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH - for
+   * for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH -
-   * Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in
+   * for Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in
    * LD_LIBRARY_PATH
    */
   @Test
   public void testMapRedExecutionEnv() {
-    // test if the env variable can be set
+    // for windows, test if the env variable can be set
-    try {
+    // this may be removed as part of MAPREDUCE-6588
-      // Application environment
+    if (Shell.WINDOWS) {
-      Map<String, String> environment = new HashMap<String, String>();
+      try {
-      String setupHadoopHomeCommand = Shell.WINDOWS ? 
+        // Application environment
-          "HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home" :
+        Map<String, String> environment = new HashMap<String, String>();
-          "HADOOP_COMMON_HOME=/fake/path/to/hadoop/common/home";
+        String setupHadoopHomeCommand =
-      MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
+          "HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home";
+        MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
 
-      // Add the env variables passed by the admin
+        // Add the env variables passed by the admin
-      MRApps.setEnvFromInputString(environment, conf.get(
+        MRApps.setEnvFromInputString(environment, conf.get(
-          MRJobConfig.MAPRED_ADMIN_USER_ENV,
+            MRJobConfig.MAPRED_ADMIN_USER_ENV,
-          MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
+            MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
 
-      String executionPaths = environment.get(
+        String executionPaths = environment.get("PATH");
-          Shell.WINDOWS ? "PATH" : "LD_LIBRARY_PATH");
+        String toFind =
-      String toFind = Shell.WINDOWS ? 
+            "C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin";
-          "C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin" : 
-          "/fake/path/to/hadoop/common/home/lib/native";
 
-      // Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
+        // Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
-      assertTrue("execution path does not include the hadoop lib location "
+        assertTrue("execution path does not include the hadoop lib location "
-          + toFind, executionPaths.contains(toFind));
+            + toFind, executionPaths.contains(toFind));
-    } catch (Exception e) {
+      } catch (Exception e) {
-      e.printStackTrace();
+        e.printStackTrace();
-      fail("Exception in testing execution environment for MapReduce task");
+        fail("Exception in testing execution environment for MapReduce task");
-      tearDown();
+        tearDown();
+      }
     }
 
     // now launch a mapreduce job to ensure that the child 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java

@@ -63,6 +63,7 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -570,16 +571,34 @@ public class TestYARNRunner extends TestCase {
   }
 
   @Test
-  public void testAMStandardEnv() throws Exception {
+  public void testAMStandardEnvWithDefaultLibPath() throws Exception {
+    testAMStandardEnv(false);
+  }
+
+  @Test
+  public void testAMStandardEnvWithCustomLibPath() throws Exception {
+    testAMStandardEnv(true);
+  }
+
+  private void testAMStandardEnv(boolean customLibPath) throws Exception {
+    // the Windows behavior is different and this test currently doesn't really
+    // apply
+    // MAPREDUCE-6588 should revisit this test
+    if (Shell.WINDOWS) {
+      return;
+    }
+
     final String ADMIN_LIB_PATH = "foo";
     final String USER_LIB_PATH = "bar";
     final String USER_SHELL = "shell";
     JobConf jobConf = new JobConf();
+    String pathKey = Environment.LD_LIBRARY_PATH.name();
 
-    jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
+    if (customLibPath) {
-        ADMIN_LIB_PATH);
+      jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, pathKey + "=" +
-    jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
+          ADMIN_LIB_PATH);
-        + USER_LIB_PATH);
+      jobConf.set(MRJobConfig.MR_AM_ENV, pathKey + "=" + USER_LIB_PATH);
+    }
     jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);
 
     YARNRunner yarnRunner = new YARNRunner(jobConf);
@@ -589,15 +608,23 @@ public class TestYARNRunner extends TestCase {
     // make sure PWD is first in the lib path
     ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
     Map<String, String> env = clc.getEnvironment();
-    String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
+    String libPath = env.get(pathKey);
-    assertNotNull("LD_LIBRARY_PATH not set", libPath);
+    assertNotNull(pathKey + " not set", libPath);
     String cps = jobConf.getBoolean(
         MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
         MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
         ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
-    assertEquals("Bad AM LD_LIBRARY_PATH setting",
+    String expectedLibPath =
-        MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
+        MRApps.crossPlatformifyMREnv(conf, Environment.PWD);
-        + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);
+    if (customLibPath) {
+      // append admin libpath and user libpath
+      expectedLibPath += cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH;
+    } else {
+      expectedLibPath += cps +
+          MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV.substring(
+              pathKey.length() + 1);
+    }
+    assertEquals("Bad AM " + pathKey + " setting", expectedLibPath, libPath);
 
     // make sure SHELL is set
     String shell = env.get(Environment.SHELL.name());

hadoop-project/pom.xml

@@ -1133,6 +1133,7 @@
           <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
           <argLine>${maven-surefire-plugin.argLine}</argLine>
           <environmentVariables>
+            <HADOOP_COMMON_HOME>${hadoop.common.build.dir}</HADOOP_COMMON_HOME>
             <!-- HADOOP_HOME required for tests on Windows to find winutils -->
             <HADOOP_HOME>${hadoop.common.build.dir}</HADOOP_HOME>
             <LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib</LD_LIBRARY_PATH>