MAPREDUCE-6577. MR AM unable to load native library without MR_AM_ADMIN_USER_ENV set (sjlee)
(cherry picked from commit f6f16118d3)
This commit is contained in:
Sangjin Lee
parent
150c9a80ee
commit
2dfa0613a1
|
|
@ -12,6 +12,9 @@ Release 2.9.0 - UNRELEASED
|
|||
|
||||
BUG FIXES
|
||||
|
||||
MAPREDUCE-6577. MR AM unable to load native library without
|
||||
MR_AM_ADMIN_USER_ENV set (sjlee)
|
||||
|
||||
Release 2.8.0 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
|||
|
|
@ -731,6 +731,16 @@ public interface MRJobConfig {
|
|||
public static final String MR_AM_ADMIN_USER_ENV =
|
||||
MR_AM_PREFIX + "admin.user.env";
|
||||
|
||||
// although the AM admin user env default should be the same as the task user
|
||||
// env default, there are problems in making it work on Windows currently
|
||||
// MAPREDUCE-6588 should address the issue and set it to a proper non-empty
|
||||
// value
|
||||
public static final String DEFAULT_MR_AM_ADMIN_USER_ENV =
|
||||
Shell.WINDOWS ?
|
||||
"" :
|
||||
"LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
|
||||
"/lib/native";
|
||||
|
||||
public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile";
|
||||
public static final boolean DEFAULT_MR_AM_PROFILE = false;
|
||||
public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX
|
||||
|
|
@ -754,10 +764,13 @@ public interface MRJobConfig {
|
|||
public static final String MAPRED_ADMIN_USER_ENV =
|
||||
"mapreduce.admin.user.env";
|
||||
|
||||
public final String DEFAULT_MAPRED_ADMIN_USER_ENV =
|
||||
Shell.WINDOWS ?
|
||||
"PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin":
|
||||
"LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native";
|
||||
// the "%...%" macros can be expanded prematurely and are probably not OK
|
||||
// this should be addressed by MAPREDUCE-6588
|
||||
public static final String DEFAULT_MAPRED_ADMIN_USER_ENV =
|
||||
Shell.WINDOWS ?
|
||||
"PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin" :
|
||||
"LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
|
||||
"/lib/native";
|
||||
|
||||
public static final String WORKDIR = "work";
|
||||
|
||||
|
|
|
|||
|
|
@ -472,13 +472,14 @@ public class YARNRunner implements ClientProtocol {
|
|||
conf.get(MRJobConfig.MAPRED_ADMIN_USER_SHELL,
|
||||
MRJobConfig.DEFAULT_SHELL));
|
||||
|
||||
// Add the container working directory at the front of LD_LIBRARY_PATH
|
||||
// Add the container working directory in front of LD_LIBRARY_PATH
|
||||
MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
|
||||
MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
|
||||
|
||||
// Setup the environment variables for Admin first
|
||||
MRApps.setEnvFromInputString(environment,
|
||||
conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), conf);
|
||||
conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV,
|
||||
MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV), conf);
|
||||
// Setup the environment variables (LD_LIBRARY_PATH, etc)
|
||||
MRApps.setEnvFromInputString(environment,
|
||||
conf.get(MRJobConfig.MR_AM_ENV), conf);
|
||||
|
|
|
|||
|
|
@ -395,56 +395,56 @@ public class TestMiniMRChildTask {
|
|||
/**
|
||||
* To test OS dependent setting of default execution path for a MapRed task.
|
||||
* Mainly that we can use MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV to set -
|
||||
* for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH - for
|
||||
* Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in
|
||||
* for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH -
|
||||
* for Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in
|
||||
* LD_LIBRARY_PATH
|
||||
*/
|
||||
@Test
|
||||
public void testMapRedExecutionEnv() {
|
||||
// test if the env variable can be set
|
||||
try {
|
||||
// Application environment
|
||||
Map<String, String> environment = new HashMap<String, String>();
|
||||
String setupHadoopHomeCommand = Shell.WINDOWS ?
|
||||
"HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home" :
|
||||
"HADOOP_COMMON_HOME=/fake/path/to/hadoop/common/home";
|
||||
MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
|
||||
// for windows, test if the env variable can be set
|
||||
// this may be removed as part of MAPREDUCE-6588
|
||||
if (Shell.WINDOWS) {
|
||||
try {
|
||||
// Application environment
|
||||
Map<String, String> environment = new HashMap<String, String>();
|
||||
String setupHadoopHomeCommand =
|
||||
"HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home";
|
||||
MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
|
||||
|
||||
// Add the env variables passed by the admin
|
||||
MRApps.setEnvFromInputString(environment, conf.get(
|
||||
MRJobConfig.MAPRED_ADMIN_USER_ENV,
|
||||
MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
|
||||
|
||||
String executionPaths = environment.get(
|
||||
Shell.WINDOWS ? "PATH" : "LD_LIBRARY_PATH");
|
||||
String toFind = Shell.WINDOWS ?
|
||||
"C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin" :
|
||||
"/fake/path/to/hadoop/common/home/lib/native";
|
||||
|
||||
// Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
|
||||
assertTrue("execution path does not include the hadoop lib location "
|
||||
+ toFind, executionPaths.contains(toFind));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Exception in testing execution environment for MapReduce task");
|
||||
tearDown();
|
||||
// Add the env variables passed by the admin
|
||||
MRApps.setEnvFromInputString(environment, conf.get(
|
||||
MRJobConfig.MAPRED_ADMIN_USER_ENV,
|
||||
MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
|
||||
|
||||
String executionPaths = environment.get("PATH");
|
||||
String toFind =
|
||||
"C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin";
|
||||
|
||||
// Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
|
||||
assertTrue("execution path does not include the hadoop lib location "
|
||||
+ toFind, executionPaths.contains(toFind));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Exception in testing execution environment for MapReduce task");
|
||||
tearDown();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// now launch a mapreduce job to ensure that the child
|
||||
// also gets the configured setting for hadoop lib
|
||||
try {
|
||||
|
||||
JobConf conf = new JobConf(mr.getConfig());
|
||||
|
||||
JobConf conf = new JobConf(mr.getConfig());
|
||||
// initialize input, output directories
|
||||
Path inDir = new Path("input");
|
||||
Path outDir = new Path("output");
|
||||
String input = "The input";
|
||||
|
||||
|
||||
// set config to use the ExecutionEnvCheckMapClass map class
|
||||
configure(conf, inDir, outDir, input,
|
||||
ExecutionEnvCheckMapClass.class, IdentityReducer.class);
|
||||
launchTest(conf, inDir, outDir, input);
|
||||
|
||||
|
||||
} catch(Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Exception in testing propagation of env setting to child task");
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ import org.apache.hadoop.security.Credentials;
|
|||
import org.apache.hadoop.security.SecurityUtil;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.token.Token;
|
||||
import org.apache.hadoop.util.Shell;
|
||||
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||
import org.apache.hadoop.yarn.api.ApplicationConstants;
|
||||
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
|
||||
|
|
@ -570,16 +571,34 @@ public class TestYARNRunner extends TestCase {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testAMStandardEnv() throws Exception {
|
||||
public void testAMStandardEnvWithDefaultLibPath() throws Exception {
|
||||
testAMStandardEnv(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAMStandardEnvWithCustomLibPath() throws Exception {
|
||||
testAMStandardEnv(true);
|
||||
}
|
||||
|
||||
private void testAMStandardEnv(boolean customLibPath) throws Exception {
|
||||
// the Windows behavior is different and this test currently doesn't really
|
||||
// apply
|
||||
// MAPREDUCE-6588 should revisit this test
|
||||
if (Shell.WINDOWS) {
|
||||
return;
|
||||
}
|
||||
|
||||
final String ADMIN_LIB_PATH = "foo";
|
||||
final String USER_LIB_PATH = "bar";
|
||||
final String USER_SHELL = "shell";
|
||||
JobConf jobConf = new JobConf();
|
||||
String pathKey = Environment.LD_LIBRARY_PATH.name();
|
||||
|
||||
jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
|
||||
ADMIN_LIB_PATH);
|
||||
jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
|
||||
+ USER_LIB_PATH);
|
||||
if (customLibPath) {
|
||||
jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, pathKey + "=" +
|
||||
ADMIN_LIB_PATH);
|
||||
jobConf.set(MRJobConfig.MR_AM_ENV, pathKey + "=" + USER_LIB_PATH);
|
||||
}
|
||||
jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);
|
||||
|
||||
YARNRunner yarnRunner = new YARNRunner(jobConf);
|
||||
|
|
@ -589,15 +608,23 @@ public class TestYARNRunner extends TestCase {
|
|||
// make sure PWD is first in the lib path
|
||||
ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
|
||||
Map<String, String> env = clc.getEnvironment();
|
||||
String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
|
||||
assertNotNull("LD_LIBRARY_PATH not set", libPath);
|
||||
String libPath = env.get(pathKey);
|
||||
assertNotNull(pathKey + " not set", libPath);
|
||||
String cps = jobConf.getBoolean(
|
||||
MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
|
||||
MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
|
||||
? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
|
||||
assertEquals("Bad AM LD_LIBRARY_PATH setting",
|
||||
MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
|
||||
+ cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);
|
||||
String expectedLibPath =
|
||||
MRApps.crossPlatformifyMREnv(conf, Environment.PWD);
|
||||
if (customLibPath) {
|
||||
// append admin libpath and user libpath
|
||||
expectedLibPath += cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH;
|
||||
} else {
|
||||
expectedLibPath += cps +
|
||||
MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV.substring(
|
||||
pathKey.length() + 1);
|
||||
}
|
||||
assertEquals("Bad AM " + pathKey + " setting", expectedLibPath, libPath);
|
||||
|
||||
// make sure SHELL is set
|
||||
String shell = env.get(Environment.SHELL.name());
|
||||
|
|
|
|||
|
|
@ -1176,6 +1176,7 @@
|
|||
<forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
|
||||
<argLine>${maven-surefire-plugin.argLine}</argLine>
|
||||
<environmentVariables>
|
||||
<HADOOP_COMMON_HOME>${hadoop.common.build.dir}</HADOOP_COMMON_HOME>
|
||||
<!-- HADOOP_HOME required for tests on Windows to find winutils -->
|
||||
<HADOOP_HOME>${hadoop.common.build.dir}</HADOOP_HOME>
|
||||
<LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib</LD_LIBRARY_PATH>
|
||||
|
|
|
|||
Loading…
Reference in New Issue