svn merge -c 1435386 FIXES: MAPREDUCE-4458. Warn if java.library.path is used for AM or Task (Robert Parker via jeagles)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1435394 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Turner Eagles 2013-01-18 22:47:14 +00:00
parent f18bb1a150
commit 361e05864f
5 changed files with 116 additions and 10 deletions

View File

@ -495,6 +495,9 @@ Release 0.23.7 - UNRELEASED
BUG FIXES BUG FIXES
MAPREDUCE-4458. Warn if java.library.path is used for AM or Task
(Robert Parker via jeagles)
Release 0.23.6 - UNRELEASED Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

View File

@ -527,6 +527,9 @@ public interface MRJobConfig {
public static final String MR_AM_ENV = public static final String MR_AM_ENV =
MR_AM_PREFIX + "env"; MR_AM_PREFIX + "env";
public static final String MR_AM_ADMIN_USER_ENV =
MR_AM_PREFIX + "admin.user.env";
public static final String MAPRED_MAP_ADMIN_JAVA_OPTS = public static final String MAPRED_MAP_ADMIN_JAVA_OPTS =
"mapreduce.admin.map.child.java.opts"; "mapreduce.admin.map.child.java.opts";

View File

@ -1281,6 +1281,17 @@
</description> </description>
</property> </property>
<property>
<name>yarn.app.mapreduce.am.admin.user.env</name>
<value></value>
<description> Environment variables for the MR App Master
processes for admin purposes. These values are set first and can be
overridden by the user env (yarn.app.mapreduce.am.env) Example :
1) A=foo This will set the env variable A to foo
2) B=$B:c This is inherit app master's B env variable.
</description>
</property>
<property> <property>
<name>yarn.app.mapreduce.am.command-opts</name> <name>yarn.app.mapreduce.am.command-opts</name>
<value>-Xmx1024m</value> <value>-Xmx1024m</value>

View File

@ -101,7 +101,7 @@ public class YARNRunner implements ClientProtocol {
private Configuration conf; private Configuration conf;
private final FileContext defaultFileContext; private final FileContext defaultFileContext;
/* usually is false unless the jobclient getdelegation token is /* usually is false unless the jobclient get delegation token is
* called. This is a hack wherein we do return a token from RM * called. This is a hack wherein we do return a token from RM
* on getDelegationtoken but due to the restricted api on jobclient * on getDelegationtoken but due to the restricted api on jobclient
* we just add a job history DT token when submitting a job. * we just add a job history DT token when submitting a job.
@ -394,13 +394,30 @@ public class YARNRunner implements ClientProtocol {
MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL); MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
MRApps.addLog4jSystemProperties(logLevel, logSize, vargs); MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
// Check for Java Lib Path usage in MAP and REDUCE configs
warnForJavaLibPath(conf.get(MRJobConfig.MAP_JAVA_OPTS,""), "map",
MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.MAP_ENV);
warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,""), "map",
MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
warnForJavaLibPath(conf.get(MRJobConfig.REDUCE_JAVA_OPTS,""), "reduce",
MRJobConfig.REDUCE_JAVA_OPTS, MRJobConfig.REDUCE_ENV);
warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,""), "reduce",
MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
// Add AM admin command opts before user command opts // Add AM admin command opts before user command opts
// so that it can be overridden by user // so that it can be overridden by user
vargs.add(conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS)); MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS);
warnForJavaLibPath(mrAppMasterAdminOptions, "app master",
MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, MRJobConfig.MR_AM_ADMIN_USER_ENV);
vargs.add(mrAppMasterAdminOptions);
vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS, // Add AM user command opts
MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS)); String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS);
warnForJavaLibPath(mrAppMasterUserOptions, "app master",
MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
vargs.add(mrAppMasterUserOptions);
vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS); vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS);
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
@ -425,6 +442,9 @@ public class YARNRunner implements ClientProtocol {
Map<String, String> environment = new HashMap<String, String>(); Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, conf); MRApps.setClasspath(environment, conf);
// Setup the environment variables for Admin first
MRApps.setEnvFromInputString(environment,
conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV));
// Setup the environment variables (LD_LIBRARY_PATH, etc) // Setup the environment variables (LD_LIBRARY_PATH, etc)
MRApps.setEnvFromInputString(environment, MRApps.setEnvFromInputString(environment,
conf.get(MRJobConfig.MR_AM_ENV)); conf.get(MRJobConfig.MR_AM_ENV));
@ -582,4 +602,15 @@ public class YARNRunner implements ClientProtocol {
throws IOException { throws IOException {
return clientCache.getClient(jobID).getLogFilePath(jobID, taskAttemptID); return clientCache.getClient(jobID).getLogFilePath(jobID, taskAttemptID);
} }
private static void warnForJavaLibPath(String opts, String component,
String javaConf, String envConf) {
if (opts != null && opts.contains("-Djava.library.path")) {
LOG.warn("Usage of -Djava.library.path in " + javaConf + " can cause " +
"programs to no longer function if hadoop native libraries " +
"are used. These values should be set as part of the " +
"LD_LIBRARY_PATH in the " + component + " JVM env using " +
envConf + " config settings.");
}
}
} }

View File

@ -25,6 +25,7 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
@ -83,6 +84,11 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.log4j.Appender;
import org.apache.log4j.Layout;
import org.apache.log4j.Logger;
import org.apache.log4j.SimpleLayout;
import org.apache.log4j.WriterAppender;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.mockito.invocation.InvocationOnMock; import org.mockito.invocation.InvocationOnMock;
@ -189,7 +195,7 @@ public class TestYARNRunner extends TestCase {
@Test @Test
public void testResourceMgrDelegate() throws Exception { public void testResourceMgrDelegate() throws Exception {
/* we not want a mock of resourcemgr deleagte */ /* we not want a mock of resource mgr delegate */
final ClientRMProtocol clientRMProtocol = mock(ClientRMProtocol.class); final ClientRMProtocol clientRMProtocol = mock(ClientRMProtocol.class);
ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf) { ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf) {
@Override @Override
@ -256,6 +262,9 @@ public class TestYARNRunner extends TestCase {
@Test @Test
public void testHistoryServerToken() throws Exception { public void testHistoryServerToken() throws Exception {
//Set the master principal in the config
conf.set(YarnConfiguration.RM_PRINCIPAL,"foo@LOCAL");
final String masterPrincipal = Master.getMasterPrincipal(conf); final String masterPrincipal = Master.getMasterPrincipal(conf);
final MRClientProtocol hsProxy = mock(MRClientProtocol.class); final MRClientProtocol hsProxy = mock(MRClientProtocol.class);
@ -265,7 +274,7 @@ public class TestYARNRunner extends TestCase {
GetDelegationTokenRequest request = GetDelegationTokenRequest request =
(GetDelegationTokenRequest)invocation.getArguments()[0]; (GetDelegationTokenRequest)invocation.getArguments()[0];
// check that the renewer matches the cluster's RM principal // check that the renewer matches the cluster's RM principal
assertEquals(request.getRenewer(), masterPrincipal); assertEquals(masterPrincipal, request.getRenewer() );
DelegationToken token = DelegationToken token =
recordFactory.newRecordInstance(DelegationToken.class); recordFactory.newRecordInstance(DelegationToken.class);
@ -357,4 +366,53 @@ public class TestYARNRunner extends TestCase {
assertTrue("AM admin command opts is after user command opts.", adminIndex < userIndex); assertTrue("AM admin command opts is after user command opts.", adminIndex < userIndex);
} }
} }
@Test
public void testWarnCommandOpts() throws Exception {
Logger logger = Logger.getLogger(YARNRunner.class);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
Layout layout = new SimpleLayout();
Appender appender = new WriterAppender(layout, bout);
logger.addAppender(appender);
JobConf jobConf = new JobConf();
jobConf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, "-Djava.net.preferIPv4Stack=true -Djava.library.path=foo");
jobConf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx1024m -Djava.library.path=bar");
YARNRunner yarnRunner = new YARNRunner(jobConf);
File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE);
OutputStream out = new FileOutputStream(jobxml);
conf.writeXml(out);
out.close();
File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT);
out = new FileOutputStream(jobsplit);
out.close();
File jobsplitmetainfo = new File(testWorkDir, MRJobConfig.JOB_SPLIT_METAINFO);
out = new FileOutputStream(jobsplitmetainfo);
out.close();
File appTokens = new File(testWorkDir, MRJobConfig.APPLICATION_TOKENS_FILE);
out = new FileOutputStream(appTokens);
out.close();
@SuppressWarnings("unused")
ApplicationSubmissionContext submissionContext =
yarnRunner.createApplicationSubmissionContext(jobConf, testWorkDir.toString(), new Credentials());
String logMsg = bout.toString();
assertTrue(logMsg.contains("WARN - Usage of -Djava.library.path in " +
"yarn.app.mapreduce.am.admin-command-opts can cause programs to no " +
"longer function if hadoop native libraries are used. These values " +
"should be set as part of the LD_LIBRARY_PATH in the app master JVM " +
"env using yarn.app.mapreduce.am.admin.user.env config settings."));
assertTrue(logMsg.contains("WARN - Usage of -Djava.library.path in " +
"yarn.app.mapreduce.am.command-opts can cause programs to no longer " +
"function if hadoop native libraries are used. These values should " +
"be set as part of the LD_LIBRARY_PATH in the app master JVM env " +
"using yarn.app.mapreduce.am.env config settings."));
}
} }