Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-5970. Provide a boolean switch to enable MR-AM profiling. Contributed by Gera Shegalov 

(cherry picked from commit f19771a24c)
This commit is contained in:
Jason Lowe 2014-10-15 17:50:31 +00:00
parent e68fef4d81
commit 025b6c13e9
5 changed files with 58 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -55,6 +55,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6072. Remove INSTALL document (Akira AJISAKA via aw)
MAPREDUCE-5970. Provide a boolean switch to enable MR-AM profiling (Gera
Shegalov via jlowe)
OPTIMIZATIONS
BUG FIXES

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
View File

@ -1643,8 +1643,7 @@ public class JobConf extends Configuration {
*/
public String getProfileParams() {
return get(JobContext.TASK_PROFILE_PARAMS,
"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y," +
"verbose=n,file=%s");
MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS);
}
/**

11
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
View File

@ -179,6 +179,10 @@ public interface MRJobConfig {
public static final String TASK_PROFILE_PARAMS = "mapreduce.task.profile.params";
public static final String DEFAULT_TASK_PROFILE_PARAMS =
"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,"
+ "verbose=n,file=%s";
public static final String NUM_MAP_PROFILES = "mapreduce.task.profile.maps";
public static final String NUM_REDUCE_PROFILES = "mapreduce.task.profile.reduces";
@ -603,7 +607,12 @@ public interface MRJobConfig {
public static final String MR_AM_ADMIN_USER_ENV =
MR_AM_PREFIX + "admin.user.env";
public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile";
public static final boolean DEFAULT_MR_AM_PROFILE = false;
public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX
+ "profile.params";
public static final String MAPRED_MAP_ADMIN_JAVA_OPTS =
"mapreduce.admin.map.child.java.opts";

15
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
View File

@ -408,7 +408,7 @@ public class YARNRunner implements ClientProtocol {
warnForJavaLibPath(conf.get(MRJobConfig.REDUCE_JAVA_OPTS,""), "reduce",
MRJobConfig.REDUCE_JAVA_OPTS, MRJobConfig.REDUCE_ENV);
warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,""), "reduce",
MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
// Add AM admin command opts before user command opts
// so that it can be overridden by user
@ -424,7 +424,18 @@ public class YARNRunner implements ClientProtocol {
warnForJavaLibPath(mrAppMasterUserOptions, "app master",
MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
vargs.add(mrAppMasterUserOptions);
if (jobConf.getBoolean(MRJobConfig.MR_AM_PROFILE,
MRJobConfig.DEFAULT_MR_AM_PROFILE)) {
final String profileParams = jobConf.get(MRJobConfig.MR_AM_PROFILE_PARAMS,
MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS);
if (profileParams != null) {
vargs.add(String.format(profileParams,
ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR
+ TaskLog.LogName.PROFILE));
}
}
vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS);
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
Path.SEPARATOR + ApplicationConstants.STDOUT);

32
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
View File

@ -29,7 +29,6 @@ import static org.mockito.Mockito.when;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@ -115,6 +114,11 @@ public class TestYARNRunner extends TestCase {
private static final Log LOG = LogFactory.getLog(TestYARNRunner.class);
private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
// prefix before <LOG_DIR>/profile.out
private static final String PROFILE_PARAMS =
MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS.substring(0,
MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS.lastIndexOf("%"));
private YARNRunner yarnRunner;
private ResourceMgrDelegate resourceMgrDelegate;
private YarnConfiguration conf;
@ -423,6 +427,8 @@ public class TestYARNRunner extends TestCase {
for(String command : commands) {
if(command != null) {
assertFalse("Profiler should be disabled by default",
command.contains(PROFILE_PARAMS));
adminPos = command.indexOf("-Djava.net.preferIPv4Stack=true");
if(adminPos >= 0)
adminIndex = index;
@ -479,6 +485,30 @@ public class TestYARNRunner extends TestCase {
"using yarn.app.mapreduce.am.env config settings."));
}
@Test(timeout=20000)
public void testAMProfiler() throws Exception {
JobConf jobConf = new JobConf();
jobConf.setBoolean(MRJobConfig.MR_AM_PROFILE, true);
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext submissionContext =
buildSubmitContext(yarnRunner, jobConf);
ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
List<String> commands = containerSpec.getCommands();
for(String command : commands) {
if (command != null) {
if (command.contains(PROFILE_PARAMS)) {
return;
}
}
}
throw new IllegalStateException("Profiler opts not found!");
}
@Test
public void testAMStandardEnv() throws Exception {
final String ADMIN_LIB_PATH = "foo";