diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 37e92cb2d9d..80f69406840 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -274,6 +274,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6072. Remove INSTALL document (Akira AJISAKA via aw) + MAPREDUCE-5970. Provide a boolean switch to enable MR-AM profiling (Gera + Shegalov via jlowe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java index de78e208e70..03f1160e9fe 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -1643,8 +1643,7 @@ public class JobConf extends Configuration { */ public String getProfileParams() { return get(JobContext.TASK_PROFILE_PARAMS, - "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y," + - "verbose=n,file=%s"); + MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 5b623b59df7..689e51c77b9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -184,6 +184,10 @@ public interface MRJobConfig { public static final String TASK_PROFILE_PARAMS = "mapreduce.task.profile.params"; + public static final String DEFAULT_TASK_PROFILE_PARAMS = + "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y," + + "verbose=n,file=%s"; + public static final String NUM_MAP_PROFILES = "mapreduce.task.profile.maps"; public static final String NUM_REDUCE_PROFILES = "mapreduce.task.profile.reduces"; @@ -614,7 +618,12 @@ public interface MRJobConfig { public static final String MR_AM_ADMIN_USER_ENV = MR_AM_PREFIX + "admin.user.env"; - + + public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile"; + public static final boolean DEFAULT_MR_AM_PROFILE = false; + public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX + + "profile.params"; + public static final String MAPRED_MAP_ADMIN_JAVA_OPTS = "mapreduce.admin.map.child.java.opts"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 9419d039ddc..d3b80f3830e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -408,7 +408,7 @@ public class YARNRunner implements ClientProtocol { warnForJavaLibPath(conf.get(MRJobConfig.REDUCE_JAVA_OPTS,""), "reduce", MRJobConfig.REDUCE_JAVA_OPTS, MRJobConfig.REDUCE_ENV); warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,""), "reduce", - MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV); + MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV); // Add AM admin command opts before user command opts // so that it can be overridden by user @@ -424,7 +424,18 @@ public class YARNRunner implements ClientProtocol { warnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV); vargs.add(mrAppMasterUserOptions); - + + if (jobConf.getBoolean(MRJobConfig.MR_AM_PROFILE, + MRJobConfig.DEFAULT_MR_AM_PROFILE)) { + final String profileParams = jobConf.get(MRJobConfig.MR_AM_PROFILE_PARAMS, + MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS); + if (profileParams != null) { + vargs.add(String.format(profileParams, + ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR + + TaskLog.LogName.PROFILE)); + } + } + vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR + ApplicationConstants.STDOUT); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 38a79fbb9a7..256778548fd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -29,7 +29,6 @@ import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -115,6 +114,11 @@ public class TestYARNRunner extends TestCase { private static final Log LOG = LogFactory.getLog(TestYARNRunner.class); private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + // prefix before /profile.out + private static final String PROFILE_PARAMS = + MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS.substring(0, + MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS.lastIndexOf("%")); + private YARNRunner yarnRunner; private ResourceMgrDelegate resourceMgrDelegate; private YarnConfiguration conf; @@ -423,6 +427,8 @@ public class TestYARNRunner extends TestCase { for(String command : commands) { if(command != null) { + assertFalse("Profiler should be disabled by default", + command.contains(PROFILE_PARAMS)); adminPos = command.indexOf("-Djava.net.preferIPv4Stack=true"); if(adminPos >= 0) adminIndex = index; @@ -479,6 +485,30 @@ public class TestYARNRunner extends TestCase { "using yarn.app.mapreduce.am.env config settings.")); } + @Test(timeout=20000) + public void testAMProfiler() throws Exception { + JobConf jobConf = new JobConf(); + + jobConf.setBoolean(MRJobConfig.MR_AM_PROFILE, true); + + YARNRunner yarnRunner = new YARNRunner(jobConf); + + ApplicationSubmissionContext submissionContext = + buildSubmitContext(yarnRunner, jobConf); + + ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec(); + List commands = containerSpec.getCommands(); + + for(String command : commands) { + if (command != null) { + if (command.contains(PROFILE_PARAMS)) { + return; + } + } + } + throw new IllegalStateException("Profiler opts not found!"); + } + @Test public void testAMStandardEnv() throws Exception { final String ADMIN_LIB_PATH = "foo";