diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 094a8d692ef..e8af34272d5 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -43,6 +43,9 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5754. Preserve Job diagnostics in history (Gera Shegalov via
jlowe)
+ MAPREDUCE-5773. Provide dedicated MRAppMaster syslog length limit (Gera
+ Shegalov via jlowe)
+
OPTIMIZATIONS
BUG FIXES
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index de01727b4f8..dbf005e32cb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -409,9 +409,13 @@ public interface MRJobConfig {
MR_AM_PREFIX+"log.level";
public static final String DEFAULT_MR_AM_LOG_LEVEL = "INFO";
+ public static final String MR_AM_LOG_KB =
+ MR_AM_PREFIX + "container.log.limit.kb";
+ public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll
+
public static final String MR_AM_LOG_BACKUPS =
MR_AM_PREFIX + "container.log.backups";
- public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; // don't roll
+ public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0;
/**The number of splits when reporting progress in MR*/
public static final String MR_AM_NUM_PROGRESS_SPLITS =
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 77f800cbc37..b1ea034ba40 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -815,6 +815,14 @@
+
+ yarn.app.mapreduce.am.container.log.limit.kb
+ 0
+ The maximum size of the MRAppMaster attempt container logs in KB.
+ 0 disables the cap.
+
+
+
yarn.app.mapreduce.task.container.log.backups
0
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index ce475c197c1..0258f161df1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -391,7 +391,8 @@ public class YARNRunner implements ClientProtocol {
vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
// TODO: why do we use 'conf' some places and 'jobConf' others?
- long logSize = TaskLog.getTaskLogLength(new JobConf(conf));
+ long logSize = jobConf.getLong(MRJobConfig.MR_AM_LOG_KB,
+ MRJobConfig.DEFAULT_MR_AM_LOG_KB) << 10;
String logLevel = jobConf.get(
MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
int numBackups = jobConf.getInt(MRJobConfig.MR_AM_LOG_BACKUPS,
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
index 441fd98cb08..2d8972bfc92 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.io.StringReader;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
@@ -442,9 +443,12 @@ public class TestMRJobs {
final SleepJob sleepJob = new SleepJob();
final JobConf sleepConf = new JobConf(mrCluster.getConfig());
sleepConf.set(MRJobConfig.MAP_LOG_LEVEL, Level.ALL.toString());
- sleepConf.set(MRJobConfig.MR_AM_LOG_LEVEL, Level.ALL.toString());
- sleepConf.setLong(MRJobConfig.TASK_USERLOG_LIMIT, 1);
+ final long userLogKb = 4;
+ sleepConf.setLong(MRJobConfig.TASK_USERLOG_LIMIT, userLogKb);
sleepConf.setInt(MRJobConfig.TASK_LOG_BACKUPS, 3);
+ sleepConf.set(MRJobConfig.MR_AM_LOG_LEVEL, Level.ALL.toString());
+ final long amLogKb = 7;
+ sleepConf.setLong(MRJobConfig.MR_AM_LOG_KB, amLogKb);
sleepConf.setInt(MRJobConfig.MR_AM_LOG_BACKUPS, 7);
sleepJob.setConf(sleepConf);
@@ -503,6 +507,8 @@ public class TestMRJobs {
final FileStatus[] sysSiblings = localFs.globStatus(new Path(
containerPathComponent, TaskLog.LogName.SYSLOG + "*"));
+ // sort to ensure for i > 0 sysSiblings[i] == "syslog.i"
+ Arrays.sort(sysSiblings);
if (foundAppMaster) {
numAppMasters++;
@@ -510,11 +516,19 @@ public class TestMRJobs {
numMapTasks++;
}
- Assert.assertSame("Number of sylog* files",
- foundAppMaster
- ? sleepConf.getInt(MRJobConfig.MR_AM_LOG_BACKUPS, 0) + 1
- : sleepConf.getInt(MRJobConfig.TASK_LOG_BACKUPS, 0) + 1,
- sysSiblings.length);
+ if (foundAppMaster) {
+ Assert.assertSame("Unexpected number of AM sylog* files",
+ sleepConf.getInt(MRJobConfig.MR_AM_LOG_BACKUPS, 0) + 1,
+ sysSiblings.length);
+ Assert.assertTrue("AM syslog.1 length kb should be >= " + amLogKb,
+ sysSiblings[1].getLen() >= amLogKb * 1024);
+ } else {
+ Assert.assertSame("Unexpected number of MR task sylog* files",
+ sleepConf.getInt(MRJobConfig.TASK_LOG_BACKUPS, 0) + 1,
+ sysSiblings.length);
+ Assert.assertTrue("MR syslog.1 length kb should be >= " + userLogKb,
+ sysSiblings[1].getLen() >= userLogKb * 1024);
+ }
}
}
}