MAPREDUCE-5773. Provide dedicated MRAppMaster syslog length limit. Contributed by Gera Shegalov
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1573775 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
c26bdddcc3
commit
a756de93ef
|
@ -185,6 +185,9 @@ Release 2.4.0 - UNRELEASED
|
|||
MAPREDUCE-5766. Moved ping messages from TaskAttempts to be at DEBUG level
|
||||
inside the ApplicationMaster log. (Jian He via vinodkv)
|
||||
|
||||
MAPREDUCE-5773. Provide dedicated MRAppMaster syslog length limit (Gera
|
||||
Shegalov via jlowe)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
@ -414,9 +414,13 @@ public interface MRJobConfig {
|
|||
MR_AM_PREFIX+"log.level";
|
||||
public static final String DEFAULT_MR_AM_LOG_LEVEL = "INFO";
|
||||
|
||||
public static final String MR_AM_LOG_KB =
|
||||
MR_AM_PREFIX + "container.log.limit.kb";
|
||||
public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll
|
||||
|
||||
public static final String MR_AM_LOG_BACKUPS =
|
||||
MR_AM_PREFIX + "container.log.backups";
|
||||
public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; // don't roll
|
||||
public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0;
|
||||
|
||||
/**The number of splits when reporting progress in MR*/
|
||||
public static final String MR_AM_NUM_PROGRESS_SPLITS =
|
||||
|
|
|
@ -510,6 +510,14 @@
|
|||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.app.mapreduce.am.container.log.limit.kb</name>
|
||||
<value>0</value>
|
||||
<description>The maximum size of the MRAppMaster attempt container logs in KB.
|
||||
0 disables the cap.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.app.mapreduce.task.container.log.backups</name>
|
||||
<value>0</value>
|
||||
|
|
|
@ -391,7 +391,8 @@ public class YARNRunner implements ClientProtocol {
|
|||
vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
|
||||
|
||||
// TODO: why do we use 'conf' some places and 'jobConf' others?
|
||||
long logSize = TaskLog.getTaskLogLength(new JobConf(conf));
|
||||
long logSize = jobConf.getLong(MRJobConfig.MR_AM_LOG_KB,
|
||||
MRJobConfig.DEFAULT_MR_AM_LOG_KB) << 10;
|
||||
String logLevel = jobConf.get(
|
||||
MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
|
||||
int numBackups = jobConf.getInt(MRJobConfig.MR_AM_LOG_BACKUPS,
|
||||
|
|
|
@ -27,6 +27,7 @@ import java.io.IOException;
|
|||
import java.io.StringReader;
|
||||
import java.net.URI;
|
||||
import java.security.PrivilegedExceptionAction;
|
||||
import java.util.Arrays;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
@ -442,9 +443,12 @@ public class TestMRJobs {
|
|||
final SleepJob sleepJob = new SleepJob();
|
||||
final JobConf sleepConf = new JobConf(mrCluster.getConfig());
|
||||
sleepConf.set(MRJobConfig.MAP_LOG_LEVEL, Level.ALL.toString());
|
||||
sleepConf.set(MRJobConfig.MR_AM_LOG_LEVEL, Level.ALL.toString());
|
||||
sleepConf.setLong(MRJobConfig.TASK_USERLOG_LIMIT, 1);
|
||||
final long userLogKb = 4;
|
||||
sleepConf.setLong(MRJobConfig.TASK_USERLOG_LIMIT, userLogKb);
|
||||
sleepConf.setInt(MRJobConfig.TASK_LOG_BACKUPS, 3);
|
||||
sleepConf.set(MRJobConfig.MR_AM_LOG_LEVEL, Level.ALL.toString());
|
||||
final long amLogKb = 7;
|
||||
sleepConf.setLong(MRJobConfig.MR_AM_LOG_KB, amLogKb);
|
||||
sleepConf.setInt(MRJobConfig.MR_AM_LOG_BACKUPS, 7);
|
||||
sleepJob.setConf(sleepConf);
|
||||
|
||||
|
@ -503,6 +507,8 @@ public class TestMRJobs {
|
|||
|
||||
final FileStatus[] sysSiblings = localFs.globStatus(new Path(
|
||||
containerPathComponent, TaskLog.LogName.SYSLOG + "*"));
|
||||
// sort to ensure for i > 0 sysSiblings[i] == "syslog.i"
|
||||
Arrays.sort(sysSiblings);
|
||||
|
||||
if (foundAppMaster) {
|
||||
numAppMasters++;
|
||||
|
@ -510,11 +516,19 @@ public class TestMRJobs {
|
|||
numMapTasks++;
|
||||
}
|
||||
|
||||
Assert.assertSame("Number of sylog* files",
|
||||
foundAppMaster
|
||||
? sleepConf.getInt(MRJobConfig.MR_AM_LOG_BACKUPS, 0) + 1
|
||||
: sleepConf.getInt(MRJobConfig.TASK_LOG_BACKUPS, 0) + 1,
|
||||
sysSiblings.length);
|
||||
if (foundAppMaster) {
|
||||
Assert.assertSame("Unexpected number of AM sylog* files",
|
||||
sleepConf.getInt(MRJobConfig.MR_AM_LOG_BACKUPS, 0) + 1,
|
||||
sysSiblings.length);
|
||||
Assert.assertTrue("AM syslog.1 length kb should be >= " + amLogKb,
|
||||
sysSiblings[1].getLen() >= amLogKb * 1024);
|
||||
} else {
|
||||
Assert.assertSame("Unexpected number of MR task sylog* files",
|
||||
sleepConf.getInt(MRJobConfig.TASK_LOG_BACKUPS, 0) + 1,
|
||||
sysSiblings.length);
|
||||
Assert.assertTrue("MR syslog.1 length kb should be >= " + userLogKb,
|
||||
sysSiblings[1].getLen() >= userLogKb * 1024);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue