Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-546. Allow disabling the Fair Scheduler event log. (Sandy Ryza)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1548362 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sanford Ryza 2013-12-06 01:34:42 +00:00
parent 6855f10a33
commit 339c11b406
4 changed files with 34 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -125,6 +125,8 @@ Release 2.4.0 - UNRELEASED
YARN-1181. Augment MiniYARNCluster to support HA mode (Karthik Kambatla) YARN-1181. Augment MiniYARNCluster to support HA mode (Karthik Kambatla)
YARN-546. Allow disabling the Fair Scheduler event log (Sandy Ryza)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
View File

@ -53,6 +53,11 @@ public class FairSchedulerConfiguration extends Configuration {
public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file"; public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml"; protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
/** Whether to enable the Fair Scheduler event log */
public static final String EVENT_LOG_ENABLED = CONF_PREFIX + "event-log-enabled";
public static final boolean DEFAULT_EVENT_LOG_ENABLED = false;
protected static final String EVENT_LOG_DIR = "eventlog.dir"; protected static final String EVENT_LOG_DIR = "eventlog.dir";
/** Whether pools can be created that were not specified in the FS configuration file /** Whether pools can be created that were not specified in the FS configuration file
@ -192,6 +197,10 @@ public boolean getSizeBasedWeight() {
return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT); return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT);
} }
public boolean isEventLogEnabled() {
return getBoolean(EVENT_LOG_ENABLED, DEFAULT_EVENT_LOG_ENABLED);
}
public String getEventlogDir() { public String getEventlogDir() {
return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir", return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir",
"/tmp/")).getAbsolutePath() + File.separator + "fairscheduler"); "/tmp/")).getAbsolutePath() + File.separator + "fairscheduler");

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
View File

@ -75,26 +75,30 @@ class FairSchedulerEventLog {
private DailyRollingFileAppender appender; private DailyRollingFileAppender appender;
boolean init(FairSchedulerConfiguration conf) { boolean init(FairSchedulerConfiguration conf) {
try { if (conf.isEventLogEnabled()) {
logDir = conf.getEventlogDir(); try {
File logDirFile = new File(logDir); logDir = conf.getEventlogDir();
if (!logDirFile.exists()) { File logDirFile = new File(logDir);
if (!logDirFile.mkdirs()) { if (!logDirFile.exists()) {
throw new IOException( if (!logDirFile.mkdirs()) {
"Mkdirs failed to create " + logDirFile.toString()); throw new IOException(
"Mkdirs failed to create " + logDirFile.toString());
}
} }
String username = System.getProperty("user.name");
logFile = String.format("%s%shadoop-%s-fairscheduler.log",
logDir, File.separator, username);
logDisabled = false;
PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n");
appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd");
appender.activateOptions();
LOG.info("Initialized fair scheduler event log, logging to " + logFile);
} catch (IOException e) {
LOG.error(
"Failed to initialize fair scheduler event log. Disabling it.", e);
logDisabled = true;
} }
String username = System.getProperty("user.name"); } else {
logFile = String.format("%s%shadoop-%s-fairscheduler.log",
logDir, File.separator, username);
logDisabled = false;
PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n");
appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd");
appender.activateOptions();
LOG.info("Initialized fair scheduler event log, logging to " + logFile);
} catch (IOException e) {
LOG.error(
"Failed to initialize fair scheduler event log. Disabling it.", e);
logDisabled = true; logDisabled = true;
} }
return !(logDisabled); return !(logDisabled);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
View File

@ -44,7 +44,7 @@ public void setUp() throws IOException {
Configuration conf = new YarnConfiguration(); Configuration conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
ResourceScheduler.class); ResourceScheduler.class);
conf.set("mapred.fairscheduler.eventlog.enabled", "true"); conf.set("yarn.scheduler.fair.event-log-enabled", "true");
// All tests assume only one assignment per node update // All tests assume only one assignment per node update
conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");