From 339c11b406e08c12e0e2f74512b5ff7bd401f5bb Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Fri, 6 Dec 2013 01:34:42 +0000 Subject: [PATCH] YARN-546. Allow disabling the Fair Scheduler event log. (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1548362 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../fair/FairSchedulerConfiguration.java | 9 +++++ .../scheduler/fair/FairSchedulerEventLog.java | 40 ++++++++++--------- .../fair/TestFairSchedulerEventLog.java | 2 +- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b02e97587fc..a2e8d46d3d1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -125,6 +125,8 @@ Release 2.4.0 - UNRELEASED YARN-1181. Augment MiniYARNCluster to support HA mode (Karthik Kambatla) + YARN-546. Allow disabling the Fair Scheduler event log (Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index ec45cca158e..e271b053737 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -53,6 +53,11 @@ public class FairSchedulerConfiguration extends Configuration { public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file"; protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml"; + + /** Whether to enable the Fair Scheduler event log */ + public static final String EVENT_LOG_ENABLED = CONF_PREFIX + "event-log-enabled"; + public static final boolean DEFAULT_EVENT_LOG_ENABLED = false; + protected static final String EVENT_LOG_DIR = "eventlog.dir"; /** Whether pools can be created that were not specified in the FS configuration file @@ -192,6 +197,10 @@ public class FairSchedulerConfiguration extends Configuration { return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT); } + public boolean isEventLogEnabled() { + return getBoolean(EVENT_LOG_ENABLED, DEFAULT_EVENT_LOG_ENABLED); + } + public String getEventlogDir() { return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir", "/tmp/")).getAbsolutePath() + File.separator + "fairscheduler"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java index 3f6b1e8babb..05ba330c896 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java @@ -75,26 +75,30 @@ class FairSchedulerEventLog { private DailyRollingFileAppender appender; boolean init(FairSchedulerConfiguration conf) { - try { - logDir = conf.getEventlogDir(); - File logDirFile = new File(logDir); - if (!logDirFile.exists()) { - if (!logDirFile.mkdirs()) { - throw new IOException( - "Mkdirs failed to create " + logDirFile.toString()); + if (conf.isEventLogEnabled()) { + try { + logDir = conf.getEventlogDir(); + File logDirFile = new File(logDir); + if (!logDirFile.exists()) { + if (!logDirFile.mkdirs()) { + throw new IOException( + "Mkdirs failed to create " + logDirFile.toString()); + } } + String username = System.getProperty("user.name"); + logFile = String.format("%s%shadoop-%s-fairscheduler.log", + logDir, File.separator, username); + logDisabled = false; + PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n"); + appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd"); + appender.activateOptions(); + LOG.info("Initialized fair scheduler event log, logging to " + logFile); + } catch (IOException e) { + LOG.error( + "Failed to initialize fair scheduler event log. Disabling it.", e); + logDisabled = true; } - String username = System.getProperty("user.name"); - logFile = String.format("%s%shadoop-%s-fairscheduler.log", - logDir, File.separator, username); - logDisabled = false; - PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n"); - appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd"); - appender.activateOptions(); - LOG.info("Initialized fair scheduler event log, logging to " + logFile); - } catch (IOException e) { - LOG.error( - "Failed to initialize fair scheduler event log. Disabling it.", e); + } else { logDisabled = true; } return !(logDisabled); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java index db777954ca7..633bca494ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java @@ -44,7 +44,7 @@ public class TestFairSchedulerEventLog { Configuration conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, ResourceScheduler.class); - conf.set("mapred.fairscheduler.eventlog.enabled", "true"); + conf.set("yarn.scheduler.fair.event-log-enabled", "true"); // All tests assume only one assignment per node update conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");