From 867cd2f5a2e5966d6a7c5f5accb2fce78f9e7778 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Sun, 22 May 2016 16:02:57 -0700 Subject: [PATCH] YARN-5112. Excessive log warnings for directory permission issue on NM recovery. Contributed by Jian He. --- .../logaggregation/AppLogAggregatorImpl.java | 45 +----------------- .../logaggregation/LogAggregationService.java | 47 +++++++++++++++++-- 2 files changed, 45 insertions(+), 47 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 32b0934807c..ac43ecf5a61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -95,11 +95,6 @@ public class AppLogAggregatorImpl implements AppLogAggregator { // NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS. private static final String NM_LOG_AGGREGATION_DEBUG_ENABLED = YarnConfiguration.NM_PREFIX + "log-aggregation.debug-enabled"; - private static final boolean - DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED = false; - - private static final long - NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS = 3600; private final LocalDirsHandlerService dirsHandler; private final Dispatcher dispatcher; @@ -142,7 +137,7 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, LocalDirsHandlerService dirsHandler, Path remoteNodeLogFileForApp, Map appAcls, LogAggregationContext logAggregationContext, Context context, - FileContext lfs) { + FileContext lfs, long rollingMonitorInterval) { this.dispatcher = dispatcher; this.conf = conf; this.delService = deletionService; @@ -167,43 +162,7 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, } else { this.retentionSize = configuredRentionSize; } - long configuredRollingMonitorInterval = conf.getLong( - YarnConfiguration - .NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, - YarnConfiguration - .DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS); - boolean debug_mode = - conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED, - DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED); - if (configuredRollingMonitorInterval > 0 - && configuredRollingMonitorInterval < - NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS) { - if (debug_mode) { - this.rollingMonitorInterval = configuredRollingMonitorInterval; - } else { - LOG.warn( - "rollingMonitorIntervall should be more than or equal to " - + NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS - + " seconds. Using " - + NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS - + " seconds instead."); - this.rollingMonitorInterval = - NM_LOG_AGGREGATION_MIN_ROLL_MONITORING_INTERVAL_SECONDS; - } - } else { - if (configuredRollingMonitorInterval <= 0) { - LOG.info("rollingMonitorInterval is set as " - + configuredRollingMonitorInterval + ". " - + "The log rolling monitoring interval is disabled. " - + "The logs will be aggregated after this application is finished."); - } else { - LOG.info("rollingMonitorInterval is set as " - + configuredRollingMonitorInterval + ". " - + "The logs will be aggregated every " - + configuredRollingMonitorInterval + " seconds"); - } - this.rollingMonitorInterval = configuredRollingMonitorInterval; - } + this.rollingMonitorInterval = rollingMonitorInterval; this.logAggregationInRolling = this.rollingMonitorInterval <= 0 || this.logAggregationContext == null || this.logAggregationContext.getRolledLogsIncludePattern() == null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index 189e7ff9dc4..afc75c1af70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -69,8 +69,14 @@ public class LogAggregationService extends AbstractService implements LogHandler { - private static final Log LOG = LogFactory - .getLog(LogAggregationService.class); + private static final Log LOG = LogFactory.getLog(LogAggregationService.class); + private static final long MIN_LOG_ROLLING_INTERVAL = 3600; + // This configuration is for debug and test purpose. By setting + // this configuration as true. We can break the lower bound of + // NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS. + private static final String NM_LOG_AGGREGATION_DEBUG_ENABLED + = YarnConfiguration.NM_PREFIX + "log-aggregation.debug-enabled"; + private long rollingMonitorInterval; /* * Expected deployment TLD will be 1777, owner=, group= appLogAggregators; + private boolean logPermError = true; @VisibleForTesting ExecutorService threadPool; @@ -128,6 +135,35 @@ protected void serviceInit(Configuration conf) throws Exception { new ThreadFactoryBuilder() .setNameFormat("LogAggregationService #%d") .build()); + + rollingMonitorInterval = conf.getLong( + YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, + YarnConfiguration.DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS); + + boolean logAggregationDebugMode = + conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED, false); + + if (rollingMonitorInterval > 0 + && rollingMonitorInterval < MIN_LOG_ROLLING_INTERVAL) { + if (logAggregationDebugMode) { + LOG.info("Log aggregation debug mode enabled. rollingMonitorInterval = " + + rollingMonitorInterval); + } else { + LOG.warn("rollingMonitorIntervall should be more than or equal to " + + MIN_LOG_ROLLING_INTERVAL + " seconds. Using " + + MIN_LOG_ROLLING_INTERVAL + " seconds instead."); + this.rollingMonitorInterval = MIN_LOG_ROLLING_INTERVAL; + } + } else if (rollingMonitorInterval <= 0) { + LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval + + ". The log rolling monitoring interval is disabled. " + + "The logs will be aggregated after this application is finished."); + } else { + LOG.info("rollingMonitorInterval is set as " + rollingMonitorInterval + + ". The logs will be aggregated every " + rollingMonitorInterval + + " seconds"); + } + super.serviceInit(conf); } @@ -197,11 +233,14 @@ void verifyAndCreateRemoteLogDir(Configuration conf) { try { FsPermission perms = remoteFS.getFileStatus(this.remoteRootLogDir).getPermission(); - if (!perms.equals(TLDIR_PERMISSIONS)) { + if (!perms.equals(TLDIR_PERMISSIONS) && logPermError) { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir + "] already exist, but with incorrect permissions. " + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms + "]." + " The cluster may have problems with multiple users."); + logPermError = false; + } else { + logPermError = true; } } catch (FileNotFoundException e) { remoteExists = false; @@ -360,7 +399,7 @@ protected void initAppAggregator(final ApplicationId appId, String user, getConfig(), appId, userUgi, this.nodeId, dirsHandler, getRemoteNodeLogFileForApp(appId, user), appAcls, logAggregationContext, this.context, - getLocalFileContext(getConfig())); + getLocalFileContext(getConfig()), this.rollingMonitorInterval); if (this.appLogAggregators.putIfAbsent(appId, appLogAggregator) != null) { throw new YarnRuntimeException("Duplicate initApp for " + appId); }