Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Logging fileSize of log files under NM Local Dir. Contributed by Prabhu Joseph

This commit is contained in:
Szilard Nemeth 2019-08-02 13:38:06 +02:00
parent 1930a7bf60
commit 54ac80176e
3 changed files with 30 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1370,6 +1370,11 @@ public class YarnConfiguration extends Configuration {
public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX
+ "log-aggregation.retain-seconds";
public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1;
public static final String LOG_AGGREGATION_DEBUG_FILESIZE = YARN_PREFIX
+ "log-aggregation.debug.filesize";
public static final long DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE
= 100 * 1024 * 1024;
/**
* How long to wait between aggregated log retention checks. If set to

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1291,6 +1291,14 @@
<value>-1</value>
</property>
<property>
<description>The log files created under NM Local Directories
will be logged if it exceeds the configured bytes. This
only takes effect if log4j level is at least Debug.</description>
<name>yarn.log-aggregation.debug.filesize</name>
<value>104857600</value>
</property>
<property>
<description>Specify which log file controllers we will support. The first
file controller we add will be used to write the aggregated logs.

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
View File

@ -109,7 +109,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
private final AtomicBoolean waiting = new AtomicBoolean(false);
private int logAggregationTimes = 0;
private int cleanupOldLogTimes = 0;
private long logFileSizeThreshold;
private boolean renameTemporaryLogFileFailed = false;
private final Map<ContainerId, ContainerLogAggregator> containerLogAggregators =
@ -176,6 +176,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
this.nodeId = nodeId;
this.logAggPolicy = getLogAggPolicy(conf);
this.recoveredLogInitedTime = recoveredLogInitedTime;
this.logFileSizeThreshold =
conf.getLong(YarnConfiguration.LOG_AGGREGATION_DEBUG_FILESIZE,
YarnConfiguration.DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE);
if (logAggregationFileController == null) {
// by default, use T-File Controller
this.logAggregationFileController = new LogAggregationTFileController();
@ -330,6 +333,19 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
uploadedLogsInThisCycle = true;
List<Path> uploadedFilePathsInThisCycleList = new ArrayList<>();
uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle);
if (LOG.isDebugEnabled()) {
for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) {
try {
long fileSize = lfs.getFileStatus(uploadedFilePath).getLen();
if (fileSize >= logFileSizeThreshold) {
LOG.debug("Log File " + uploadedFilePath
+ " size is " + fileSize + " bytes");
}
} catch (Exception e1) {
LOG.error("Failed to get log file size " + e1);
}
}
}
deletionTask = new FileDeletionTask(delService,
this.userUgi.getShortUserName(), null,
uploadedFilePathsInThisCycleList);