Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3476. Nodemanager can fail to delete local logs if log aggregation fails. Contributed by Rohith 

(cherry picked from commit 25e2b02122)
This commit is contained in:
Jason Lowe 2015-05-08 22:45:52 +00:00
parent f57b1bfbdd
commit a75f4bed6e
2 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -71,6 +71,9 @@ Release 2.7.1 - UNRELEASED
YARN-3554. Default value for maximum nodemanager connect wait time is too
high (Naganarasimha G R via jlowe)
YARN-3476. Nodemanager can fail to delete local logs if log aggregation
fails (Rohith via jlowe)
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
View File

@ -385,6 +385,11 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
public void run() {
try {
doAppLogAggregation();
} catch (Exception e) {
// do post clean up of log directories on any exception
LOG.error("Error occured while aggregating the log for the application "
+ appId, e);
doAppLogAggregationPostCleanUp();
} finally {
if (!this.appAggregationFinished.get()) {
LOG.warn("Aggregation did not complete for application " + appId);
@ -422,6 +427,15 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
// App is finished, upload the container logs.
uploadLogsForContainers(true);
doAppLogAggregationPostCleanUp();
this.dispatcher.getEventHandler().handle(
new ApplicationEvent(this.appId,
ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED));
this.appAggregationFinished.set(true);
}
private void doAppLogAggregationPostCleanUp() {
// Remove the local app-log-dirs
List<Path> localAppLogDirs = new ArrayList<Path>();
for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
@ -442,11 +456,6 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
this.delService.delete(this.userUgi.getShortUserName(), null,
localAppLogDirs.toArray(new Path[localAppLogDirs.size()]));
}
this.dispatcher.getEventHandler().handle(
new ApplicationEvent(this.appId,
ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED));
this.appAggregationFinished.set(true);
}
private Path getRemoteNodeTmpLogFileForApp() {