From e9aa5aac6805c7e40e3fd0a0e0a046fbbb4f94d2 Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Tue, 17 Jul 2012 19:32:14 +0000 Subject: [PATCH] MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't init. (Jason Lowe via daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1362618 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../logaggregation/LogAggregationService.java | 24 ++++++++++--------- .../TestLogAggregationService.java | 13 ++++++++-- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index eb487b13d0d..9daa28dd156 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -724,6 +724,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4283. Display tail of aggregated logs by default (Jason Lowe via bobby) + MAPREDUCE-4448. Fix NM crash during app cleanup if aggregation didn't + init. (Jason Lowe via daryn) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index d00c61ed50e..d29e6a0cf49 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -342,14 +342,14 @@ public class LogAggregationService extends AbstractService implements // A container is complete. Put this containers' logs up for aggregation if // this containers' logs are needed. - if (!this.appLogAggregators.containsKey( - containerId.getApplicationAttemptId().getApplicationId())) { - throw new YarnException("Application is not initialized yet for " - + containerId); + AppLogAggregator aggregator = this.appLogAggregators.get( + containerId.getApplicationAttemptId().getApplicationId()); + if (aggregator == null) { + LOG.warn("Log aggregation is not initialized for " + containerId + + ", did it fail to start?"); + return; } - this.appLogAggregators.get( - containerId.getApplicationAttemptId().getApplicationId()) - .startContainerLogAggregation(containerId, exitCode == 0); + aggregator.startContainerLogAggregation(containerId, exitCode == 0); } private void stopApp(ApplicationId appId) { @@ -357,11 +357,13 @@ public class LogAggregationService extends AbstractService implements // App is complete. Finish up any containers' pending log aggregation and // close the application specific logFile. - if (!this.appLogAggregators.containsKey(appId)) { - throw new YarnException("Application is not initialized yet for " - + appId); + AppLogAggregator aggregator = this.appLogAggregators.get(appId); + if (aggregator == null) { + LOG.warn("Log aggregation is not initialized for " + appId + + ", did it fail to start?"); + return; } - this.appLogAggregators.get(appId).finishLogAggregation(); + aggregator.finishLogAggregation(); } @Override diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 2d300310a28..16278048052 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -380,7 +380,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { @Test @SuppressWarnings("unchecked") - public void testLogAggregationInitFailsWithoutKillingNM() throws Exception { + public void testLogAggregationFailsWithoutKillingNM() throws Exception { this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, @@ -412,7 +412,16 @@ public class TestLogAggregationService extends BaseContainerManagerTest { new ApplicationFinishEvent(appId, "Application failed to init aggregation: KABOOM!") }; checkEvents(appEventHandler, expectedEvents, false, - "getType", "getApplicationID", "getDiagnostic"); + "getType", "getApplicationID", "getDiagnostic"); + + // verify trying to collect logs for containers/apps we don't know about + // doesn't blow up and tear down the NM + logAggregationService.handle(new LogHandlerContainerFinishedEvent( + BuilderUtils.newContainerId(4, 1, 1, 1), 0)); + dispatcher.await(); + logAggregationService.handle(new LogHandlerAppFinishedEvent( + BuilderUtils.newApplicationId(1, 5))); + dispatcher.await(); } private void writeContainerLogs(File appLogDir, ContainerId containerId)