From 50bd067e1d63d4c80dc1e7bf4024bfaf42cf4416 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Wed, 16 Dec 2015 10:57:39 -0800 Subject: [PATCH] YARN-4452. NPE when submit Unmanaged application. Contributed by Naganarasimha G R. --- hadoop-yarn-project/CHANGES.txt | 9 +++++ .../metrics/SystemMetricsPublisher.java | 22 +++++++----- .../metrics/TestSystemMetricsPublisher.java | 34 +++++++++++++++---- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a8375ecfc00..9b2397fdd95 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1138,6 +1138,9 @@ Release 2.8.0 - UNRELEASED YARN-4440. FSAppAttempt#getAllowedLocalityLevelByTime should init the lastScheduler time. (Lin Yiqun via zxu) + YARN-4452. NPE when submit Unmanaged application. (Naganarasimha G R + via junping_du) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES @@ -1186,6 +1189,9 @@ Release 2.7.3 - UNRELEASED YARN-4439. Clarify NMContainerStatus#toString method. (Jian He via xgong) + YARN-4452. NPE when submit Unmanaged application. (Naganarasimha G R via + junping_du) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES @@ -2040,6 +2046,9 @@ Release 2.6.4 - UNRELEASED YARN-3535. Scheduler must re-request container resources when RMContainer transitions from ALLOCATED to KILLED (rohithsharma and peng.zhang via asuresh) + YARN-4452. NPE when submit Unmanaged application. (Naganarasimha G R + via junping_du) + Release 2.6.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index 44cb8075315..f240660da46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -160,6 +160,8 @@ public class SystemMetricsPublisher extends CompositeService { public void appAttemptRegistered(RMAppAttempt appAttempt, long registeredTime) { if (publishSystemMetrics) { + ContainerId container = (appAttempt.getMasterContainer() == null) ? null + : appAttempt.getMasterContainer().getId(); dispatcher.getEventHandler().handle( new AppAttemptRegisteredEvent( appAttempt.getAppAttemptId(), @@ -167,7 +169,7 @@ public class SystemMetricsPublisher extends CompositeService { appAttempt.getRpcPort(), appAttempt.getTrackingUrl(), appAttempt.getOriginalTrackingUrl(), - appAttempt.getMasterContainer().getId(), + container, registeredTime)); } } @@ -176,6 +178,8 @@ public class SystemMetricsPublisher extends CompositeService { public void appAttemptFinished(RMAppAttempt appAttempt, RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) { if (publishSystemMetrics) { + ContainerId container = (appAttempt.getMasterContainer() == null) ? null + : appAttempt.getMasterContainer().getId(); dispatcher.getEventHandler().handle( new AppAttemptFinishedEvent( appAttempt.getAppAttemptId(), @@ -187,7 +191,7 @@ public class SystemMetricsPublisher extends CompositeService { app.getFinalApplicationStatus(), RMServerUtils.createApplicationAttemptState(appAttemtpState), finishedTime, - appAttempt.getMasterContainer().getId())); + container)); } } @@ -390,9 +394,10 @@ public class SystemMetricsPublisher extends CompositeService { event.getHost()); eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, event.getRpcPort()); - eventInfo.put( - AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, - event.getMasterContainerId().toString()); + if (event.getMasterContainerId() != null) { + eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + event.getMasterContainerId().toString()); + } tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); @@ -417,9 +422,10 @@ public class SystemMetricsPublisher extends CompositeService { event.getFinalApplicationStatus().toString()); eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, event.getYarnApplicationAttemptState().toString()); - eventInfo.put( - AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, - event.getMasterContainerId().toString()); + if (event.getMasterContainerId() != null) { + eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + event.getMasterContainerId().toString()); + } tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 4835ade7d12..f2b02bcfad7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -256,11 +256,31 @@ public class TestSystemMetricsPublisher { } } + @Test(timeout = 10000) + public void testPublishAppAttemptMetricsForUnmanagedAM() throws Exception { + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); + RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId,true); + metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L); + RMApp app = mock(RMApp.class); + when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED); + metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app, + Integer.MAX_VALUE + 2L); + TimelineEntity entity = null; + do { + entity = + store.getEntity(appAttemptId.toString(), + AppAttemptMetricsConstants.ENTITY_TYPE, + EnumSet.allOf(Field.class)); + // ensure two events are both published before leaving the loop + } while (entity == null || entity.getEvents().size() < 2); + } + @Test(timeout = 10000) public void testPublishAppAttemptMetrics() throws Exception { ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); - RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId); + RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId, false); metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L); RMApp app = mock(RMApp.class); when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED); @@ -435,15 +455,17 @@ public class TestSystemMetricsPublisher { } private static RMAppAttempt createRMAppAttempt( - ApplicationAttemptId appAttemptId) { + ApplicationAttemptId appAttemptId, boolean unmanagedAMAttempt) { RMAppAttempt appAttempt = mock(RMAppAttempt.class); when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId); when(appAttempt.getHost()).thenReturn("test host"); when(appAttempt.getRpcPort()).thenReturn(-100); - Container container = mock(Container.class); - when(container.getId()) - .thenReturn(ContainerId.newContainerId(appAttemptId, 1)); - when(appAttempt.getMasterContainer()).thenReturn(container); + if (!unmanagedAMAttempt) { + Container container = mock(Container.class); + when(container.getId()) + .thenReturn(ContainerId.newContainerId(appAttemptId, 1)); + when(appAttempt.getMasterContainer()).thenReturn(container); + } when(appAttempt.getDiagnostics()).thenReturn("test diagnostics info"); when(appAttempt.getTrackingUrl()).thenReturn("test tracking url"); when(appAttempt.getOriginalTrackingUrl()).thenReturn(