From 6f4d705a63efe60c2bbeeade7eada83bab675f4f Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Tue, 11 Mar 2014 22:19:30 +0000 Subject: [PATCH] YARN-1821. NPE on registerNodeManager if the request has containers for UnmanagedAMs (kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1576527 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../ResourceTrackerService.java | 18 ++++++----- .../TestResourceTrackerService.java | 31 +++++++++++++++++++ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index bf58ab9a5ff..d63004e465e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -428,6 +428,9 @@ Release 2.4.0 - UNRELEASED apps-killed metrics correctly for killed applications. (Varun Vasudev via vinodkv) + YARN-1821. NPE on registerNodeManager if the request has containers for + UnmanagedAMs. (kasha) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index aad4d92c73a..8a2c53958cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -210,14 +210,16 @@ public class ResourceTrackerService extends AbstractService implements rmContext.getRMApps().get(appAttemptId.getApplicationId()); if (rmApp != null) { RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId); - if (rmAppAttempt.getMasterContainer().getId() - .equals(containerStatus.getContainerId()) - && containerStatus.getState() == ContainerState.COMPLETE) { - // sending master container finished event. - RMAppAttemptContainerFinishedEvent evt = - new RMAppAttemptContainerFinishedEvent(appAttemptId, - containerStatus); - rmContext.getDispatcher().getEventHandler().handle(evt); + if (rmAppAttempt != null) { + if (rmAppAttempt.getMasterContainer().getId() + .equals(containerStatus.getContainerId()) + && containerStatus.getState() == ContainerState.COMPLETE) { + // sending master container finished event. + RMAppAttemptContainerFinishedEvent evt = + new RMAppAttemptContainerFinishedEvent(appAttemptId, + containerStatus); + rmContext.getDispatcher().getEventHandler().handle(evt); + } } } else { LOG.error("Received finished container :" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 803e95f0e07..eed9ecf9fa7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -29,7 +31,11 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; @@ -42,6 +48,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.records.NodeAction; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -50,6 +57,8 @@ import org.apache.hadoop.yarn.util.YarnVersionInfo; import org.junit.After; import org.junit.Test; +import static org.junit.Assert.assertEquals; + public class TestResourceTrackerService { private final static File TEMP_DIR = new File(System.getProperty( @@ -457,6 +466,28 @@ public class TestResourceTrackerService { ClusterMetrics.getMetrics().getUnhealthyNMs()); } + @Test + public void testNodeRegistrationWithContainers() throws Exception { + MockRM rm = new MockRM(); + rm.init(new YarnConfiguration()); + rm.start(); + RMApp app = rm.submitApp(1024); + + MockNM nm = rm.registerNode("host1:1234", 8192); + nm.nodeHeartbeat(true); + + // Register node with some container statuses + ContainerStatus status = ContainerStatus.newInstance( + ContainerId.newInstance(ApplicationAttemptId.newInstance( + app.getApplicationId(), 2), 1), + ContainerState.COMPLETE, "Dummy Completed", 0); + + // The following shouldn't throw NPE + nm.registerNode(Collections.singletonList(status)); + assertEquals("Incorrect number of nodes", 1, + rm.getRMContext().getRMNodes().size()); + } + @Test public void testReconnectNode() throws Exception { final DrainDispatcher dispatcher = new DrainDispatcher();