From c789e944b7166e1430c4b5c8d6def87515d7f46d Mon Sep 17 00:00:00 2001 From: Jim Brennan Date: Tue, 6 Oct 2020 18:18:08 +0000 Subject: [PATCH] YARN-10451. RM (v1) UI NodesPage can NPE when yarn.io/gpu resource type is defined. Contributed by Eric Payne (cherry picked from commit b361f29dda08fa9b4e406d07945d188a914bc437) --- .../resourcemanager/webapp/NodesPage.java | 3 ++- .../resourcemanager/webapp/TestNodesPage.java | 27 +++++++++++++++++-- .../resourcemanager/webapp/TestRMWebApp.java | 23 ++++++++++++++-- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index 446a81098fb..169cfea6e97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -175,7 +175,8 @@ protected void render(Block html) { .get(ResourceInformation.GPU_URI); long usedGPUs = 0; long availableGPUs = 0; - if (gpuIndex != null) { + if (gpuIndex != null && info.getUsedResource() != null + && info.getAvailableResource() != null) { usedGPUs = info.getUsedResource().getResource() .getResourceValue(ResourceInformation.GPU_URI); availableGPUs = info.getAvailableResource().getResource() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java index 1f6a8c08994..00ca6970dd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java @@ -19,12 +19,16 @@ import java.io.IOException; import java.io.PrintWriter; +import java.util.Map; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodesPage.NodesBlock; +import org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.webapp.test.WebAppTests; import org.junit.Before; import org.junit.Test; @@ -56,6 +60,10 @@ public class TestNodesPage { @Before public void setUp() throws Exception { + setUpInternal(false); + } + + private void setUpInternal(final boolean useDRC) throws Exception { final RMContext mockRMContext = TestRMWebApp.mockRMContext(3, numberOfRacks, numberOfNodesPerRack, 8 * TestRMWebApp.GiB); @@ -66,7 +74,7 @@ public void setUp() throws Exception { public void configure(Binder binder) { try { binder.bind(ResourceManager.class).toInstance( - TestRMWebApp.mockRm(mockRMContext)); + TestRMWebApp.mockRm(mockRMContext, useDRC)); } catch (IOException e) { throw new IllegalStateException(e); } @@ -101,7 +109,22 @@ public void testNodesBlockRenderForLostNodes() { Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable)) .print(" oldRtMap = + ResourceUtils.getResourceTypes(); + CustomResourceTypesConfigurationProvider. + initResourceTypes(ResourceInformation.GPU_URI); + this.setUpInternal(true); + try { + this.testNodesBlockRenderForLostNodes(); + } finally { + ResourceUtils.initializeResourcesFromResourceInformationMap(oldRtMap); + } + } + @Test public void testNodesBlockRenderForNodeLabelFilterWithNonEmptyLabel() { NodesBlock nodesBlock = injector.getInstance(NodesBlock.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 1a4d6e3a6c2..0a83f060951 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -247,8 +247,13 @@ public static ResourceManager mockRm(int apps, int racks, int nodes, } public static ResourceManager mockRm(RMContext rmContext) throws IOException { + return mockRm(rmContext, false); + } + + public static ResourceManager mockRm(RMContext rmContext, + boolean useDRC) throws IOException { ResourceManager rm = mock(ResourceManager.class); - ResourceScheduler rs = mockCapacityScheduler(); + ResourceScheduler rs = mockCapacityScheduler(useDRC); ApplicationACLsManager aclMgr = mockAppACLsManager(); ClientRMService clientRMService = mockClientRMService(rmContext); when(rm.getResourceScheduler()).thenReturn(rs); @@ -259,9 +264,14 @@ public static ResourceManager mockRm(RMContext rmContext) throws IOException { } public static CapacityScheduler mockCapacityScheduler() throws IOException { + return mockCapacityScheduler(false); + } + + public static CapacityScheduler mockCapacityScheduler(boolean useDRC) + throws IOException { // stolen from TestCapacityScheduler CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - setupQueueConfiguration(conf); + setupQueueConfiguration(conf, useDRC); CapacityScheduler cs = new CapacityScheduler(); YarnConfiguration yarnConf = new YarnConfiguration(); @@ -313,6 +323,11 @@ public static ClientRMService mockClientRMService(RMContext rmContext) { static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { + setupQueueConfiguration(conf, false); + } + + static void setupQueueConfiguration(CapacitySchedulerConfiguration conf, + boolean useDRC) { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); @@ -358,6 +373,10 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { conf.setCapacity(C11, 15); conf.setCapacity(C12, 45); conf.setCapacity(C13, 40); + if (useDRC) { + conf.set("yarn.scheduler.capacity.resource-calculator", + "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator"); + } } public static ResourceManager mockFifoRm(int apps, int racks, int nodes,