From cd148cb34776f2130a2ecb9fc52ed595faf81e97 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Fri, 15 Apr 2016 20:36:45 +0000 Subject: [PATCH] YARN-4940. yarn node -list -all failed if RM start with decommissioned node. Contributed by sandflee (cherry picked from commit 69f3d428d5c3ab0c79cacffc22b1f59408622ae7) --- .../resourcemanager/NodesListManager.java | 36 ++------------ .../resourcemanager/rmnode/RMNodeImpl.java | 4 +- .../resourcemanager/TestClientRMService.java | 49 ++++++++++++++++++- .../TestRMNodeTransitions.java | 4 +- 4 files changed, 55 insertions(+), 38 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java index ec2708ebb3c..121c418fc3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java @@ -163,7 +163,7 @@ public class NodesListManager extends CompositeService implements private void setDecomissionedNMs() { Set excludeList = hostsReader.getExcludedHosts(); for (final String host : excludeList) { - UnknownNodeId nodeId = new UnknownNodeId(host); + NodeId nodeId = createUnknownNodeId(host); RMNodeImpl rmNode = new RMNodeImpl(nodeId, rmContext, host, -1, -1, new UnknownNode(host), null, null); rmContext.getInactiveRMNodes().put(nodeId, rmNode); @@ -430,38 +430,8 @@ public class NodesListManager extends CompositeService implements * A NodeId instance needed upon startup for populating inactive nodes Map. * It only knows the hostname/ip and marks the port to -1 or invalid. */ - public static class UnknownNodeId extends NodeId { - - private String host; - - public UnknownNodeId(String host) { - this.host = host; - } - - @Override - public String getHost() { - return this.host; - } - - @Override - protected void setHost(String hst) { - - } - - @Override - public int getPort() { - return -1; - } - - @Override - protected void setPort(int port) { - - } - - @Override - protected void build() { - - } + public static NodeId createUnknownNodeId(String host) { + return NodeId.newInstance(host, -1); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index 5f8317e890a..9b807164e7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -786,8 +786,8 @@ public class RMNodeImpl implements RMNode, EventHandler { if (previousRMNode != null) { rmNode.updateMetricsForRejoinedNode(previousRMNode.getState()); } else { - NodesListManager.UnknownNodeId unknownNodeId = - new NodesListManager.UnknownNodeId(nodeId.getHost()); + NodeId unknownNodeId = + NodesListManager.createUnknownNodeId(nodeId.getHost()); previousRMNode = rmNode.context.getInactiveRMNodes().remove(unknownNodeId); if (previousRMNode != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 331f3acde65..bb31f6e35f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -28,6 +28,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -249,7 +251,7 @@ public class TestClientRMService { Assert.assertTrue(report.getNodeLabels() != null && report.getNodeLabels().isEmpty()); } - + rpc.stopProxy(client, conf); rm.close(); } @@ -1566,4 +1568,49 @@ public class TestClientRMService { Assert.assertEquals("Incorrect priority has been returned", expected, updateApplicationPriority.getApplicationPriority().getPriority()); } + + private void createExcludeFile(String filename) throws IOException { + File file = new File(filename); + if (file.exists()) { + file.delete(); + } + + FileOutputStream out = new FileOutputStream(file); + out.write("decommisssionedHost".getBytes()); + out.close(); + } + + @Test + public void testRMStartWithDecommissionedNode() throws Exception { + String excludeFile = "excludeFile"; + createExcludeFile(excludeFile); + YarnConfiguration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, + excludeFile); + MockRM rm = new MockRM(conf) { + protected ClientRMService createClientRMService() { + return new ClientRMService(this.rmContext, scheduler, + this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, + this.getRMContext().getRMDelegationTokenSecretManager()); + }; + }; + rm.start(); + + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); + LOG.info("Connecting to ResourceManager at " + rmAddress); + ApplicationClientProtocol client = + (ApplicationClientProtocol) rpc + .getProxy(ApplicationClientProtocol.class, rmAddress, conf); + + // Make call + GetClusterNodesRequest request = + GetClusterNodesRequest.newInstance(EnumSet.allOf(NodeState.class)); + List nodeReports = client.getClusterNodes(request).getNodeReports(); + Assert.assertEquals(1, nodeReports.size()); + + rm.stop(); + rpc.stopProxy(client, conf); + new File(excludeFile).delete(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index 6ba360bda4f..7c03574cbcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -690,8 +690,8 @@ public class TestRMNodeTransitions { @Test public void testUnknownNodeId() { - NodesListManager.UnknownNodeId nodeId = - new NodesListManager.UnknownNodeId("host1"); + NodeId nodeId = + NodesListManager.createUnknownNodeId("host1"); RMNodeImpl node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null); rmContext.getInactiveRMNodes().putIfAbsent(nodeId,node);