diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9d78a6cbbd9..58eb7ea4ee8 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -220,6 +220,9 @@ Release 2.8.0 - UNRELEASED YARN-3594. WintuilsProcessStubExecutor.startStreamReader leaks streams. (Lars Francke via junping_du) + YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager + to get NodeLabel object. (Sunil G via wangda) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index 2d57ad66895..f741f68b478 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -993,13 +993,18 @@ public class CommonNodeLabelsManager extends AbstractService { } } - private Set getLabelsInfoByNode(NodeId nodeId) { - Set labels = getLabelsByNode(nodeId, nodeCollections); - if (labels.isEmpty()) { - return EMPTY_NODELABEL_SET; + public Set getLabelsInfoByNode(NodeId nodeId) { + try { + readLock.lock(); + Set labels = getLabelsByNode(nodeId, nodeCollections); + if (labels.isEmpty()) { + return EMPTY_NODELABEL_SET; + } + Set nodeLabels = createNodeLabelFromLabelNames(labels); + return nodeLabels; + } finally { + readLock.unlock(); } - Set nodeLabels = createNodeLabelFromLabelNames(labels); - return nodeLabels; } private Set createNodeLabelFromLabelNames(Set labels) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index 696b99b82fb..10092ca9a63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -34,6 +34,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index c39f2b9f857..b1fa80ac4ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -810,12 +810,12 @@ public class RMWebServices { NodeToLabelsInfo ntl = new NodeToLabelsInfo(); HashMap ntlMap = ntl.getNodeToLabels(); - Map> nodeIdToLabels = rm.getRMContext() - .getNodeLabelManager().getNodeLabels(); + Map> nodeIdToLabels = rm.getRMContext() + .getNodeLabelManager().getNodeLabelsInfo(); - for (Map.Entry> nitle : nodeIdToLabels.entrySet()) { - ntlMap.put(nitle.getKey().toString(), - new NodeLabelsInfo(nitle.getValue())); + for (Map.Entry> nitle : nodeIdToLabels.entrySet()) { + List labels = new ArrayList(nitle.getValue()); + ntlMap.put(nitle.getKey().toString(), new NodeLabelsInfo(labels)); } return ntl; @@ -830,16 +830,16 @@ public class RMWebServices { LabelsToNodesInfo lts = new LabelsToNodesInfo(); Map ltsMap = lts.getLabelsToNodes(); - Map> labelsToNodeId = null; + Map> labelsToNodeId = null; if (labels == null || labels.size() == 0) { labelsToNodeId = - rm.getRMContext().getNodeLabelManager().getLabelsToNodes(); + rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes(); } else { labelsToNodeId = - rm.getRMContext().getNodeLabelManager().getLabelsToNodes(labels); + rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes(labels); } - for (Entry> entry : labelsToNodeId.entrySet()) { + for (Entry> entry : labelsToNodeId.entrySet()) { List nodeIdStrList = new ArrayList(); for (NodeId nodeId : entry.getValue()) { nodeIdStrList.add(nodeId.toString()); @@ -985,14 +985,13 @@ public class RMWebServices { @Path("/nodes/{nodeId}/get-labels") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr, - @PathParam("nodeId") String nodeId) - throws IOException { + @PathParam("nodeId") String nodeId) throws IOException { init(); NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId); - return new NodeLabelsInfo( - rm.getRMContext().getNodeLabelManager().getLabelsOnNode(nid)); - + List labels = new ArrayList(rm.getRMContext() + .getNodeLabelManager().getLabelsInfoByNode(nid)); + return new NodeLabelsInfo(labels); } protected Response killApp(RMApp app, UserGroupInformation callerUGI, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java index 54fe00d2fd4..53a99029cd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java @@ -133,12 +133,15 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName()); assertEquals(1, nlsifo.getNodeLabels().size()); + for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) { + assertEquals("a", nl.getName()); + assertTrue(nl.getExclusivity()); + } // Add another nlsifo = new NodeLabelsInfo(); - nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b")); + nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b", false)); response = r.path("ws").path("v1").path("cluster") .path("add-node-labels").queryParam("user.name", userName) @@ -154,6 +157,12 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); assertEquals(2, nlsifo.getNodeLabels().size()); + // Verify exclusivity for 'y' as false + for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) { + if (nl.getName().equals("b")) { + assertFalse(nl.getExclusivity()); + } + } // Add labels to a node MultivaluedMapImpl params = new MultivaluedMapImpl(); @@ -202,7 +211,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class); assertEquals(2, ltni.getLabelsToNodes().size()); - NodeIDsInfo nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("b")); + NodeIDsInfo nodes = ltni.getLabelsToNodes().get( + new NodeLabelInfo("b", false)); assertTrue(nodes.getNodeIDs().contains("nid2:0")); assertTrue(nodes.getNodeIDs().contains("nid1:0")); nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("a")); @@ -231,7 +241,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertTrue(nlsifo.getNodeLabelsName().contains("a")); + assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a"))); // Replace @@ -255,7 +265,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertTrue(nlsifo.getNodeLabelsName().contains("b")); + assertTrue(nlsifo.getNodeLabelsInfo().contains( + new NodeLabelInfo("b", false))); // Replace labels using node-to-labels NodeToLabelsEntryList ntli = new NodeToLabelsEntryList(); @@ -281,7 +292,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { NodeToLabelsInfo ntlinfo = response.getEntity(NodeToLabelsInfo.class); NodeLabelsInfo nlinfo = ntlinfo.getNodeToLabels().get("nid:0"); assertEquals(1, nlinfo.getNodeLabels().size()); - assertTrue(nlinfo.getNodeLabelsName().contains("a")); + assertTrue(nlinfo.getNodeLabelsInfo().contains(new NodeLabelInfo("a"))); // Remove all params = new MultivaluedMapImpl(); @@ -303,7 +314,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertTrue(nlsifo.getNodeLabelsName().contains("")); + assertTrue(nlsifo.getNodeLabelsInfo().isEmpty()); // Add a label back for auth tests params = new MultivaluedMapImpl(); @@ -326,7 +337,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertTrue(nlsifo.getNodeLabelsName().contains("a")); + assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a"))); // Auth fail replace labels on node params = new MultivaluedMapImpl(); @@ -347,7 +358,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertTrue(nlsifo.getNodeLabelsName().contains("a")); + assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a"))); // Fail to add a label with post response = @@ -383,8 +394,11 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName()); assertEquals(1, nlsifo.getNodeLabels().size()); + for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) { + assertEquals("a", nl.getName()); + assertTrue(nl.getExclusivity()); + } // Remove cluster label with post params = new MultivaluedMapImpl(); @@ -409,8 +423,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { // configuration is on // Reset for testing : add cluster labels nlsifo = new NodeLabelsInfo(); - nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x")); - nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y")); + nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x", false)); + nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y", false)); response = r.path("ws") .path("v1") @@ -421,8 +435,6 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .entity(toJson(nlsifo, NodeLabelsInfo.class), MediaType.APPLICATION_JSON).post(ClientResponse.class); // Reset for testing : Add labels to a node - nlsifo = new NodeLabelsInfo(); - nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y")); params = new MultivaluedMapImpl(); params.add("labels", "y"); response = @@ -461,7 +473,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { ntlinfo = response.getEntity(NodeToLabelsInfo.class); nlinfo = ntlinfo.getNodeToLabels().get("nid:0"); assertEquals(1, nlinfo.getNodeLabels().size()); - assertFalse(nlinfo.getNodeLabels().contains("x")); + assertFalse(nlinfo.getNodeLabelsInfo().contains( + new NodeLabelInfo("x", false))); // Case2 : failure to Replace labels using replace-labels response = @@ -481,7 +494,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { ntlinfo = response.getEntity(NodeToLabelsInfo.class); nlinfo = ntlinfo.getNodeToLabels().get("nid:0"); assertEquals(1, nlinfo.getNodeLabels().size()); - assertFalse(nlinfo.getNodeLabels().contains("x")); + assertFalse(nlinfo.getNodeLabelsInfo().contains( + new NodeLabelInfo("x", false))); // Case3 : Remove cluster label should be successful params = new MultivaluedMapImpl(); @@ -500,7 +514,10 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); + assertEquals(new NodeLabelInfo("y", false), + nlsifo.getNodeLabelsInfo().get(0)); assertEquals("y", nlsifo.getNodeLabelsInfo().get(0).getName()); + assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity()); // Remove y params = new MultivaluedMapImpl(); @@ -543,7 +560,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); assertEquals("z", nlsifo.getNodeLabelsInfo().get(0).getName()); - assertEquals(false, nlsifo.getNodeLabelsInfo().get(0).getExclusivity()); + assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity()); assertEquals(1, nlsifo.getNodeLabels().size()); }