Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager to get NodeLabel object. (Sunil G via wangda)

Browse Source 
(cherry picked from commit ec0a852a37d5c91a62d3d0ff3ddbd9d58235b312)
This commit is contained in:
Wangda Tan 2015-05-27 12:51:20 -07:00
parent 3f321be24d
commit 8857cba72e
5 changed files with 62 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -220,6 +220,9 @@ Release 2.8.0 - UNRELEASED
YARN-3594. WintuilsProcessStubExecutor.startStreamReader leaks streams. YARN-3594. WintuilsProcessStubExecutor.startStreamReader leaks streams.
(Lars Francke via junping_du) (Lars Francke via junping_du)
YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager
to get NodeLabel object. (Sunil G via wangda)
OPTIMIZATIONS OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not YARN-3339. TestDockerContainerExecutor should pull a single image and not

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
View File

@ -993,13 +993,18 @@ protected Set<String> getLabelsByNode(NodeId nodeId, Map<String, Host> map) {
} }
} }
private Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) { public Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) {
Set<String> labels = getLabelsByNode(nodeId, nodeCollections); try {
if (labels.isEmpty()) { readLock.lock();
return EMPTY_NODELABEL_SET; Set<String> labels = getLabelsByNode(nodeId, nodeCollections);
if (labels.isEmpty()) {
return EMPTY_NODELABEL_SET;
}
Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels);
return nodeLabels;
} finally {
readLock.unlock();
} }
Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels);
return nodeLabels;
} }
private Set<NodeLabel> createNodeLabelFromLabelNames(Set<String> labels) { private Set<NodeLabel> createNodeLabelFromLabelNames(Set<String> labels) {

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
View File

@ -34,6 +34,7 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; import org.apache.hadoop.yarn.nodelabels.RMNodeLabel;

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -810,12 +810,12 @@ public NodeToLabelsInfo getNodeToLabels(@Context HttpServletRequest hsr)
NodeToLabelsInfo ntl = new NodeToLabelsInfo(); NodeToLabelsInfo ntl = new NodeToLabelsInfo();
HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels(); HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels();
Map<NodeId, Set<String>> nodeIdToLabels = rm.getRMContext() Map<NodeId, Set<NodeLabel>> nodeIdToLabels = rm.getRMContext()
.getNodeLabelManager().getNodeLabels(); .getNodeLabelManager().getNodeLabelsInfo();
for (Map.Entry<NodeId, Set<String>> nitle : nodeIdToLabels.entrySet()) { for (Map.Entry<NodeId, Set<NodeLabel>> nitle : nodeIdToLabels.entrySet()) {
ntlMap.put(nitle.getKey().toString(), List<NodeLabel> labels = new ArrayList<NodeLabel>(nitle.getValue());
new NodeLabelsInfo(nitle.getValue())); ntlMap.put(nitle.getKey().toString(), new NodeLabelsInfo(labels));
} }
return ntl; return ntl;
@ -830,16 +830,16 @@ public LabelsToNodesInfo getLabelsToNodes(
LabelsToNodesInfo lts = new LabelsToNodesInfo(); LabelsToNodesInfo lts = new LabelsToNodesInfo();
Map<NodeLabelInfo, NodeIDsInfo> ltsMap = lts.getLabelsToNodes(); Map<NodeLabelInfo, NodeIDsInfo> ltsMap = lts.getLabelsToNodes();
Map<String, Set<NodeId>> labelsToNodeId = null; Map<NodeLabel, Set<NodeId>> labelsToNodeId = null;
if (labels == null || labels.size() == 0) { if (labels == null || labels.size() == 0) {
labelsToNodeId = labelsToNodeId =
rm.getRMContext().getNodeLabelManager().getLabelsToNodes(); rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes();
} else { } else {
labelsToNodeId = labelsToNodeId =
rm.getRMContext().getNodeLabelManager().getLabelsToNodes(labels); rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes(labels);
} }
for (Entry<String, Set<NodeId>> entry : labelsToNodeId.entrySet()) { for (Entry<NodeLabel, Set<NodeId>> entry : labelsToNodeId.entrySet()) {
List<String> nodeIdStrList = new ArrayList<String>(); List<String> nodeIdStrList = new ArrayList<String>();
for (NodeId nodeId : entry.getValue()) { for (NodeId nodeId : entry.getValue()) {
nodeIdStrList.add(nodeId.toString()); nodeIdStrList.add(nodeId.toString());
@ -985,14 +985,13 @@ public Response removeFromCluserNodeLabels(
@Path("/nodes/{nodeId}/get-labels") @Path("/nodes/{nodeId}/get-labels")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr, public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr,
@PathParam("nodeId") String nodeId) @PathParam("nodeId") String nodeId) throws IOException {
throws IOException {
init(); init();
NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId); NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId);
return new NodeLabelsInfo( List<NodeLabel> labels = new ArrayList<NodeLabel>(rm.getRMContext()
rm.getRMContext().getNodeLabelManager().getLabelsOnNode(nid)); .getNodeLabelManager().getLabelsInfoByNode(nid));
return new NodeLabelsInfo(labels);
} }
protected Response killApp(RMApp app, UserGroupInformation callerUGI, protected Response killApp(RMApp app, UserGroupInformation callerUGI,

51
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
View File

@ -133,12 +133,15 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class); nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName());
assertEquals(1, nlsifo.getNodeLabels().size()); assertEquals(1, nlsifo.getNodeLabels().size());
for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
assertEquals("a", nl.getName());
assertTrue(nl.getExclusivity());
}
// Add another // Add another
nlsifo = new NodeLabelsInfo(); nlsifo = new NodeLabelsInfo();
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b")); nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b", false));
response = response =
r.path("ws").path("v1").path("cluster") r.path("ws").path("v1").path("cluster")
.path("add-node-labels").queryParam("user.name", userName) .path("add-node-labels").queryParam("user.name", userName)
@ -154,6 +157,12 @@ public void testNodeLabels() throws JSONException, Exception {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class); nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals(2, nlsifo.getNodeLabels().size()); assertEquals(2, nlsifo.getNodeLabels().size());
// Verify exclusivity for 'y' as false
for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
if (nl.getName().equals("b")) {
assertFalse(nl.getExclusivity());
}
}
// Add labels to a node // Add labels to a node
MultivaluedMapImpl params = new MultivaluedMapImpl(); MultivaluedMapImpl params = new MultivaluedMapImpl();
@ -202,7 +211,8 @@ public void testNodeLabels() throws JSONException, Exception {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class); LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class);
assertEquals(2, ltni.getLabelsToNodes().size()); assertEquals(2, ltni.getLabelsToNodes().size());
NodeIDsInfo nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("b")); NodeIDsInfo nodes = ltni.getLabelsToNodes().get(
new NodeLabelInfo("b", false));
assertTrue(nodes.getNodeIDs().contains("nid2:0")); assertTrue(nodes.getNodeIDs().contains("nid2:0"));
assertTrue(nodes.getNodeIDs().contains("nid1:0")); assertTrue(nodes.getNodeIDs().contains("nid1:0"));
nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("a")); nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("a"));
@ -231,7 +241,7 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class); nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("a")); assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Replace // Replace
@ -255,7 +265,8 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class); nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("b")); assertTrue(nlsifo.getNodeLabelsInfo().contains(
new NodeLabelInfo("b", false)));
// Replace labels using node-to-labels // Replace labels using node-to-labels
NodeToLabelsEntryList ntli = new NodeToLabelsEntryList(); NodeToLabelsEntryList ntli = new NodeToLabelsEntryList();
@ -281,7 +292,7 @@ public void testNodeLabels() throws JSONException, Exception {
NodeToLabelsInfo ntlinfo = response.getEntity(NodeToLabelsInfo.class); NodeToLabelsInfo ntlinfo = response.getEntity(NodeToLabelsInfo.class);
NodeLabelsInfo nlinfo = ntlinfo.getNodeToLabels().get("nid:0"); NodeLabelsInfo nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size()); assertEquals(1, nlinfo.getNodeLabels().size());
assertTrue(nlinfo.getNodeLabelsName().contains("a")); assertTrue(nlinfo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Remove all // Remove all
params = new MultivaluedMapImpl(); params = new MultivaluedMapImpl();
@ -303,7 +314,7 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class); nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("")); assertTrue(nlsifo.getNodeLabelsInfo().isEmpty());
// Add a label back for auth tests // Add a label back for auth tests
params = new MultivaluedMapImpl(); params = new MultivaluedMapImpl();
@ -326,7 +337,7 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class); nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("a")); assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Auth fail replace labels on node // Auth fail replace labels on node
params = new MultivaluedMapImpl(); params = new MultivaluedMapImpl();
@ -347,7 +358,7 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class); nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("a")); assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Fail to add a label with post // Fail to add a label with post
response = response =
@ -383,8 +394,11 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class); nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName());
assertEquals(1, nlsifo.getNodeLabels().size()); assertEquals(1, nlsifo.getNodeLabels().size());
for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
assertEquals("a", nl.getName());
assertTrue(nl.getExclusivity());
}
// Remove cluster label with post // Remove cluster label with post
params = new MultivaluedMapImpl(); params = new MultivaluedMapImpl();
@ -409,8 +423,8 @@ public void testNodeLabels() throws JSONException, Exception {
// configuration is on // configuration is on
// Reset for testing : add cluster labels // Reset for testing : add cluster labels
nlsifo = new NodeLabelsInfo(); nlsifo = new NodeLabelsInfo();
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x")); nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x", false));
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y")); nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y", false));
response = response =
r.path("ws") r.path("ws")
.path("v1") .path("v1")
@ -421,8 +435,6 @@ public void testNodeLabels() throws JSONException, Exception {
.entity(toJson(nlsifo, NodeLabelsInfo.class), .entity(toJson(nlsifo, NodeLabelsInfo.class),
MediaType.APPLICATION_JSON).post(ClientResponse.class); MediaType.APPLICATION_JSON).post(ClientResponse.class);
// Reset for testing : Add labels to a node // Reset for testing : Add labels to a node
nlsifo = new NodeLabelsInfo();
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y"));
params = new MultivaluedMapImpl(); params = new MultivaluedMapImpl();
params.add("labels", "y"); params.add("labels", "y");
response = response =
@ -461,7 +473,8 @@ public void testNodeLabels() throws JSONException, Exception {
ntlinfo = response.getEntity(NodeToLabelsInfo.class); ntlinfo = response.getEntity(NodeToLabelsInfo.class);
nlinfo = ntlinfo.getNodeToLabels().get("nid:0"); nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size()); assertEquals(1, nlinfo.getNodeLabels().size());
assertFalse(nlinfo.getNodeLabels().contains("x")); assertFalse(nlinfo.getNodeLabelsInfo().contains(
new NodeLabelInfo("x", false)));
// Case2 : failure to Replace labels using replace-labels // Case2 : failure to Replace labels using replace-labels
response = response =
@ -481,7 +494,8 @@ public void testNodeLabels() throws JSONException, Exception {
ntlinfo = response.getEntity(NodeToLabelsInfo.class); ntlinfo = response.getEntity(NodeToLabelsInfo.class);
nlinfo = ntlinfo.getNodeToLabels().get("nid:0"); nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size()); assertEquals(1, nlinfo.getNodeLabels().size());
assertFalse(nlinfo.getNodeLabels().contains("x")); assertFalse(nlinfo.getNodeLabelsInfo().contains(
new NodeLabelInfo("x", false)));
// Case3 : Remove cluster label should be successful // Case3 : Remove cluster label should be successful
params = new MultivaluedMapImpl(); params = new MultivaluedMapImpl();
@ -500,7 +514,10 @@ public void testNodeLabels() throws JSONException, Exception {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class); nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals(new NodeLabelInfo("y", false),
nlsifo.getNodeLabelsInfo().get(0));
assertEquals("y", nlsifo.getNodeLabelsInfo().get(0).getName()); assertEquals("y", nlsifo.getNodeLabelsInfo().get(0).getName());
assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
// Remove y // Remove y
params = new MultivaluedMapImpl(); params = new MultivaluedMapImpl();
@ -543,7 +560,7 @@ public void testNodeLabels() throws JSONException, Exception {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class); nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals("z", nlsifo.getNodeLabelsInfo().get(0).getName()); assertEquals("z", nlsifo.getNodeLabelsInfo().get(0).getName());
assertEquals(false, nlsifo.getNodeLabelsInfo().get(0).getExclusivity()); assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
assertEquals(1, nlsifo.getNodeLabels().size()); assertEquals(1, nlsifo.getNodeLabels().size());
} }