Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager to get NodeLabel object. (Sunil G via wangda) 

(cherry picked from commit ec0a852a37)
This commit is contained in:
Wangda Tan 2015-05-27 12:51:20 -07:00
parent 3f321be24d
commit 8857cba72e
5 changed files with 62 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -220,6 +220,9 @@ Release 2.8.0 - UNRELEASED
YARN-3594. WintuilsProcessStubExecutor.startStreamReader leaks streams.
(Lars Francke via junping_du)
YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager
to get NodeLabel object. (Sunil G via wangda)
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
View File

@ -993,13 +993,18 @@ public class CommonNodeLabelsManager extends AbstractService {
}
}
private Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) {
Set<String> labels = getLabelsByNode(nodeId, nodeCollections);
if (labels.isEmpty()) {
return EMPTY_NODELABEL_SET;
public Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) {
try {
readLock.lock();
Set<String> labels = getLabelsByNode(nodeId, nodeCollections);
if (labels.isEmpty()) {
return EMPTY_NODELABEL_SET;
}
Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels);
return nodeLabels;
} finally {
readLock.unlock();
}
Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels);
return nodeLabels;
}
private Set<NodeLabel> createNodeLabelFromLabelNames(Set<String> labels) {

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
View File

@ -34,6 +34,7 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.RMNodeLabel;

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -810,12 +810,12 @@ public class RMWebServices {
NodeToLabelsInfo ntl = new NodeToLabelsInfo();
HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels();
Map<NodeId, Set<String>> nodeIdToLabels = rm.getRMContext()
.getNodeLabelManager().getNodeLabels();
Map<NodeId, Set<NodeLabel>> nodeIdToLabels = rm.getRMContext()
.getNodeLabelManager().getNodeLabelsInfo();
for (Map.Entry<NodeId, Set<String>> nitle : nodeIdToLabels.entrySet()) {
ntlMap.put(nitle.getKey().toString(),
new NodeLabelsInfo(nitle.getValue()));
for (Map.Entry<NodeId, Set<NodeLabel>> nitle : nodeIdToLabels.entrySet()) {
List<NodeLabel> labels = new ArrayList<NodeLabel>(nitle.getValue());
ntlMap.put(nitle.getKey().toString(), new NodeLabelsInfo(labels));
}
return ntl;
@ -830,16 +830,16 @@ public class RMWebServices {
LabelsToNodesInfo lts = new LabelsToNodesInfo();
Map<NodeLabelInfo, NodeIDsInfo> ltsMap = lts.getLabelsToNodes();
Map<String, Set<NodeId>> labelsToNodeId = null;
Map<NodeLabel, Set<NodeId>> labelsToNodeId = null;
if (labels == null || labels.size() == 0) {
labelsToNodeId =
rm.getRMContext().getNodeLabelManager().getLabelsToNodes();
rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes();
} else {
labelsToNodeId =
rm.getRMContext().getNodeLabelManager().getLabelsToNodes(labels);
rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes(labels);
}
for (Entry<String, Set<NodeId>> entry : labelsToNodeId.entrySet()) {
for (Entry<NodeLabel, Set<NodeId>> entry : labelsToNodeId.entrySet()) {
List<String> nodeIdStrList = new ArrayList<String>();
for (NodeId nodeId : entry.getValue()) {
nodeIdStrList.add(nodeId.toString());
@ -985,14 +985,13 @@ public class RMWebServices {
@Path("/nodes/{nodeId}/get-labels")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr,
@PathParam("nodeId") String nodeId)
throws IOException {
@PathParam("nodeId") String nodeId) throws IOException {
init();
NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId);
return new NodeLabelsInfo(
rm.getRMContext().getNodeLabelManager().getLabelsOnNode(nid));
List<NodeLabel> labels = new ArrayList<NodeLabel>(rm.getRMContext()
.getNodeLabelManager().getLabelsInfoByNode(nid));
return new NodeLabelsInfo(labels);
}
protected Response killApp(RMApp app, UserGroupInformation callerUGI,

51
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
View File

@ -133,12 +133,15 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName());
assertEquals(1, nlsifo.getNodeLabels().size());
for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
assertEquals("a", nl.getName());
assertTrue(nl.getExclusivity());
}
// Add another
nlsifo = new NodeLabelsInfo();
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b"));
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b", false));
response =
r.path("ws").path("v1").path("cluster")
.path("add-node-labels").queryParam("user.name", userName)
@ -154,6 +157,12 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals(2, nlsifo.getNodeLabels().size());
// Verify exclusivity for 'y' as false
for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
if (nl.getName().equals("b")) {
assertFalse(nl.getExclusivity());
}
}
// Add labels to a node
MultivaluedMapImpl params = new MultivaluedMapImpl();
@ -202,7 +211,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class);
assertEquals(2, ltni.getLabelsToNodes().size());
NodeIDsInfo nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("b"));
NodeIDsInfo nodes = ltni.getLabelsToNodes().get(
new NodeLabelInfo("b", false));
assertTrue(nodes.getNodeIDs().contains("nid2:0"));
assertTrue(nodes.getNodeIDs().contains("nid1:0"));
nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("a"));
@ -231,7 +241,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("a"));
assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Replace
@ -255,7 +265,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("b"));
assertTrue(nlsifo.getNodeLabelsInfo().contains(
new NodeLabelInfo("b", false)));
// Replace labels using node-to-labels
NodeToLabelsEntryList ntli = new NodeToLabelsEntryList();
@ -281,7 +292,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
NodeToLabelsInfo ntlinfo = response.getEntity(NodeToLabelsInfo.class);
NodeLabelsInfo nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size());
assertTrue(nlinfo.getNodeLabelsName().contains("a"));
assertTrue(nlinfo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Remove all
params = new MultivaluedMapImpl();
@ -303,7 +314,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains(""));
assertTrue(nlsifo.getNodeLabelsInfo().isEmpty());
// Add a label back for auth tests
params = new MultivaluedMapImpl();
@ -326,7 +337,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("a"));
assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Auth fail replace labels on node
params = new MultivaluedMapImpl();
@ -347,7 +358,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertTrue(nlsifo.getNodeLabelsName().contains("a"));
assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Fail to add a label with post
response =
@ -383,8 +394,11 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName());
assertEquals(1, nlsifo.getNodeLabels().size());
for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
assertEquals("a", nl.getName());
assertTrue(nl.getExclusivity());
}
// Remove cluster label with post
params = new MultivaluedMapImpl();
@ -409,8 +423,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
// configuration is on
// Reset for testing : add cluster labels
nlsifo = new NodeLabelsInfo();
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x"));
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y"));
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x", false));
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y", false));
response =
r.path("ws")
.path("v1")
@ -421,8 +435,6 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.entity(toJson(nlsifo, NodeLabelsInfo.class),
MediaType.APPLICATION_JSON).post(ClientResponse.class);
// Reset for testing : Add labels to a node
nlsifo = new NodeLabelsInfo();
nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y"));
params = new MultivaluedMapImpl();
params.add("labels", "y");
response =
@ -461,7 +473,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
ntlinfo = response.getEntity(NodeToLabelsInfo.class);
nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size());
assertFalse(nlinfo.getNodeLabels().contains("x"));
assertFalse(nlinfo.getNodeLabelsInfo().contains(
new NodeLabelInfo("x", false)));
// Case2 : failure to Replace labels using replace-labels
response =
@ -481,7 +494,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
ntlinfo = response.getEntity(NodeToLabelsInfo.class);
nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size());
assertFalse(nlinfo.getNodeLabels().contains("x"));
assertFalse(nlinfo.getNodeLabelsInfo().contains(
new NodeLabelInfo("x", false)));
// Case3 : Remove cluster label should be successful
params = new MultivaluedMapImpl();
@ -500,7 +514,10 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals(new NodeLabelInfo("y", false),
nlsifo.getNodeLabelsInfo().get(0));
assertEquals("y", nlsifo.getNodeLabelsInfo().get(0).getName());
assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
// Remove y
params = new MultivaluedMapImpl();
@ -543,7 +560,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals("z", nlsifo.getNodeLabelsInfo().get(0).getName());
assertEquals(false, nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
assertEquals(1, nlsifo.getNodeLabels().size());
}