Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-2843. Fixed NodeLabelsManager to trim inputs for hosts and labels so as to make them work correctly. Contributed by Wangda Tan.

This commit is contained in:
Vinod Kumar Vavilapalli 2014-11-11 12:33:10 -08:00
parent 061bc293c8
commit 0fd97f9c19
6 changed files with 53 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -77,6 +77,9 @@ Release 2.7.0 - UNRELEASED
YARN-2841. RMProxy should retry EOFException. (Jian He via xgong)
YARN-2843. Fixed NodeLabelsManager to trim inputs for hosts and labels so
as to make them work correctly. (Wangda Tan via vinodkv)
Release 2.6.0 - 2014-11-15
INCOMPATIBLE CHANGES

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
View File

@ -57,6 +57,7 @@
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest;
import org.apache.hadoop.yarn.util.ConverterUtils;
import com.google.common.collect.ImmutableMap;
@ -393,18 +394,7 @@ private Map<NodeId, Set<String>> buildNodeLabelsFromStr(String args)
throw new IOException("node name cannot be empty");
}
String nodeName;
int port;
if (nodeIdStr.contains(":")) {
nodeName = nodeIdStr.substring(0, nodeIdStr.indexOf(":"));
port = Integer.valueOf(nodeIdStr.substring(nodeIdStr.indexOf(":") + 1));
} else {
nodeName = nodeIdStr;
port = 0;
}
NodeId nodeId = NodeId.newInstance(nodeName, port);
NodeId nodeId = ConverterUtils.toNodeIdWithDefaultPort(nodeIdStr);
map.put(nodeId, new HashSet<String>());
for (int i = 1; i < splits.length; i++) {

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
View File

@ -344,6 +344,7 @@ protected void internalAddLabelsToNode(
*/
public void addLabelsToNode(Map<NodeId, Set<String>> addedLabelsToNode)
throws IOException {
addedLabelsToNode = normalizeNodeIdToLabels(addedLabelsToNode);
checkAddLabelsToNode(addedLabelsToNode);
internalAddLabelsToNode(addedLabelsToNode);
}
@ -409,6 +410,8 @@ protected void internalRemoveFromClusterNodeLabels(Collection<String> labelsToRe
*/
public void removeFromClusterNodeLabels(Collection<String> labelsToRemove)
throws IOException {
labelsToRemove = normalizeLabels(labelsToRemove);
checkRemoveFromClusterNodeLabels(labelsToRemove);
internalRemoveFromClusterNodeLabels(labelsToRemove);
@ -518,6 +521,8 @@ protected void internalRemoveLabelsFromNode(
public void
removeLabelsFromNode(Map<NodeId, Set<String>> removeLabelsFromNode)
throws IOException {
removeLabelsFromNode = normalizeNodeIdToLabels(removeLabelsFromNode);
checkRemoveLabelsFromNode(removeLabelsFromNode);
internalRemoveLabelsFromNode(removeLabelsFromNode);
@ -590,6 +595,8 @@ protected void internalReplaceLabelsOnNode(
*/
public void replaceLabelsOnNode(Map<NodeId, Set<String>> replaceLabelsToNode)
throws IOException {
replaceLabelsToNode = normalizeNodeIdToLabels(replaceLabelsToNode);
checkReplaceLabelsOnNode(replaceLabelsToNode);
internalReplaceLabelsOnNode(replaceLabelsToNode);
@ -665,7 +672,7 @@ protected String normalizeLabel(String label) {
return NO_LABEL;
}
private Set<String> normalizeLabels(Set<String> labels) {
private Set<String> normalizeLabels(Collection<String> labels) {
Set<String> newLabels = new HashSet<String>();
for (String label : labels) {
newLabels.add(normalizeLabel(label));
@ -732,4 +739,15 @@ protected void createHostIfNonExisted(String hostName) {
nodeCollections.put(hostName, host);
}
}
protected Map<NodeId, Set<String>> normalizeNodeIdToLabels(
Map<NodeId, Set<String>> nodeIdToLabels) {
Map<NodeId, Set<String>> newMap = new HashMap<NodeId, Set<String>>();
for (Entry<NodeId, Set<String>> entry : nodeIdToLabels.entrySet()) {
NodeId id = entry.getKey();
Set<String> labels = entry.getValue();
newMap.put(id, normalizeLabels(labels));
}
return newMap;
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
View File

@ -167,7 +167,7 @@ public static NodeId toNodeId(String nodeIdStr) {
}
try {
NodeId nodeId =
NodeId.newInstance(parts[0], Integer.parseInt(parts[1]));
NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1]));
return nodeId;
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Invalid port: " + parts[1], e);

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java
View File

@ -19,7 +19,7 @@
package org.apache.hadoop.yarn.nodelabels;
import java.util.Collection;
import java.util.Iterator;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@ -49,12 +49,10 @@ public static void assertMapContains(Map<NodeId, Set<String>> m1,
public static void assertCollectionEquals(Collection<String> c1,
Collection<String> c2) {
Assert.assertEquals(c1.size(), c2.size());
Iterator<String> i1 = c1.iterator();
Iterator<String> i2 = c2.iterator();
while (i1.hasNext()) {
Assert.assertEquals(i1.next(), i2.next());
}
Set<String> s1 = new HashSet<String>(c1);
Set<String> s2 = new HashSet<String>(c2);
Assert.assertEquals(s1, s2);
Assert.assertTrue(s1.containsAll(s2));
}
public static <E> Set<E> toSet(E... elements) {

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
View File

@ -258,4 +258,27 @@ public void testRemovelabelWithNodes() throws Exception {
Assert.assertTrue(mgr.getClusterNodeLabels().isEmpty());
assertCollectionEquals(mgr.lastRemovedlabels, Arrays.asList("p2", "p3"));
}
@Test(timeout = 5000)
public void testTrimLabelsWhenAddRemoveNodeLabels() throws IOException {
mgr.addToCluserNodeLabels(toSet(" p1"));
assertCollectionEquals(mgr.getClusterNodeLabels(), toSet("p1"));
mgr.removeFromClusterNodeLabels(toSet("p1 "));
Assert.assertTrue(mgr.getClusterNodeLabels().isEmpty());
}
@Test(timeout = 5000)
public void testTrimLabelsWhenModifyLabelsOnNodes() throws IOException {
mgr.addToCluserNodeLabels(toSet(" p1", "p2"));
mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1 ", "p2")));
assertMapEquals(
mgr.getNodeLabels(),
ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2")));
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet(" p2")));
assertMapEquals(
mgr.getNodeLabels(),
ImmutableMap.of(toNodeId("n1"), toSet("p2")));
mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet(" p2 ")));
Assert.assertTrue(mgr.getNodeLabels().isEmpty());
}
}