YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource tracking per label when a host runs multiple node-managers. Contributed by Wangda Tan.
This commit is contained in:
parent
f2ea555ac6
commit
b3056c266a
|
@ -583,6 +583,10 @@ Release 2.6.0 - UNRELEASED
|
||||||
YARN-2628. Capacity scheduler with DominantResourceCalculator carries out
|
YARN-2628. Capacity scheduler with DominantResourceCalculator carries out
|
||||||
reservation even though slots are free. (Varun Vasudev via jianhe)
|
reservation even though slots are free. (Varun Vasudev via jianhe)
|
||||||
|
|
||||||
|
YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource
|
||||||
|
tracking per label when a host runs multiple node-managers. (Wangda Tan via
|
||||||
|
vinodkv)
|
||||||
|
|
||||||
BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS
|
BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS
|
||||||
|
|
||||||
YARN-1707. Introduce APIs to add/remove/resize queues in the
|
YARN-1707. Introduce APIs to add/remove/resize queues in the
|
||||||
|
|
|
@ -129,6 +129,7 @@ public class CommonNodeLabelsManager extends AbstractService {
|
||||||
if (labels != null) {
|
if (labels != null) {
|
||||||
c.labels =
|
c.labels =
|
||||||
Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
|
Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
|
||||||
|
c.labels.addAll(labels);
|
||||||
} else {
|
} else {
|
||||||
c.labels = null;
|
c.labels = null;
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,6 +18,7 @@
|
||||||
|
|
||||||
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
|
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
@ -25,6 +26,7 @@ import java.util.Set;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
|
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
|
||||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
|
@ -364,4 +366,31 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
|
||||||
Assert.assertEquals(clusterResource,
|
Assert.assertEquals(clusterResource,
|
||||||
mgr.getQueueResource("Q5", q5Label, clusterResource));
|
mgr.getQueueResource("Q5", q5Label, clusterResource));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test(timeout=5000)
|
||||||
|
public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOException {
|
||||||
|
// active two NM to n1, one large and one small
|
||||||
|
mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE);
|
||||||
|
mgr.activateNode(NodeId.newInstance("n1", 2), SMALL_RESOURCE);
|
||||||
|
mgr.activateNode(NodeId.newInstance("n1", 3), SMALL_RESOURCE);
|
||||||
|
mgr.activateNode(NodeId.newInstance("n1", 4), SMALL_RESOURCE);
|
||||||
|
|
||||||
|
// check resource of no label, it should be small * 4
|
||||||
|
Assert.assertEquals(
|
||||||
|
mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null),
|
||||||
|
Resources.multiply(SMALL_RESOURCE, 4));
|
||||||
|
|
||||||
|
// change two of these nodes to p1, check resource of no_label and P1
|
||||||
|
mgr.addToCluserNodeLabels(toSet("p1"));
|
||||||
|
mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"),
|
||||||
|
toNodeId("n1:2"), toSet("p1")));
|
||||||
|
|
||||||
|
// check resource
|
||||||
|
Assert.assertEquals(
|
||||||
|
mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null),
|
||||||
|
Resources.multiply(SMALL_RESOURCE, 2));
|
||||||
|
Assert.assertEquals(
|
||||||
|
mgr.getResourceByLabel("p1", null),
|
||||||
|
Resources.multiply(SMALL_RESOURCE, 2));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue