YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource tracking per label when a host runs multiple node-managers. Contributed by Wangda Tan.

This commit is contained in:
Vinod Kumar Vavilapalli 2014-10-15 18:47:26 -07:00
parent f2ea555ac6
commit b3056c266a
3 changed files with 34 additions and 0 deletions

View File

@ -583,6 +583,10 @@ Release 2.6.0 - UNRELEASED
YARN-2628. Capacity scheduler with DominantResourceCalculator carries out YARN-2628. Capacity scheduler with DominantResourceCalculator carries out
reservation even though slots are free. (Varun Vasudev via jianhe) reservation even though slots are free. (Varun Vasudev via jianhe)
YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource
tracking per label when a host runs multiple node-managers. (Wangda Tan via
vinodkv)
BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS
YARN-1707. Introduce APIs to add/remove/resize queues in the YARN-1707. Introduce APIs to add/remove/resize queues in the

View File

@ -129,6 +129,7 @@ public class CommonNodeLabelsManager extends AbstractService {
if (labels != null) { if (labels != null) {
c.labels = c.labels =
Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
c.labels.addAll(labels);
} else { } else {
c.labels = null; c.labels = null;
} }

View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -25,6 +26,7 @@ import java.util.Set;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase; import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After; import org.junit.After;
@ -364,4 +366,31 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
Assert.assertEquals(clusterResource, Assert.assertEquals(clusterResource,
mgr.getQueueResource("Q5", q5Label, clusterResource)); mgr.getQueueResource("Q5", q5Label, clusterResource));
} }
@Test(timeout=5000)
public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOException {
// active two NM to n1, one large and one small
mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE);
mgr.activateNode(NodeId.newInstance("n1", 2), SMALL_RESOURCE);
mgr.activateNode(NodeId.newInstance("n1", 3), SMALL_RESOURCE);
mgr.activateNode(NodeId.newInstance("n1", 4), SMALL_RESOURCE);
// check resource of no label, it should be small * 4
Assert.assertEquals(
mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null),
Resources.multiply(SMALL_RESOURCE, 4));
// change two of these nodes to p1, check resource of no_label and P1
mgr.addToCluserNodeLabels(toSet("p1"));
mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"),
toNodeId("n1:2"), toSet("p1")));
// check resource
Assert.assertEquals(
mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null),
Resources.multiply(SMALL_RESOURCE, 2));
Assert.assertEquals(
mgr.getResourceByLabel("p1", null),
Resources.multiply(SMALL_RESOURCE, 2));
}
} }