From 563eb1ad2ae848a23bbbf32ebfaf107e8fa14e87 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Tue, 19 May 2015 16:54:38 -0700 Subject: [PATCH] YARN-3583. Support of NodeLabel object instead of plain String in YarnClient side. (Sunil G via wangda) --- .../hadoop/mapred/ResourceMgrDelegate.java | 6 +- hadoop-yarn-project/CHANGES.txt | 3 + .../GetLabelsToNodesResponse.java | 7 +- .../GetNodesToLabelsResponse.java | 7 +- ...erver_resourcemanager_service_protos.proto | 7 +- .../src/main/proto/yarn_protos.proto | 6 +- .../src/main/proto/yarn_service_protos.proto | 2 +- .../hadoop/yarn/client/api/YarnClient.java | 8 +- .../yarn/client/api/impl/YarnClientImpl.java | 6 +- .../yarn/client/api/impl/TestYarnClient.java | 74 ++++++++++-- .../pb/GetLabelsToNodesResponsePBImpl.java | 28 +++-- .../pb/GetNodesToLabelsResponsePBImpl.java | 56 +++++---- .../pb/ReplaceLabelsOnNodeRequestPBImpl.java | 18 +-- .../resourcemanager/ClientRMService.java | 6 +- .../resourcemanager/TestClientRMService.java | 110 +++++++++++------- 15 files changed, 225 insertions(+), 119 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 2b7cd5fd6f2..90f6876c17a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -444,19 +444,19 @@ public class ResourceMgrDelegate extends YarnClient { } @Override - public Map> getNodeToLabels() throws YarnException, + public Map> getNodeToLabels() throws YarnException, IOException { return client.getNodeToLabels(); } @Override - public Map> getLabelsToNodes() throws YarnException, + public Map> getLabelsToNodes() throws YarnException, IOException { return client.getLabelsToNodes(); } @Override - public Map> getLabelsToNodes(Set labels) + public Map> getLabelsToNodes(Set labels) throws YarnException, IOException { return client.getLabelsToNodes(labels); } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ab6f488f3a6..9ba9fd87358 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -250,6 +250,9 @@ Release 2.8.0 - UNRELEASED YARN-3565. NodeHeartbeatRequest/RegisterNodeManagerRequest should use NodeLabel object instead of String. (Naganarasimha G R via wangda) + YARN-3583. Support of NodeLabel object instead of plain String + in YarnClient side. (Sunil G via wangda) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java index f1053591104..da2be28830e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java @@ -24,11 +24,12 @@ import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.util.Records; public abstract class GetLabelsToNodesResponse { public static GetLabelsToNodesResponse newInstance( - Map> map) { + Map> map) { GetLabelsToNodesResponse response = Records.newRecord(GetLabelsToNodesResponse.class); response.setLabelsToNodes(map); @@ -37,9 +38,9 @@ public abstract class GetLabelsToNodesResponse { @Public @Evolving - public abstract void setLabelsToNodes(Map> map); + public abstract void setLabelsToNodes(Map> map); @Public @Evolving - public abstract Map> getLabelsToNodes(); + public abstract Map> getLabelsToNodes(); } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java index bcd5421e7eb..432485c358a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java @@ -24,11 +24,12 @@ import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.util.Records; public abstract class GetNodesToLabelsResponse { public static GetNodesToLabelsResponse newInstance( - Map> map) { + Map> map) { GetNodesToLabelsResponse response = Records.newRecord(GetNodesToLabelsResponse.class); response.setNodeToLabels(map); @@ -37,9 +38,9 @@ public abstract class GetNodesToLabelsResponse { @Public @Evolving - public abstract void setNodeToLabels(Map> map); + public abstract void setNodeToLabels(Map> map); @Public @Evolving - public abstract Map> getNodeToLabels(); + public abstract Map> getNodeToLabels(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index d6d8713a247..e20b4aea447 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -91,7 +91,7 @@ message RemoveFromClusterNodeLabelsResponseProto { } message ReplaceLabelsOnNodeRequestProto { - repeated NodeIdToLabelsProto nodeToLabels = 1; + repeated NodeIdToLabelsNameProto nodeToLabels = 1; } message ReplaceLabelsOnNodeResponseProto { @@ -107,6 +107,11 @@ message CheckForDecommissioningNodesResponseProto { repeated NodeIdProto decommissioningNodes = 1; } +message NodeIdToLabelsNameProto { + optional NodeIdProto nodeId = 1; + repeated string nodeLabels = 2; +} + enum DecommissionTypeProto { NORMAL = 1; GRACEFUL = 2; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 3c4aa524c3d..b9969b0ef41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -248,13 +248,13 @@ message NodeReportProto { repeated string node_labels = 10; } -message NodeIdToLabelsProto { +message NodeIdToLabelsInfoProto { optional NodeIdProto nodeId = 1; - repeated string nodeLabels = 2; + repeated NodeLabelProto nodeLabels = 2; } message LabelsToNodeIdsProto { - optional string nodeLabels = 1; + optional NodeLabelProto nodeLabels = 1; repeated NodeIdProto nodeId = 2; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 410b66382c1..098785aac5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -198,7 +198,7 @@ message GetNodesToLabelsRequestProto { } message GetNodesToLabelsResponseProto { - repeated NodeIdToLabelsProto nodeToLabels = 1; + repeated NodeIdToLabelsInfoProto nodeToLabels = 1; } message GetLabelsToNodesRequestProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 5ce626c46ab..ff03c7d5af7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -619,7 +619,7 @@ public abstract class YarnClient extends AbstractService { */ @Public @Unstable - public abstract Map> getNodeToLabels() + public abstract Map> getNodeToLabels() throws YarnException, IOException; /** @@ -634,7 +634,7 @@ public abstract class YarnClient extends AbstractService { */ @Public @Unstable - public abstract Map> getLabelsToNodes() + public abstract Map> getLabelsToNodes() throws YarnException, IOException; /** @@ -650,8 +650,8 @@ public abstract class YarnClient extends AbstractService { */ @Public @Unstable - public abstract Map> getLabelsToNodes(Set labels) - throws YarnException, IOException; + public abstract Map> getLabelsToNodes( + Set labels) throws YarnException, IOException; /** *

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 42dd5cdb623..be4c8c4e1bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -795,21 +795,21 @@ public class YarnClientImpl extends YarnClient { } @Override - public Map> getNodeToLabels() throws YarnException, + public Map> getNodeToLabels() throws YarnException, IOException { return rmClient.getNodeToLabels(GetNodesToLabelsRequest.newInstance()) .getNodeToLabels(); } @Override - public Map> getLabelsToNodes() throws YarnException, + public Map> getLabelsToNodes() throws YarnException, IOException { return rmClient.getLabelsToNodes(GetLabelsToNodesRequest.newInstance()) .getLabelsToNodes(); } @Override - public Map> getLabelsToNodes(Set labels) + public Map> getLabelsToNodes(Set labels) throws YarnException, IOException { return rmClient.getLabelsToNodes( GetLabelsToNodesRequest.newInstance(labels)).getLabelsToNodes(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index 10b9bbbbae6..511fa4acb39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -67,6 +67,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; @@ -87,6 +89,7 @@ import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; @@ -458,9 +461,9 @@ public class TestYarnClient { client.start(); // Get labels to nodes mapping - Map> expectedLabelsToNodes = + Map> expectedLabelsToNodes = ((MockYarnClient)client).getLabelsToNodesMap(); - Map> labelsToNodes = client.getLabelsToNodes(); + Map> labelsToNodes = client.getLabelsToNodes(); Assert.assertEquals(labelsToNodes, expectedLabelsToNodes); Assert.assertEquals(labelsToNodes.size(), 3); @@ -476,7 +479,32 @@ public class TestYarnClient { client.close(); } + @Test (timeout = 10000) + public void testGetNodesToLabels() throws YarnException, IOException { + Configuration conf = new Configuration(); + final YarnClient client = new MockYarnClient(); + client.init(conf); + client.start(); + + // Get labels to nodes mapping + Map> expectedNodesToLabels = ((MockYarnClient) client) + .getNodeToLabelsMap(); + Map> nodesToLabels = client.getNodeToLabels(); + Assert.assertEquals(nodesToLabels, expectedNodesToLabels); + Assert.assertEquals(nodesToLabels.size(), 1); + + // Verify exclusivity + Set labels = nodesToLabels.get(NodeId.newInstance("host", 0)); + for (NodeLabel label : labels) { + Assert.assertFalse(label.isExclusive()); + } + + client.stop(); + client.close(); + } + private static class MockYarnClient extends YarnClientImpl { + private ApplicationReport mockReport; private List reports; private HashMap> attempts = @@ -498,6 +526,8 @@ public class TestYarnClient { mock(GetContainerReportResponse.class); GetLabelsToNodesResponse mockLabelsToNodesResponse = mock(GetLabelsToNodesResponse.class); + GetNodesToLabelsResponse mockNodeToLabelsResponse = + mock(GetNodesToLabelsResponse.class); public MockYarnClient() { super(); @@ -537,6 +567,9 @@ public class TestYarnClient { when(rmClient.getLabelsToNodes(any(GetLabelsToNodesRequest.class))) .thenReturn(mockLabelsToNodesResponse); + when(rmClient.getNodeToLabels(any(GetNodesToLabelsRequest.class))) + .thenReturn(mockNodeToLabelsResponse); + historyClient = mock(AHSClient.class); } catch (YarnException e) { @@ -704,7 +737,7 @@ public class TestYarnClient { } @Override - public Map> getLabelsToNodes() + public Map> getLabelsToNodes() throws YarnException, IOException { when(mockLabelsToNodesResponse.getLabelsToNodes()).thenReturn( getLabelsToNodesMap()); @@ -712,35 +745,52 @@ public class TestYarnClient { } @Override - public Map> getLabelsToNodes(Set labels) + public Map> getLabelsToNodes(Set labels) throws YarnException, IOException { when(mockLabelsToNodesResponse.getLabelsToNodes()).thenReturn( getLabelsToNodesMap(labels)); return super.getLabelsToNodes(labels); } - public Map> getLabelsToNodesMap() { - Map> map = new HashMap>(); + public Map> getLabelsToNodesMap() { + Map> map = new HashMap>(); Set setNodeIds = new HashSet(Arrays.asList( NodeId.newInstance("host1", 0), NodeId.newInstance("host2", 0))); - map.put("x", setNodeIds); - map.put("y", setNodeIds); - map.put("z", setNodeIds); + map.put(NodeLabel.newInstance("x"), setNodeIds); + map.put(NodeLabel.newInstance("y"), setNodeIds); + map.put(NodeLabel.newInstance("z"), setNodeIds); return map; } - public Map> getLabelsToNodesMap(Set labels) { - Map> map = new HashMap>(); + public Map> getLabelsToNodesMap(Set labels) { + Map> map = new HashMap>(); Set setNodeIds = new HashSet(Arrays.asList( NodeId.newInstance("host1", 0), NodeId.newInstance("host2", 0))); for(String label : labels) { - map.put(label, setNodeIds); + map.put(NodeLabel.newInstance(label), setNodeIds); } return map; } + @Override + public Map> getNodeToLabels() throws YarnException, + IOException { + when(mockNodeToLabelsResponse.getNodeToLabels()).thenReturn( + getNodeToLabelsMap()); + return super.getNodeToLabels(); + } + + public Map> getNodeToLabelsMap() { + Map> map = new HashMap>(); + Set setNodeLabels = new HashSet(Arrays.asList( + NodeLabel.newInstance("x", false), + NodeLabel.newInstance("y", false))); + map.put(NodeId.newInstance("host", 0), setNodeLabels); + return map; + } + @Override public List getApplicationAttempts( ApplicationId appId) throws YarnException, IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java index e1979973320..418fcbd4cc6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java @@ -29,11 +29,13 @@ import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse; - import org.apache.hadoop.yarn.proto.YarnProtos.LabelsToNodeIdsProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLabelsToNodesResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLabelsToNodesResponseProtoOrBuilder; @@ -44,7 +46,7 @@ public class GetLabelsToNodesResponsePBImpl extends GetLabelsToNodesResponseProto.Builder builder = null; boolean viaProto = false; - private Map> labelsToNodes; + private Map> labelsToNodes; public GetLabelsToNodesResponsePBImpl() { this.builder = GetLabelsToNodesResponseProto.newBuilder(); @@ -61,7 +63,7 @@ public class GetLabelsToNodesResponsePBImpl extends } GetLabelsToNodesResponseProtoOrBuilder p = viaProto ? proto : builder; List list = p.getLabelsToNodesList(); - this.labelsToNodes = new HashMap>(); + this.labelsToNodes = new HashMap>(); for (LabelsToNodeIdsProto c : list) { Set setNodes = new HashSet(); @@ -69,8 +71,9 @@ public class GetLabelsToNodesResponsePBImpl extends NodeId node = new NodeIdPBImpl(n); setNodes.add(node); } - if(!setNodes.isEmpty()) { - this.labelsToNodes.put(c.getNodeLabels(), setNodes); + if (!setNodes.isEmpty()) { + this.labelsToNodes + .put(new NodeLabelPBImpl(c.getNodeLabels()), setNodes); } } } @@ -94,7 +97,7 @@ public class GetLabelsToNodesResponsePBImpl extends public Iterator iterator() { return new Iterator() { - Iterator>> iter = + Iterator>> iter = labelsToNodes.entrySet().iterator(); @Override @@ -104,13 +107,14 @@ public class GetLabelsToNodesResponsePBImpl extends @Override public LabelsToNodeIdsProto next() { - Entry> now = iter.next(); + Entry> now = iter.next(); Set nodeProtoSet = new HashSet(); for(NodeId n : now.getValue()) { nodeProtoSet.add(convertToProtoFormat(n)); } return LabelsToNodeIdsProto.newBuilder() - .setNodeLabels(now.getKey()).addAllNodeId(nodeProtoSet) + .setNodeLabels(convertToProtoFormat(now.getKey())) + .addAllNodeId(nodeProtoSet) .build(); } @@ -149,6 +153,10 @@ public class GetLabelsToNodesResponsePBImpl extends return ((NodeIdPBImpl)t).getProto(); } + private NodeLabelProto convertToProtoFormat(NodeLabel l) { + return ((NodeLabelPBImpl)l).getProto(); + } + @Override public int hashCode() { assert false : "hashCode not designed"; @@ -168,7 +176,7 @@ public class GetLabelsToNodesResponsePBImpl extends @Override @Public @Evolving - public void setLabelsToNodes(Map> map) { + public void setLabelsToNodes(Map> map) { initLabelsToNodes(); labelsToNodes.clear(); labelsToNodes.putAll(map); @@ -177,7 +185,7 @@ public class GetLabelsToNodesResponsePBImpl extends @Override @Public @Evolving - public Map> getLabelsToNodes() { + public Map> getLabelsToNodes() { initLabelsToNodes(); return this.labelsToNodes; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java index 34048300803..52be73f6a4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -26,12 +27,13 @@ import java.util.Map.Entry; import java.util.Set; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; - -import com.google.common.collect.Sets; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsInfoProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProtoOrBuilder; @@ -42,8 +44,8 @@ public class GetNodesToLabelsResponsePBImpl extends GetNodesToLabelsResponseProto.Builder builder = null; boolean viaProto = false; - private Map> nodeToLabels; - + private Map> nodeToLabels; + public GetNodesToLabelsResponsePBImpl() { this.builder = GetNodesToLabelsResponseProto.newBuilder(); } @@ -58,12 +60,15 @@ public class GetNodesToLabelsResponsePBImpl extends return; } GetNodesToLabelsResponseProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getNodeToLabelsList(); - this.nodeToLabels = new HashMap>(); + List list = p.getNodeToLabelsList(); + this.nodeToLabels = new HashMap>(); - for (NodeIdToLabelsProto c : list) { - this.nodeToLabels.put(new NodeIdPBImpl(c.getNodeId()), - Sets.newHashSet(c.getNodeLabelsList())); + for (NodeIdToLabelsInfoProto c : list) { + Set labels = new HashSet(); + for (NodeLabelProto l : c.getNodeLabelsList()) { + labels.add(new NodeLabelPBImpl(l)); + } + this.nodeToLabels.put(new NodeIdPBImpl(c.getNodeId()), labels); } } @@ -80,13 +85,13 @@ public class GetNodesToLabelsResponsePBImpl extends if (nodeToLabels == null) { return; } - Iterable iterable = - new Iterable() { + Iterable iterable = + new Iterable() { @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { - Iterator>> iter = nodeToLabels + Iterator>> iter = nodeToLabels .entrySet().iterator(); @Override @@ -95,11 +100,16 @@ public class GetNodesToLabelsResponsePBImpl extends } @Override - public NodeIdToLabelsProto next() { - Entry> now = iter.next(); - return NodeIdToLabelsProto.newBuilder() + public NodeIdToLabelsInfoProto next() { + Entry> now = iter.next(); + Set labelProtoList = + new HashSet(); + for (NodeLabel l : now.getValue()) { + labelProtoList.add(convertToProtoFormat(l)); + } + return NodeIdToLabelsInfoProto.newBuilder() .setNodeId(convertToProtoFormat(now.getKey())) - .addAllNodeLabels(now.getValue()).build(); + .addAllNodeLabels(labelProtoList).build(); } @Override @@ -134,13 +144,13 @@ public class GetNodesToLabelsResponsePBImpl extends } @Override - public Map> getNodeToLabels() { + public Map> getNodeToLabels() { initNodeToLabels(); return this.nodeToLabels; } @Override - public void setNodeToLabels(Map> map) { + public void setNodeToLabels(Map> map) { initNodeToLabels(); nodeToLabels.clear(); nodeToLabels.putAll(map); @@ -150,6 +160,10 @@ public class GetNodesToLabelsResponsePBImpl extends return ((NodeIdPBImpl)t).getProto(); } + private NodeLabelProto convertToProtoFormat(NodeLabel t) { + return ((NodeLabelPBImpl)t).getProto(); + } + @Override public int hashCode() { assert false : "hashCode not designed"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java index e296aaf13b8..22e561cd94a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java @@ -28,7 +28,7 @@ import java.util.Set; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodeIdToLabelsNameProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProtoOrBuilder; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; @@ -58,10 +58,10 @@ public class ReplaceLabelsOnNodeRequestPBImpl extends return; } ReplaceLabelsOnNodeRequestProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getNodeToLabelsList(); + List list = p.getNodeToLabelsList(); this.nodeIdToLabels = new HashMap>(); - for (NodeIdToLabelsProto c : list) { + for (NodeIdToLabelsNameProto c : list) { this.nodeIdToLabels.put(new NodeIdPBImpl(c.getNodeId()), Sets.newHashSet(c.getNodeLabelsList())); } @@ -80,11 +80,11 @@ public class ReplaceLabelsOnNodeRequestPBImpl extends if (nodeIdToLabels == null) { return; } - Iterable iterable = - new Iterable() { + Iterable iterable = + new Iterable() { @Override - public Iterator iterator() { - return new Iterator() { + public Iterator iterator() { + return new Iterator() { Iterator>> iter = nodeIdToLabels .entrySet().iterator(); @@ -95,9 +95,9 @@ public class ReplaceLabelsOnNodeRequestPBImpl extends } @Override - public NodeIdToLabelsProto next() { + public NodeIdToLabelsNameProto next() { Entry> now = iter.next(); - return NodeIdToLabelsProto.newBuilder() + return NodeIdToLabelsNameProto.newBuilder() .setNodeId(convertToProtoFormat(now.getKey())).clearNodeLabels() .addAllNodeLabels(now.getValue()).build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 428b9ebecf0..e4199be7be2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -1227,7 +1227,7 @@ public class ClientRMService extends AbstractService implements GetNodesToLabelsRequest request) throws YarnException, IOException { RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); GetNodesToLabelsResponse response = - GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabels()); + GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabelsInfo()); return response; } @@ -1237,10 +1237,10 @@ public class ClientRMService extends AbstractService implements RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); if (request.getNodeLabels() == null || request.getNodeLabels().isEmpty()) { return GetLabelsToNodesResponse.newInstance( - labelsMgr.getLabelsToNodes()); + labelsMgr.getLabelsInfoToNodes()); } else { return GetLabelsToNodesResponse.newInstance( - labelsMgr.getLabelsToNodes(request.getNodeLabels())); + labelsMgr.getLabelsInfoToNodes(request.getNodeLabels())); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index a39f94f07a2..20343a51856 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -1407,8 +1407,10 @@ public class TestClientRMService { }; }; rm.start(); + NodeLabel labelX = NodeLabel.newInstance("x", false); + NodeLabel labelY = NodeLabel.newInstance("y"); RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager(); - labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); + labelsMgr.addToCluserNodeLabels(ImmutableSet.of(labelX, labelY)); NodeId node1 = NodeId.newInstance("host1", 1234); NodeId node2 = NodeId.newInstance("host2", 1234); @@ -1422,25 +1424,37 @@ public class TestClientRMService { YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); LOG.info("Connecting to ResourceManager at " + rmAddress); - ApplicationClientProtocol client = - (ApplicationClientProtocol) rpc.getProxy( - ApplicationClientProtocol.class, rmAddress, conf); + ApplicationClientProtocol client = (ApplicationClientProtocol) rpc + .getProxy(ApplicationClientProtocol.class, rmAddress, conf); // Get node labels collection - GetClusterNodeLabelsResponse response = - client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance()); + GetClusterNodeLabelsResponse response = client + .getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance()); Assert.assertTrue(response.getNodeLabels().containsAll( - Arrays.asList(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")))); + Arrays.asList(labelX, labelY))); // Get node labels mapping - GetNodesToLabelsResponse response1 = - client.getNodeToLabels(GetNodesToLabelsRequest.newInstance()); - Map> nodeToLabels = response1.getNodeToLabels(); + GetNodesToLabelsResponse response1 = client + .getNodeToLabels(GetNodesToLabelsRequest.newInstance()); + Map> nodeToLabels = response1.getNodeToLabels(); Assert.assertTrue(nodeToLabels.keySet().containsAll( Arrays.asList(node1, node2))); - Assert.assertTrue(nodeToLabels.get(node1).containsAll(Arrays.asList("x"))); - Assert.assertTrue(nodeToLabels.get(node2).containsAll(Arrays.asList("y"))); - + Assert.assertTrue(nodeToLabels.get(node1) + .containsAll(Arrays.asList(labelX))); + Assert.assertTrue(nodeToLabels.get(node2) + .containsAll(Arrays.asList(labelY))); + // Verify whether labelX's exclusivity is false + for (NodeLabel x : nodeToLabels.get(node1)) { + Assert.assertFalse(x.isExclusive()); + } + // Verify whether labelY's exclusivity is true + for (NodeLabel y : nodeToLabels.get(node2)) { + Assert.assertTrue(y.isExclusive()); + } + // Below label "x" is not present in the response as exclusivity is true + Assert.assertFalse(nodeToLabels.get(node1).containsAll( + Arrays.asList(NodeLabel.newInstance("x")))); + rpc.stopProxy(client, conf); rm.close(); } @@ -1456,8 +1470,12 @@ public class TestClientRMService { }; }; rm.start(); + + NodeLabel labelX = NodeLabel.newInstance("x", false); + NodeLabel labelY = NodeLabel.newInstance("y", false); + NodeLabel labelZ = NodeLabel.newInstance("z", false); RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager(); - labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z")); + labelsMgr.addToCluserNodeLabels(ImmutableSet.of(labelX, labelY, labelZ)); NodeId node1A = NodeId.newInstance("host1", 1234); NodeId node1B = NodeId.newInstance("host1", 5678); @@ -1477,43 +1495,49 @@ public class TestClientRMService { YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); LOG.info("Connecting to ResourceManager at " + rmAddress); - ApplicationClientProtocol client = - (ApplicationClientProtocol) rpc.getProxy( - ApplicationClientProtocol.class, rmAddress, conf); + ApplicationClientProtocol client = (ApplicationClientProtocol) rpc + .getProxy(ApplicationClientProtocol.class, rmAddress, conf); // Get node labels collection - GetClusterNodeLabelsResponse response = - client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance()); + GetClusterNodeLabelsResponse response = client + .getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance()); Assert.assertTrue(response.getNodeLabels().containsAll( - Arrays.asList(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"), - NodeLabel.newInstance("z")))); + Arrays.asList(labelX, labelY, labelZ))); // Get labels to nodes mapping - GetLabelsToNodesResponse response1 = - client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance()); - Map> labelsToNodes = response1.getLabelsToNodes(); - Assert.assertTrue( - labelsToNodes.keySet().containsAll(Arrays.asList("x", "y", "z"))); - Assert.assertTrue( - labelsToNodes.get("x").containsAll(Arrays.asList(node1A))); - Assert.assertTrue( - labelsToNodes.get("y").containsAll(Arrays.asList(node2A, node3A))); - Assert.assertTrue( - labelsToNodes.get("z").containsAll(Arrays.asList(node1B, node3B))); + GetLabelsToNodesResponse response1 = client + .getLabelsToNodes(GetLabelsToNodesRequest.newInstance()); + Map> labelsToNodes = response1.getLabelsToNodes(); + // Verify whether all NodeLabel's exclusivity are false + for (Map.Entry> nltn : labelsToNodes.entrySet()) { + Assert.assertFalse(nltn.getKey().isExclusive()); + } + Assert.assertTrue(labelsToNodes.keySet().containsAll( + Arrays.asList(labelX, labelY, labelZ))); + Assert.assertTrue(labelsToNodes.get(labelX).containsAll( + Arrays.asList(node1A))); + Assert.assertTrue(labelsToNodes.get(labelY).containsAll( + Arrays.asList(node2A, node3A))); + Assert.assertTrue(labelsToNodes.get(labelZ).containsAll( + Arrays.asList(node1B, node3B))); // Get labels to nodes mapping for specific labels - Set setlabels = - new HashSet(Arrays.asList(new String[]{"x", "z"})); - GetLabelsToNodesResponse response2 = - client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance(setlabels)); + Set setlabels = new HashSet(Arrays.asList(new String[]{"x", + "z"})); + GetLabelsToNodesResponse response2 = client + .getLabelsToNodes(GetLabelsToNodesRequest.newInstance(setlabels)); labelsToNodes = response2.getLabelsToNodes(); - Assert.assertTrue( - labelsToNodes.keySet().containsAll(Arrays.asList("x", "z"))); - Assert.assertTrue( - labelsToNodes.get("x").containsAll(Arrays.asList(node1A))); - Assert.assertTrue( - labelsToNodes.get("z").containsAll(Arrays.asList(node1B, node3B))); - Assert.assertEquals(labelsToNodes.get("y"), null); + // Verify whether all NodeLabel's exclusivity are false + for (Map.Entry> nltn : labelsToNodes.entrySet()) { + Assert.assertFalse(nltn.getKey().isExclusive()); + } + Assert.assertTrue(labelsToNodes.keySet().containsAll( + Arrays.asList(labelX, labelZ))); + Assert.assertTrue(labelsToNodes.get(labelX).containsAll( + Arrays.asList(node1A))); + Assert.assertTrue(labelsToNodes.get(labelZ).containsAll( + Arrays.asList(node1B, node3B))); + Assert.assertEquals(labelsToNodes.get(labelY), null); rpc.stopProxy(client, conf); rm.close();