Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-2493. Added node-labels page on RM web UI. Contributed by Wangda Tan 

(cherry picked from commit b7442bf92e)
This commit is contained in:
Jian He 2014-12-30 15:35:45 -08:00
parent 6d65e74670
commit 143e48c25a
14 changed files with 421 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -123,6 +123,8 @@ Release 2.7.0 - UNRELEASED
YARN-2993. Several fixes (missing acl check, error log msg ...) and some
refinement in AdminService. (Yi Liu via junping_du)
YARN-2943. Added node-labels page on RM web UI. (Wangda Tan via jianhe)
OPTIMIZATIONS
BUG FIXES

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
View File

@ -72,8 +72,8 @@ public class CommonNodeLabelsManager extends AbstractService {
protected Dispatcher dispatcher;
protected ConcurrentMap<String, Label> labelCollections =
new ConcurrentHashMap<String, Label>();
protected ConcurrentMap<String, NodeLabel> labelCollections =
new ConcurrentHashMap<String, NodeLabel>();
protected ConcurrentMap<String, Host> nodeCollections =
new ConcurrentHashMap<String, Host>();
@ -82,19 +82,6 @@ public class CommonNodeLabelsManager extends AbstractService {
protected NodeLabelsStore store;
protected static class Label {
private Resource resource;
protected Label() {
this.resource = Resource.newInstance(0, 0);
}
public Resource getResource() {
return this.resource;
}
}
/**
* A <code>Host</code> can have multiple <code>Node</code>s
*/
@ -201,7 +188,7 @@ public class CommonNodeLabelsManager extends AbstractService {
protected void serviceInit(Configuration conf) throws Exception {
initNodeLabelStore(conf);
labelCollections.put(NO_LABEL, new Label());
labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL));
}
protected void initNodeLabelStore(Configuration conf) throws Exception {
@ -271,7 +258,7 @@ public class CommonNodeLabelsManager extends AbstractService {
for (String label : labels) {
// shouldn't overwrite it to avoid changing the Label.resource
if (this.labelCollections.get(label) == null) {
this.labelCollections.put(label, new Label());
this.labelCollections.put(label, new NodeLabel(label));
newLabels.add(label);
}
}

96
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java Normal file
View File

@ -0,0 +1,96 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.nodelabels;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.resource.Resources;
public class NodeLabel implements Comparable<NodeLabel> {
private Resource resource;
private int numActiveNMs;
private String labelName;
public NodeLabel(String labelName) {
this(labelName, Resource.newInstance(0, 0), 0);
}
protected NodeLabel(String labelName, Resource res, int activeNMs) {
this.labelName = labelName;
this.resource = res;
this.numActiveNMs = activeNMs;
}
public void addNode(Resource nodeRes) {
Resources.addTo(resource, nodeRes);
numActiveNMs++;
}
public void removeNode(Resource nodeRes) {
Resources.subtractFrom(resource, nodeRes);
numActiveNMs--;
}
public Resource getResource() {
return this.resource;
}
public int getNumActiveNMs() {
return numActiveNMs;
}
public String getLabelName() {
return labelName;
}
public NodeLabel getCopy() {
return new NodeLabel(labelName, resource, numActiveNMs);
}
@Override
public int compareTo(NodeLabel o) {
// We should always put empty label entry first after sorting
if (labelName.isEmpty() != o.getLabelName().isEmpty()) {
if (labelName.isEmpty()) {
return -1;
}
return 1;
}
return labelName.compareTo(o.getLabelName());
}
@Override
public boolean equals(Object obj) {
if (obj instanceof NodeLabel) {
NodeLabel other = (NodeLabel) obj;
return Resources.equals(resource, other.getResource())
&& StringUtils.equals(labelName, other.getLabelName())
&& (other.getNumActiveNMs() == numActiveNMs);
}
return false;
}
@Override
public int hashCode() {
final int prime = 502357;
return (int) ((((long) labelName.hashCode() << 8)
+ (resource.hashCode() << 4) + numActiveNMs) % prime);
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
View File

@ -32,4 +32,5 @@ public interface YarnWebParams {
String APP_STATE = "app.state";
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";
String NODE_LABEL = "node.label";
}

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
View File

@ -19,10 +19,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@ -37,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
@ -360,8 +363,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
// no label in the past
if (oldLabels.isEmpty()) {
// update labels
Label label = labelCollections.get(NO_LABEL);
Resources.subtractFrom(label.getResource(), oldNM.resource);
NodeLabel label = labelCollections.get(NO_LABEL);
label.removeNode(oldNM.resource);
// update queues, all queue can access this node
for (Queue q : queueCollections.values()) {
@ -370,11 +373,11 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
} else {
// update labels
for (String labelName : oldLabels) {
Label label = labelCollections.get(labelName);
NodeLabel label = labelCollections.get(labelName);
if (null == label) {
continue;
}
Resources.subtractFrom(label.getResource(), oldNM.resource);
label.removeNode(oldNM.resource);
}
// update queues, only queue can access this node will be subtract
@ -395,8 +398,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
// no label in the past
if (newLabels.isEmpty()) {
// update labels
Label label = labelCollections.get(NO_LABEL);
Resources.addTo(label.getResource(), newNM.resource);
NodeLabel label = labelCollections.get(NO_LABEL);
label.addNode(newNM.resource);
// update queues, all queue can access this node
for (Queue q : queueCollections.values()) {
@ -405,8 +408,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
} else {
// update labels
for (String labelName : newLabels) {
Label label = labelCollections.get(labelName);
Resources.addTo(label.getResource(), newNM.resource);
NodeLabel label = labelCollections.get(labelName);
label.addNode(newNM.resource);
}
// update queues, only queue can access this node will be subtract
@ -475,4 +478,21 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
public void setRMContext(RMContext rmContext) {
this.rmContext = rmContext;
}
public List<NodeLabel> pullRMNodeLabelsInfo() {
try {
readLock.lock();
List<NodeLabel> infos = new ArrayList<NodeLabel>();
for (Entry<String, NodeLabel> entry : labelCollections.entrySet()) {
NodeLabel label = entry.getValue();
infos.add(label.getCopy());
}
Collections.sort(infos);
return infos;
} finally {
readLock.unlock();
}
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
View File

@ -33,7 +33,8 @@ public class NavBlock extends HtmlBlock {
h3("Cluster").
ul().
li().a(url("cluster"), "About")._().
li().a(url("nodes"), "Nodes")._();
li().a(url("nodes"), "Nodes")._().
li().a(url("nodelabels"), "Node Labels")._();
UL<LI<UL<DIV<Hamlet>>>> subAppsList = mainList.
li().a(url("apps"), "Applications").
ul();

91
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java Normal file
View File

@ -0,0 +1,91 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
public class NodeLabelsPage extends RmView {
static class NodeLabelsBlock extends HtmlBlock {
final ResourceManager rm;
@Inject
NodeLabelsBlock(ResourceManager rm, ViewContext ctx) {
super(ctx);
this.rm = rm;
}
@Override
protected void render(Block html) {
TBODY<TABLE<Hamlet>> tbody = html.table("#nodelabels").
thead().
tr().
th(".name", "Label Name").
th(".numOfActiveNMs", "Num Of Active NMs").
th(".totalResource", "Total Resource").
_()._().
tbody();
RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager();
for (NodeLabel info : nlm.pullRMNodeLabelsInfo()) {
TR<TBODY<TABLE<Hamlet>>> row =
tbody.tr().td(
info.getLabelName().isEmpty() ? "<NO_LABEL>" : info
.getLabelName());
int nActiveNMs = info.getNumActiveNMs();
if (nActiveNMs > 0) {
row = row.td()
.a(url("nodes",
"?" + YarnWebParams.NODE_LABEL + "=" + info.getLabelName()),
String.valueOf(nActiveNMs))
._();
} else {
row = row.td(String.valueOf(nActiveNMs));
}
row.td(info.getResource().toString())._();
}
tbody._()._();
}
}
@Override protected void preHead(Page.HTML<_> html) {
commonPreHead(html);
String title = "Node labels of the cluster";
setTitle(title);
set(DATATABLES_ID, "nodelabels");
setTableStyles(html, "nodelabels", ".healthStatus {width:10em}",
".healthReport {width:10em}");
}
@Override protected Class<? extends SubView> content() {
return NodeLabelsBlock.class;
}
}

139
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
View File

@ -1,24 +1,25 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp.NODE_STATE;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
@ -28,7 +29,9 @@ import java.util.Collection;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
@ -60,26 +63,20 @@ class NodesPage extends RmView {
ResourceScheduler sched = rm.getResourceScheduler();
String type = $(NODE_STATE);
TBODY<TABLE<Hamlet>> tbody = html.table("#nodes").
thead().
tr().
th(".nodelabels", "Node Labels").
th(".rack", "Rack").
th(".state", "Node State").
th(".nodeaddress", "Node Address").
th(".nodehttpaddress", "Node HTTP Address").
th(".lastHealthUpdate", "Last health-update").
th(".healthReport", "Health-report").
th(".containers", "Containers").
th(".mem", "Mem Used").
th(".mem", "Mem Avail").
th(".vcores", "VCores Used").
th(".vcores", "VCores Avail").
th(".nodeManagerVersion", "Version").
_()._().
tbody();
String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim();
TBODY<TABLE<Hamlet>> tbody =
html.table("#nodes").thead().tr().th(".nodelabels", "Node Labels")
.th(".rack", "Rack").th(".state", "Node State")
.th(".nodeaddress", "Node Address")
.th(".nodehttpaddress", "Node HTTP Address")
.th(".lastHealthUpdate", "Last health-update")
.th(".healthReport", "Health-report")
.th(".containers", "Containers").th(".mem", "Mem Used")
.th(".mem", "Mem Avail").th(".vcores", "VCores Used")
.th(".vcores", "VCores Avail")
.th(".nodeManagerVersion", "Version")._()._().tbody();
NodeState stateFilter = null;
if(type != null && !type.isEmpty()) {
if (type != null && !type.isEmpty()) {
stateFilter = NodeState.valueOf(type.toUpperCase());
}
Collection<RMNode> rmNodes = this.rm.getRMContext().getRMNodes().values();
@ -97,9 +94,9 @@ class NodesPage extends RmView {
}
}
for (RMNode ni : rmNodes) {
if(stateFilter != null) {
if (stateFilter != null) {
NodeState state = ni.getState();
if(!stateFilter.equals(state)) {
if (!stateFilter.equals(state)) {
continue;
}
} else {
@ -109,61 +106,71 @@ class NodesPage extends RmView {
continue;
}
}
// Besides state, we need to filter label as well.
if (!labelFilter.equals(RMNodeLabelsManager.ANY)) {
if (labelFilter.isEmpty()) {
// Empty label filter means only shows nodes without label
if (!ni.getNodeLabels().isEmpty()) {
continue;
}
} else if (!ni.getNodeLabels().contains(labelFilter)) {
// Only nodes have given label can show on web page.
continue;
}
}
NodeInfo info = new NodeInfo(ni, sched);
int usedMemory = (int)info.getUsedMemory();
int availableMemory = (int)info.getAvailableMemory();
TR<TBODY<TABLE<Hamlet>>> row = tbody.tr().
td(StringUtils.join(",", info.getNodeLabels())).
td(info.getRack()).
td(info.getState()).
td(info.getNodeId());
int usedMemory = (int) info.getUsedMemory();
int availableMemory = (int) info.getAvailableMemory();
TR<TBODY<TABLE<Hamlet>>> row =
tbody.tr().td(StringUtils.join(",", info.getNodeLabels()))
.td(info.getRack()).td(info.getState()).td(info.getNodeId());
if (isInactive) {
row.td()._("N/A")._();
} else {
String httpAddress = info.getNodeHTTPAddress();
row.td().a("//" + httpAddress,
httpAddress)._();
row.td().a("//" + httpAddress, httpAddress)._();
}
row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._().
_(Times.format(info.getLastHealthUpdate()))._().
td(info.getHealthReport()).
td(String.valueOf(info.getNumContainers())).
td().br().$title(String.valueOf(usedMemory))._().
_(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
td().br().$title(String.valueOf(availableMemory))._().
_(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
td(String.valueOf(info.getUsedVirtualCores())).
td(String.valueOf(info.getAvailableVirtualCores())).
td(ni.getNodeManagerVersion()).
_();
row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._()
._(Times.format(info.getLastHealthUpdate()))._()
.td(info.getHealthReport())
.td(String.valueOf(info.getNumContainers())).td().br()
.$title(String.valueOf(usedMemory))._()
._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br()
.$title(String.valueOf(availableMemory))._()
._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._()
.td(String.valueOf(info.getUsedVirtualCores()))
.td(String.valueOf(info.getAvailableVirtualCores()))
.td(ni.getNodeManagerVersion())._();
}
tbody._()._();
}
}
@Override protected void preHead(Page.HTML<_> html) {
@Override
protected void preHead(Page.HTML<_> html) {
commonPreHead(html);
String type = $(NODE_STATE);
String title = "Nodes of the cluster";
if(type != null && !type.isEmpty()) {
title = title+" ("+type+")";
if (type != null && !type.isEmpty()) {
title = title + " (" + type + ")";
}
setTitle(title);
set(DATATABLES_ID, "nodes");
set(initID(DATATABLES, "nodes"), nodesTableInit());
setTableStyles(html, "nodes", ".healthStatus {width:10em}",
".healthReport {width:10em}");
".healthReport {width:10em}");
}
@Override protected Class<? extends SubView> content() {
@Override
protected Class<? extends SubView> content() {
return NodesBlock.class;
}
private String nodesTableInit() {
StringBuilder b = tableInit().append(", aoColumnDefs: [");
b.append("{'bSearchable': false, 'aTargets': [ 6 ]}");
b.append(", {'sType': 'title-numeric', 'bSearchable': false, " +
"'aTargets': [ 7, 8 ] }");
b.append(", {'sType': 'title-numeric', 'bSearchable': false, "
+ "'aTargets': [ 7, 8 ] }");
b.append(", {'sType': 'title-numeric', 'aTargets': [ 4 ]}");
b.append("]}");
return b.toString();

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
View File

@ -61,6 +61,7 @@ public class RMWebApp extends WebApp implements YarnWebParams {
route(pajoin("/app", APPLICATION_ID), RmController.class, "app");
route("/scheduler", RmController.class, "scheduler");
route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue");
route("/nodelabels", RmController.class, "nodelabels");
}
@Override

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
View File

@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.util.StringHelper;
import org.apache.hadoop.yarn.webapp.Controller;
import org.apache.hadoop.yarn.webapp.WebAppException;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import com.google.inject.Inject;
@ -93,4 +92,9 @@ public class RmController extends Controller {
public void submit() {
setTitle("Application Submission Not Allowed");
}
public void nodelabels() {
setTitle("Node Labels");
render(NodeLabelsPage.class);
}
}

46
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
View File

@ -30,11 +30,13 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
/**
@ -53,7 +55,12 @@ public class MockNodes {
// One unhealthy node per rack.
list.add(nodeInfo(i, perNode, NodeState.UNHEALTHY));
}
list.add(newNodeInfo(i, perNode));
if (j == 0) {
// One node with label
list.add(nodeInfo(i, perNode, NodeState.RUNNING, ImmutableSet.of("x")));
} else {
list.add(newNodeInfo(i, perNode));
}
}
}
return list;
@ -100,10 +107,12 @@ public class MockNodes {
private String healthReport;
private long lastHealthReportTime;
private NodeState state;
private Set<String> labels;
public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
Resource perNode, String rackName, String healthReport,
long lastHealthReportTime, int cmdPort, String hostName, NodeState state) {
long lastHealthReportTime, int cmdPort, String hostName, NodeState state,
Set<String> labels) {
this.nodeId = nodeId;
this.nodeAddr = nodeAddr;
this.httpAddress = httpAddress;
@ -114,6 +123,7 @@ public class MockNodes {
this.cmdPort = cmdPort;
this.hostName = hostName;
this.state = state;
this.labels = labels;
}
@Override
@ -207,16 +217,33 @@ public class MockNodes {
@Override
public Set<String> getNodeLabels() {
return RMNodeLabelsManager.EMPTY_STRING_SET;
if (labels != null) {
return labels;
}
return CommonNodeLabelsManager.EMPTY_STRING_SET;
}
};
private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) {
return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123);
private static RMNode buildRMNode(int rack, final Resource perNode,
NodeState state, String httpAddr) {
return buildRMNode(rack, perNode, state, httpAddr, null);
}
private static RMNode buildRMNode(int rack, final Resource perNode,
NodeState state, String httpAddr, Set<String> labels) {
return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123,
labels);
}
private static RMNode buildRMNode(int rack, final Resource perNode,
NodeState state, String httpAddr, int hostnum, String hostName, int port) {
return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port,
null);
}
private static RMNode buildRMNode(int rack, final Resource perNode,
NodeState state, String httpAddr, int hostnum, String hostName, int port) {
NodeState state, String httpAddr, int hostnum, String hostName, int port,
Set<String> labels) {
final String rackName = "rack"+ rack;
final int nid = hostnum;
final String nodeAddr = hostName + ":" + nid;
@ -228,13 +255,18 @@ public class MockNodes {
final String httpAddress = httpAddr;
String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode,
rackName, healthReport, 0, nid, hostName, state);
rackName, healthReport, 0, nid, hostName, state, labels);
}
public static RMNode nodeInfo(int rack, final Resource perNode,
NodeState state) {
return buildRMNode(rack, perNode, state, "N/A");
}
public static RMNode nodeInfo(int rack, final Resource perNode,
NodeState state, Set<String> labels) {
return buildRMNode(rack, perNode, state, "N/A", labels);
}
public static RMNode newNodeInfo(int rack, final Resource perNode) {
return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0");

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
View File

@ -839,7 +839,7 @@ public class TestWorkPreservingRMRestart {
// Test if RM on recovery receives the container release request from AM
// before it receives the container status reported by NM for recovery. this
// container should not be recovered.
@Test (timeout = 30000)
@Test (timeout = 50000)
public void testReleasedContainerNotRecovered() throws Exception {
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
View File

@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -27,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
@ -428,4 +430,35 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
Assert.fail("IOException from removeLabelsFromNode " + e);
}
}
private void checkNodeLabelInfo(List<NodeLabel> infos, String labelName, int activeNMs, int memory) {
for (NodeLabel info : infos) {
if (info.getLabelName().equals(labelName)) {
Assert.assertEquals(activeNMs, info.getNumActiveNMs());
Assert.assertEquals(memory, info.getResource().getMemory());
return;
}
}
Assert.fail("Failed to find info has label=" + labelName);
}
@Test(timeout = 5000)
public void testPullRMNodeLabelsInfo() throws IOException {
mgr.addToCluserNodeLabels(toSet("x", "y", "z"));
mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0));
mgr.activateNode(NodeId.newInstance("n2", 1), Resource.newInstance(10, 0));
mgr.activateNode(NodeId.newInstance("n3", 1), Resource.newInstance(10, 0));
mgr.activateNode(NodeId.newInstance("n4", 1), Resource.newInstance(10, 0));
mgr.activateNode(NodeId.newInstance("n5", 1), Resource.newInstance(10, 0));
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("x"),
toNodeId("n2"), toSet("x"), toNodeId("n3"), toSet("y")));
// x, y, z and ""
List<NodeLabel> infos = mgr.pullRMNodeLabelsInfo();
Assert.assertEquals(4, infos.size());
checkNodeLabelInfo(infos, RMNodeLabelsManager.NO_LABEL, 2, 20);
checkNodeLabelInfo(infos, "x", 2, 20);
checkNodeLabelInfo(infos, "y", 1, 10);
checkNodeLabelInfo(infos, "z", 0, 0);
}
}

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
View File

@ -106,4 +106,49 @@ public class TestNodesPage {
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
"<td");
}
@Test
public void testNodesBlockRenderForNodeLabelFilterWithNonEmptyLabel() {
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
nodesBlock.set("node.label", "x");
nodesBlock.render();
PrintWriter writer = injector.getInstance(PrintWriter.class);
WebAppTests.flushOutput(injector);
Mockito.verify(
writer,
Mockito.times(numberOfRacks
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
"<td");
}
@Test
public void testNodesBlockRenderForNodeLabelFilterWithEmptyLabel() {
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
nodesBlock.set("node.label", "");
nodesBlock.render();
PrintWriter writer = injector.getInstance(PrintWriter.class);
WebAppTests.flushOutput(injector);
Mockito.verify(
writer,
Mockito.times(numberOfRacks * (numberOfNodesPerRack - 1)
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
"<td");
}
@Test
public void testNodesBlockRenderForNodeLabelFilterWithAnyLabel() {
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
nodesBlock.set("node.label", "*");
nodesBlock.render();
PrintWriter writer = injector.getInstance(PrintWriter.class);
WebAppTests.flushOutput(injector);
Mockito.verify(
writer,
Mockito.times(numberOfRacks * numberOfNodesPerRack
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
"<td");
}
}