Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "YARN-2492(wrong jira number). Added node-labels page on RM web UI. Contributed by Wangda Tan" 

This reverts commit 5f57b904f5.
This commit is contained in:
Jian He 2014-12-30 16:46:26 -08:00
parent 5f57b904f5
commit 746ad6e989
14 changed files with 101 additions and 421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -155,8 +155,6 @@ Release 2.7.0 - UNRELEASED
YARN-2993. Several fixes (missing acl check, error log msg ...) and some YARN-2993. Several fixes (missing acl check, error log msg ...) and some
refinement in AdminService. (Yi Liu via junping_du) refinement in AdminService. (Yi Liu via junping_du)
YARN-2943. Added node-labels page on RM web UI. (Wangda Tan via jianhe)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
View File

@ -72,8 +72,8 @@ public class CommonNodeLabelsManager extends AbstractService {
protected Dispatcher dispatcher; protected Dispatcher dispatcher;
protected ConcurrentMap<String, NodeLabel> labelCollections = protected ConcurrentMap<String, Label> labelCollections =
new ConcurrentHashMap<String, NodeLabel>(); new ConcurrentHashMap<String, Label>();
protected ConcurrentMap<String, Host> nodeCollections = protected ConcurrentMap<String, Host> nodeCollections =
new ConcurrentHashMap<String, Host>(); new ConcurrentHashMap<String, Host>();
@ -82,6 +82,19 @@ public class CommonNodeLabelsManager extends AbstractService {
protected NodeLabelsStore store; protected NodeLabelsStore store;
protected static class Label {
private Resource resource;
protected Label() {
this.resource = Resource.newInstance(0, 0);
}
public Resource getResource() {
return this.resource;
}
}
/** /**
* A <code>Host</code> can have multiple <code>Node</code>s * A <code>Host</code> can have multiple <code>Node</code>s
*/ */
@ -188,7 +201,7 @@ public class CommonNodeLabelsManager extends AbstractService {
protected void serviceInit(Configuration conf) throws Exception { protected void serviceInit(Configuration conf) throws Exception {
initNodeLabelStore(conf); initNodeLabelStore(conf);
labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL)); labelCollections.put(NO_LABEL, new Label());
} }
protected void initNodeLabelStore(Configuration conf) throws Exception { protected void initNodeLabelStore(Configuration conf) throws Exception {
@ -258,7 +271,7 @@ public class CommonNodeLabelsManager extends AbstractService {
for (String label : labels) { for (String label : labels) {
// shouldn't overwrite it to avoid changing the Label.resource // shouldn't overwrite it to avoid changing the Label.resource
if (this.labelCollections.get(label) == null) { if (this.labelCollections.get(label) == null) {
this.labelCollections.put(label, new NodeLabel(label)); this.labelCollections.put(label, new Label());
newLabels.add(label); newLabels.add(label);
} }
} }

96
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java
View File

@ -1,96 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.nodelabels;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.resource.Resources;
public class NodeLabel implements Comparable<NodeLabel> {
private Resource resource;
private int numActiveNMs;
private String labelName;
public NodeLabel(String labelName) {
this(labelName, Resource.newInstance(0, 0), 0);
}
protected NodeLabel(String labelName, Resource res, int activeNMs) {
this.labelName = labelName;
this.resource = res;
this.numActiveNMs = activeNMs;
}
public void addNode(Resource nodeRes) {
Resources.addTo(resource, nodeRes);
numActiveNMs++;
}
public void removeNode(Resource nodeRes) {
Resources.subtractFrom(resource, nodeRes);
numActiveNMs--;
}
public Resource getResource() {
return this.resource;
}
public int getNumActiveNMs() {
return numActiveNMs;
}
public String getLabelName() {
return labelName;
}
public NodeLabel getCopy() {
return new NodeLabel(labelName, resource, numActiveNMs);
}
@Override
public int compareTo(NodeLabel o) {
// We should always put empty label entry first after sorting
if (labelName.isEmpty() != o.getLabelName().isEmpty()) {
if (labelName.isEmpty()) {
return -1;
}
return 1;
}
return labelName.compareTo(o.getLabelName());
}
@Override
public boolean equals(Object obj) {
if (obj instanceof NodeLabel) {
NodeLabel other = (NodeLabel) obj;
return Resources.equals(resource, other.getResource())
&& StringUtils.equals(labelName, other.getLabelName())
&& (other.getNumActiveNMs() == numActiveNMs);
}
return false;
}
@Override
public int hashCode() {
final int prime = 502357;
return (int) ((((long) labelName.hashCode() << 8)
+ (resource.hashCode() << 4) + numActiveNMs) % prime);
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
View File

@ -32,5 +32,4 @@ public interface YarnWebParams {
String APP_STATE = "app.state"; String APP_STATE = "app.state";
String QUEUE_NAME = "queue.name"; String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state"; String NODE_STATE = "node.state";
String NODE_LABEL = "node.label";
} }

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
View File

@ -19,12 +19,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set; import java.util.Set;
@ -39,7 +37,6 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent;
import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.resource.Resources;
@ -363,8 +360,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
// no label in the past // no label in the past
if (oldLabels.isEmpty()) { if (oldLabels.isEmpty()) {
// update labels // update labels
NodeLabel label = labelCollections.get(NO_LABEL); Label label = labelCollections.get(NO_LABEL);
label.removeNode(oldNM.resource); Resources.subtractFrom(label.getResource(), oldNM.resource);
// update queues, all queue can access this node // update queues, all queue can access this node
for (Queue q : queueCollections.values()) { for (Queue q : queueCollections.values()) {
@ -373,11 +370,11 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
} else { } else {
// update labels // update labels
for (String labelName : oldLabels) { for (String labelName : oldLabels) {
NodeLabel label = labelCollections.get(labelName); Label label = labelCollections.get(labelName);
if (null == label) { if (null == label) {
continue; continue;
} }
label.removeNode(oldNM.resource); Resources.subtractFrom(label.getResource(), oldNM.resource);
} }
// update queues, only queue can access this node will be subtract // update queues, only queue can access this node will be subtract
@ -398,8 +395,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
// no label in the past // no label in the past
if (newLabels.isEmpty()) { if (newLabels.isEmpty()) {
// update labels // update labels
NodeLabel label = labelCollections.get(NO_LABEL); Label label = labelCollections.get(NO_LABEL);
label.addNode(newNM.resource); Resources.addTo(label.getResource(), newNM.resource);
// update queues, all queue can access this node // update queues, all queue can access this node
for (Queue q : queueCollections.values()) { for (Queue q : queueCollections.values()) {
@ -408,8 +405,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
} else { } else {
// update labels // update labels
for (String labelName : newLabels) { for (String labelName : newLabels) {
NodeLabel label = labelCollections.get(labelName); Label label = labelCollections.get(labelName);
label.addNode(newNM.resource); Resources.addTo(label.getResource(), newNM.resource);
} }
// update queues, only queue can access this node will be subtract // update queues, only queue can access this node will be subtract
@ -478,21 +475,4 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
public void setRMContext(RMContext rmContext) { public void setRMContext(RMContext rmContext) {
this.rmContext = rmContext; this.rmContext = rmContext;
} }
public List<NodeLabel> pullRMNodeLabelsInfo() {
try {
readLock.lock();
List<NodeLabel> infos = new ArrayList<NodeLabel>();
for (Entry<String, NodeLabel> entry : labelCollections.entrySet()) {
NodeLabel label = entry.getValue();
infos.add(label.getCopy());
}
Collections.sort(infos);
return infos;
} finally {
readLock.unlock();
}
}
} }

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
View File

@ -33,8 +33,7 @@ public class NavBlock extends HtmlBlock {
h3("Cluster"). h3("Cluster").
ul(). ul().
li().a(url("cluster"), "About")._(). li().a(url("cluster"), "About")._().
li().a(url("nodes"), "Nodes")._(). li().a(url("nodes"), "Nodes")._();
li().a(url("nodelabels"), "Node Labels")._();
UL<LI<UL<DIV<Hamlet>>>> subAppsList = mainList. UL<LI<UL<DIV<Hamlet>>>> subAppsList = mainList.
li().a(url("apps"), "Applications"). li().a(url("apps"), "Applications").
ul(); ul();

91
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java
View File

@ -1,91 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
public class NodeLabelsPage extends RmView {
static class NodeLabelsBlock extends HtmlBlock {
final ResourceManager rm;
@Inject
NodeLabelsBlock(ResourceManager rm, ViewContext ctx) {
super(ctx);
this.rm = rm;
}
@Override
protected void render(Block html) {
TBODY<TABLE<Hamlet>> tbody = html.table("#nodelabels").
thead().
tr().
th(".name", "Label Name").
th(".numOfActiveNMs", "Num Of Active NMs").
th(".totalResource", "Total Resource").
_()._().
tbody();
RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager();
for (NodeLabel info : nlm.pullRMNodeLabelsInfo()) {
TR<TBODY<TABLE<Hamlet>>> row =
tbody.tr().td(
info.getLabelName().isEmpty() ? "<NO_LABEL>" : info
.getLabelName());
int nActiveNMs = info.getNumActiveNMs();
if (nActiveNMs > 0) {
row = row.td()
.a(url("nodes",
"?" + YarnWebParams.NODE_LABEL + "=" + info.getLabelName()),
String.valueOf(nActiveNMs))
._();
} else {
row = row.td(String.valueOf(nActiveNMs));
}
row.td(info.getResource().toString())._();
}
tbody._()._();
}
}
@Override protected void preHead(Page.HTML<_> html) {
commonPreHead(html);
String title = "Node labels of the cluster";
setTitle(title);
set(DATATABLES_ID, "nodelabels");
setTableStyles(html, "nodelabels", ".healthStatus {width:10em}",
".healthReport {width:10em}");
}
@Override protected Class<? extends SubView> content() {
return NodeLabelsBlock.class;
}
}

91
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
View File

@ -18,8 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp; package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp.NODE_STATE;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
@ -29,9 +28,7 @@ import java.util.Collection;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
@ -63,18 +60,24 @@ class NodesPage extends RmView {
ResourceScheduler sched = rm.getResourceScheduler(); ResourceScheduler sched = rm.getResourceScheduler();
String type = $(NODE_STATE); String type = $(NODE_STATE);
String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim(); TBODY<TABLE<Hamlet>> tbody = html.table("#nodes").
TBODY<TABLE<Hamlet>> tbody = thead().
html.table("#nodes").thead().tr().th(".nodelabels", "Node Labels") tr().
.th(".rack", "Rack").th(".state", "Node State") th(".nodelabels", "Node Labels").
.th(".nodeaddress", "Node Address") th(".rack", "Rack").
.th(".nodehttpaddress", "Node HTTP Address") th(".state", "Node State").
.th(".lastHealthUpdate", "Last health-update") th(".nodeaddress", "Node Address").
.th(".healthReport", "Health-report") th(".nodehttpaddress", "Node HTTP Address").
.th(".containers", "Containers").th(".mem", "Mem Used") th(".lastHealthUpdate", "Last health-update").
.th(".mem", "Mem Avail").th(".vcores", "VCores Used") th(".healthReport", "Health-report").
.th(".vcores", "VCores Avail") th(".containers", "Containers").
.th(".nodeManagerVersion", "Version")._()._().tbody(); th(".mem", "Mem Used").
th(".mem", "Mem Avail").
th(".vcores", "VCores Used").
th(".vcores", "VCores Avail").
th(".nodeManagerVersion", "Version").
_()._().
tbody();
NodeState stateFilter = null; NodeState stateFilter = null;
if(type != null && !type.isEmpty()) { if(type != null && !type.isEmpty()) {
stateFilter = NodeState.valueOf(type.toUpperCase()); stateFilter = NodeState.valueOf(type.toUpperCase());
@ -106,48 +109,39 @@ class NodesPage extends RmView {
continue; continue;
} }
} }
// Besides state, we need to filter label as well.
if (!labelFilter.equals(RMNodeLabelsManager.ANY)) {
if (labelFilter.isEmpty()) {
// Empty label filter means only shows nodes without label
if (!ni.getNodeLabels().isEmpty()) {
continue;
}
} else if (!ni.getNodeLabels().contains(labelFilter)) {
// Only nodes have given label can show on web page.
continue;
}
}
NodeInfo info = new NodeInfo(ni, sched); NodeInfo info = new NodeInfo(ni, sched);
int usedMemory = (int)info.getUsedMemory(); int usedMemory = (int)info.getUsedMemory();
int availableMemory = (int)info.getAvailableMemory(); int availableMemory = (int)info.getAvailableMemory();
TR<TBODY<TABLE<Hamlet>>> row = TR<TBODY<TABLE<Hamlet>>> row = tbody.tr().
tbody.tr().td(StringUtils.join(",", info.getNodeLabels())) td(StringUtils.join(",", info.getNodeLabels())).
.td(info.getRack()).td(info.getState()).td(info.getNodeId()); td(info.getRack()).
td(info.getState()).
td(info.getNodeId());
if (isInactive) { if (isInactive) {
row.td()._("N/A")._(); row.td()._("N/A")._();
} else { } else {
String httpAddress = info.getNodeHTTPAddress(); String httpAddress = info.getNodeHTTPAddress();
row.td().a("//" + httpAddress, httpAddress)._(); row.td().a("//" + httpAddress,
httpAddress)._();
} }
row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._() row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._().
._(Times.format(info.getLastHealthUpdate()))._() _(Times.format(info.getLastHealthUpdate()))._().
.td(info.getHealthReport()) td(info.getHealthReport()).
.td(String.valueOf(info.getNumContainers())).td().br() td(String.valueOf(info.getNumContainers())).
.$title(String.valueOf(usedMemory))._() td().br().$title(String.valueOf(usedMemory))._().
._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br() _(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
.$title(String.valueOf(availableMemory))._() td().br().$title(String.valueOf(availableMemory))._().
._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._() _(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
.td(String.valueOf(info.getUsedVirtualCores())) td(String.valueOf(info.getUsedVirtualCores())).
.td(String.valueOf(info.getAvailableVirtualCores())) td(String.valueOf(info.getAvailableVirtualCores())).
.td(ni.getNodeManagerVersion())._(); td(ni.getNodeManagerVersion()).
_();
} }
tbody._()._(); tbody._()._();
} }
} }
@Override @Override protected void preHead(Page.HTML<_> html) {
protected void preHead(Page.HTML<_> html) {
commonPreHead(html); commonPreHead(html);
String type = $(NODE_STATE); String type = $(NODE_STATE);
String title = "Nodes of the cluster"; String title = "Nodes of the cluster";
@ -161,16 +155,15 @@ class NodesPage extends RmView {
".healthReport {width:10em}"); ".healthReport {width:10em}");
} }
@Override @Override protected Class<? extends SubView> content() {
protected Class<? extends SubView> content() {
return NodesBlock.class; return NodesBlock.class;
} }
private String nodesTableInit() { private String nodesTableInit() {
StringBuilder b = tableInit().append(", aoColumnDefs: ["); StringBuilder b = tableInit().append(", aoColumnDefs: [");
b.append("{'bSearchable': false, 'aTargets': [ 6 ]}"); b.append("{'bSearchable': false, 'aTargets': [ 6 ]}");
b.append(", {'sType': 'title-numeric', 'bSearchable': false, " b.append(", {'sType': 'title-numeric', 'bSearchable': false, " +
+ "'aTargets': [ 7, 8 ] }"); "'aTargets': [ 7, 8 ] }");
b.append(", {'sType': 'title-numeric', 'aTargets': [ 4 ]}"); b.append(", {'sType': 'title-numeric', 'aTargets': [ 4 ]}");
b.append("]}"); b.append("]}");
return b.toString(); return b.toString();

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
View File

@ -61,7 +61,6 @@ public class RMWebApp extends WebApp implements YarnWebParams {
route(pajoin("/app", APPLICATION_ID), RmController.class, "app"); route(pajoin("/app", APPLICATION_ID), RmController.class, "app");
route("/scheduler", RmController.class, "scheduler"); route("/scheduler", RmController.class, "scheduler");
route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue"); route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue");
route("/nodelabels", RmController.class, "nodelabels");
} }
@Override @Override

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
View File

@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.util.StringHelper;
import org.apache.hadoop.yarn.webapp.Controller; import org.apache.hadoop.yarn.webapp.Controller;
import org.apache.hadoop.yarn.webapp.WebAppException;
import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.YarnWebParams;
import com.google.inject.Inject; import com.google.inject.Inject;
@ -92,9 +93,4 @@ public class RmController extends Controller {
public void submit() { public void submit() {
setTitle("Application Submission Not Allowed"); setTitle("Application Submission Not Allowed");
} }
public void nodelabels() {
setTitle("Node Labels");
render(NodeLabelsPage.class);
}
} }

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
View File

@ -30,13 +30,11 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
/** /**
@ -55,14 +53,9 @@ public class MockNodes {
// One unhealthy node per rack. // One unhealthy node per rack.
list.add(nodeInfo(i, perNode, NodeState.UNHEALTHY)); list.add(nodeInfo(i, perNode, NodeState.UNHEALTHY));
} }
if (j == 0) {
// One node with label
list.add(nodeInfo(i, perNode, NodeState.RUNNING, ImmutableSet.of("x")));
} else {
list.add(newNodeInfo(i, perNode)); list.add(newNodeInfo(i, perNode));
} }
} }
}
return list; return list;
} }
@ -107,12 +100,10 @@ public class MockNodes {
private String healthReport; private String healthReport;
private long lastHealthReportTime; private long lastHealthReportTime;
private NodeState state; private NodeState state;
private Set<String> labels;
public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
Resource perNode, String rackName, String healthReport, Resource perNode, String rackName, String healthReport,
long lastHealthReportTime, int cmdPort, String hostName, NodeState state, long lastHealthReportTime, int cmdPort, String hostName, NodeState state) {
Set<String> labels) {
this.nodeId = nodeId; this.nodeId = nodeId;
this.nodeAddr = nodeAddr; this.nodeAddr = nodeAddr;
this.httpAddress = httpAddress; this.httpAddress = httpAddress;
@ -123,7 +114,6 @@ public class MockNodes {
this.cmdPort = cmdPort; this.cmdPort = cmdPort;
this.hostName = hostName; this.hostName = hostName;
this.state = state; this.state = state;
this.labels = labels;
} }
@Override @Override
@ -217,33 +207,16 @@ public class MockNodes {
@Override @Override
public Set<String> getNodeLabels() { public Set<String> getNodeLabels() {
if (labels != null) { return RMNodeLabelsManager.EMPTY_STRING_SET;
return labels;
}
return CommonNodeLabelsManager.EMPTY_STRING_SET;
} }
}; };
private static RMNode buildRMNode(int rack, final Resource perNode, private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) {
NodeState state, String httpAddr) { return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123);
return buildRMNode(rack, perNode, state, httpAddr, null);
}
private static RMNode buildRMNode(int rack, final Resource perNode,
NodeState state, String httpAddr, Set<String> labels) {
return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123,
labels);
} }
private static RMNode buildRMNode(int rack, final Resource perNode, private static RMNode buildRMNode(int rack, final Resource perNode,
NodeState state, String httpAddr, int hostnum, String hostName, int port) { NodeState state, String httpAddr, int hostnum, String hostName, int port) {
return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port,
null);
}
private static RMNode buildRMNode(int rack, final Resource perNode,
NodeState state, String httpAddr, int hostnum, String hostName, int port,
Set<String> labels) {
final String rackName = "rack"+ rack; final String rackName = "rack"+ rack;
final int nid = hostnum; final int nid = hostnum;
final String nodeAddr = hostName + ":" + nid; final String nodeAddr = hostName + ":" + nid;
@ -255,7 +228,7 @@ public class MockNodes {
final String httpAddress = httpAddr; final String httpAddress = httpAddr;
String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe"; String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode, return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode,
rackName, healthReport, 0, nid, hostName, state, labels); rackName, healthReport, 0, nid, hostName, state);
} }
public static RMNode nodeInfo(int rack, final Resource perNode, public static RMNode nodeInfo(int rack, final Resource perNode,
@ -263,11 +236,6 @@ public class MockNodes {
return buildRMNode(rack, perNode, state, "N/A"); return buildRMNode(rack, perNode, state, "N/A");
} }
public static RMNode nodeInfo(int rack, final Resource perNode,
NodeState state, Set<String> labels) {
return buildRMNode(rack, perNode, state, "N/A", labels);
}
public static RMNode newNodeInfo(int rack, final Resource perNode) { public static RMNode newNodeInfo(int rack, final Resource perNode) {
return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0"); return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0");
} }

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
View File

@ -839,7 +839,7 @@ public class TestWorkPreservingRMRestart {
// Test if RM on recovery receives the container release request from AM // Test if RM on recovery receives the container release request from AM
// before it receives the container status reported by NM for recovery. this // before it receives the container status reported by NM for recovery. this
// container should not be recovered. // container should not be recovered.
@Test (timeout = 50000) @Test (timeout = 30000)
public void testReleasedContainerNotRecovered() throws Exception { public void testReleasedContainerNotRecovered() throws Exception {
MemoryRMStateStore memStore = new MemoryRMStateStore(); MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf); memStore.init(conf);

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
View File

@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -28,7 +27,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase; import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After; import org.junit.After;
@ -430,35 +428,4 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
Assert.fail("IOException from removeLabelsFromNode " + e); Assert.fail("IOException from removeLabelsFromNode " + e);
} }
} }
private void checkNodeLabelInfo(List<NodeLabel> infos, String labelName, int activeNMs, int memory) {
for (NodeLabel info : infos) {
if (info.getLabelName().equals(labelName)) {
Assert.assertEquals(activeNMs, info.getNumActiveNMs());
Assert.assertEquals(memory, info.getResource().getMemory());
return;
}
}
Assert.fail("Failed to find info has label=" + labelName);
}
@Test(timeout = 5000)
public void testPullRMNodeLabelsInfo() throws IOException {
mgr.addToCluserNodeLabels(toSet("x", "y", "z"));
mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0));
mgr.activateNode(NodeId.newInstance("n2", 1), Resource.newInstance(10, 0));
mgr.activateNode(NodeId.newInstance("n3", 1), Resource.newInstance(10, 0));
mgr.activateNode(NodeId.newInstance("n4", 1), Resource.newInstance(10, 0));
mgr.activateNode(NodeId.newInstance("n5", 1), Resource.newInstance(10, 0));
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("x"),
toNodeId("n2"), toSet("x"), toNodeId("n3"), toSet("y")));
// x, y, z and ""
List<NodeLabel> infos = mgr.pullRMNodeLabelsInfo();
Assert.assertEquals(4, infos.size());
checkNodeLabelInfo(infos, RMNodeLabelsManager.NO_LABEL, 2, 20);
checkNodeLabelInfo(infos, "x", 2, 20);
checkNodeLabelInfo(infos, "y", 1, 10);
checkNodeLabelInfo(infos, "z", 0, 0);
}
} }

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
View File

@ -106,49 +106,4 @@ public class TestNodesPage {
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print( * numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
"<td"); "<td");
} }
@Test
public void testNodesBlockRenderForNodeLabelFilterWithNonEmptyLabel() {
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
nodesBlock.set("node.label", "x");
nodesBlock.render();
PrintWriter writer = injector.getInstance(PrintWriter.class);
WebAppTests.flushOutput(injector);
Mockito.verify(
writer,
Mockito.times(numberOfRacks
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
"<td");
}
@Test
public void testNodesBlockRenderForNodeLabelFilterWithEmptyLabel() {
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
nodesBlock.set("node.label", "");
nodesBlock.render();
PrintWriter writer = injector.getInstance(PrintWriter.class);
WebAppTests.flushOutput(injector);
Mockito.verify(
writer,
Mockito.times(numberOfRacks * (numberOfNodesPerRack - 1)
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
"<td");
}
@Test
public void testNodesBlockRenderForNodeLabelFilterWithAnyLabel() {
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
nodesBlock.set("node.label", "*");
nodesBlock.render();
PrintWriter writer = injector.getInstance(PrintWriter.class);
WebAppTests.flushOutput(injector);
Mockito.verify(
writer,
Mockito.times(numberOfRacks * numberOfNodesPerRack
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
"<td");
}
} }