YARN-2493. Added node-labels page on RM web UI. Contributed by Wangda Tan
(cherry picked from commit b7442bf92e)
This commit is contained in:
Jian He
parent
6d65e74670
commit
143e48c25a
|
|
@ -123,6 +123,8 @@ Release 2.7.0 - UNRELEASED
|
|||
YARN-2993. Several fixes (missing acl check, error log msg ...) and some
|
||||
refinement in AdminService. (Yi Liu via junping_du)
|
||||
|
||||
YARN-2943. Added node-labels page on RM web UI. (Wangda Tan via jianhe)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
|||
|
|
@ -72,8 +72,8 @@ public class CommonNodeLabelsManager extends AbstractService {
|
|||
|
||||
protected Dispatcher dispatcher;
|
||||
|
||||
protected ConcurrentMap<String, Label> labelCollections =
|
||||
new ConcurrentHashMap<String, Label>();
|
||||
protected ConcurrentMap<String, NodeLabel> labelCollections =
|
||||
new ConcurrentHashMap<String, NodeLabel>();
|
||||
protected ConcurrentMap<String, Host> nodeCollections =
|
||||
new ConcurrentHashMap<String, Host>();
|
||||
|
||||
|
|
@ -82,19 +82,6 @@ public class CommonNodeLabelsManager extends AbstractService {
|
|||
|
||||
protected NodeLabelsStore store;
|
||||
|
||||
protected static class Label {
|
||||
private Resource resource;
|
||||
|
||||
protected Label() {
|
||||
this.resource = Resource.newInstance(0, 0);
|
||||
}
|
||||
|
||||
public Resource getResource() {
|
||||
return this.resource;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* A <code>Host</code> can have multiple <code>Node</code>s
|
||||
*/
|
||||
|
|
@ -201,7 +188,7 @@ public class CommonNodeLabelsManager extends AbstractService {
|
|||
protected void serviceInit(Configuration conf) throws Exception {
|
||||
initNodeLabelStore(conf);
|
||||
|
||||
labelCollections.put(NO_LABEL, new Label());
|
||||
labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL));
|
||||
}
|
||||
|
||||
protected void initNodeLabelStore(Configuration conf) throws Exception {
|
||||
|
|
@ -271,7 +258,7 @@ public class CommonNodeLabelsManager extends AbstractService {
|
|||
for (String label : labels) {
|
||||
// shouldn't overwrite it to avoid changing the Label.resource
|
||||
if (this.labelCollections.get(label) == null) {
|
||||
this.labelCollections.put(label, new Label());
|
||||
this.labelCollections.put(label, new NodeLabel(label));
|
||||
newLabels.add(label);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.nodelabels;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||
|
||||
public class NodeLabel implements Comparable<NodeLabel> {
|
||||
private Resource resource;
|
||||
private int numActiveNMs;
|
||||
private String labelName;
|
||||
|
||||
public NodeLabel(String labelName) {
|
||||
this(labelName, Resource.newInstance(0, 0), 0);
|
||||
}
|
||||
|
||||
protected NodeLabel(String labelName, Resource res, int activeNMs) {
|
||||
this.labelName = labelName;
|
||||
this.resource = res;
|
||||
this.numActiveNMs = activeNMs;
|
||||
}
|
||||
|
||||
public void addNode(Resource nodeRes) {
|
||||
Resources.addTo(resource, nodeRes);
|
||||
numActiveNMs++;
|
||||
}
|
||||
|
||||
public void removeNode(Resource nodeRes) {
|
||||
Resources.subtractFrom(resource, nodeRes);
|
||||
numActiveNMs--;
|
||||
}
|
||||
|
||||
public Resource getResource() {
|
||||
return this.resource;
|
||||
}
|
||||
|
||||
public int getNumActiveNMs() {
|
||||
return numActiveNMs;
|
||||
}
|
||||
|
||||
public String getLabelName() {
|
||||
return labelName;
|
||||
}
|
||||
|
||||
public NodeLabel getCopy() {
|
||||
return new NodeLabel(labelName, resource, numActiveNMs);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(NodeLabel o) {
|
||||
// We should always put empty label entry first after sorting
|
||||
if (labelName.isEmpty() != o.getLabelName().isEmpty()) {
|
||||
if (labelName.isEmpty()) {
|
||||
return -1;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
return labelName.compareTo(o.getLabelName());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (obj instanceof NodeLabel) {
|
||||
NodeLabel other = (NodeLabel) obj;
|
||||
return Resources.equals(resource, other.getResource())
|
||||
&& StringUtils.equals(labelName, other.getLabelName())
|
||||
&& (other.getNumActiveNMs() == numActiveNMs);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
final int prime = 502357;
|
||||
return (int) ((((long) labelName.hashCode() << 8)
|
||||
+ (resource.hashCode() << 4) + numActiveNMs) % prime);
|
||||
}
|
||||
}
|
||||
|
|
@ -32,4 +32,5 @@ public interface YarnWebParams {
|
|||
String APP_STATE = "app.state";
|
||||
String QUEUE_NAME = "queue.name";
|
||||
String NODE_STATE = "node.state";
|
||||
String NODE_LABEL = "node.label";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,10 +19,12 @@
|
|||
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
|
|
@ -37,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
|
|||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.event.Dispatcher;
|
||||
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
||||
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent;
|
||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||
|
|
@ -360,8 +363,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
|
|||
// no label in the past
|
||||
if (oldLabels.isEmpty()) {
|
||||
// update labels
|
||||
Label label = labelCollections.get(NO_LABEL);
|
||||
Resources.subtractFrom(label.getResource(), oldNM.resource);
|
||||
NodeLabel label = labelCollections.get(NO_LABEL);
|
||||
label.removeNode(oldNM.resource);
|
||||
|
||||
// update queues, all queue can access this node
|
||||
for (Queue q : queueCollections.values()) {
|
||||
|
|
@ -370,11 +373,11 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
|
|||
} else {
|
||||
// update labels
|
||||
for (String labelName : oldLabels) {
|
||||
Label label = labelCollections.get(labelName);
|
||||
NodeLabel label = labelCollections.get(labelName);
|
||||
if (null == label) {
|
||||
continue;
|
||||
}
|
||||
Resources.subtractFrom(label.getResource(), oldNM.resource);
|
||||
label.removeNode(oldNM.resource);
|
||||
}
|
||||
|
||||
// update queues, only queue can access this node will be subtract
|
||||
|
|
@ -395,8 +398,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
|
|||
// no label in the past
|
||||
if (newLabels.isEmpty()) {
|
||||
// update labels
|
||||
Label label = labelCollections.get(NO_LABEL);
|
||||
Resources.addTo(label.getResource(), newNM.resource);
|
||||
NodeLabel label = labelCollections.get(NO_LABEL);
|
||||
label.addNode(newNM.resource);
|
||||
|
||||
// update queues, all queue can access this node
|
||||
for (Queue q : queueCollections.values()) {
|
||||
|
|
@ -405,8 +408,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
|
|||
} else {
|
||||
// update labels
|
||||
for (String labelName : newLabels) {
|
||||
Label label = labelCollections.get(labelName);
|
||||
Resources.addTo(label.getResource(), newNM.resource);
|
||||
NodeLabel label = labelCollections.get(labelName);
|
||||
label.addNode(newNM.resource);
|
||||
}
|
||||
|
||||
// update queues, only queue can access this node will be subtract
|
||||
|
|
@ -475,4 +478,21 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
|
|||
public void setRMContext(RMContext rmContext) {
|
||||
this.rmContext = rmContext;
|
||||
}
|
||||
|
||||
public List<NodeLabel> pullRMNodeLabelsInfo() {
|
||||
try {
|
||||
readLock.lock();
|
||||
List<NodeLabel> infos = new ArrayList<NodeLabel>();
|
||||
|
||||
for (Entry<String, NodeLabel> entry : labelCollections.entrySet()) {
|
||||
NodeLabel label = entry.getValue();
|
||||
infos.add(label.getCopy());
|
||||
}
|
||||
|
||||
Collections.sort(infos);
|
||||
return infos;
|
||||
} finally {
|
||||
readLock.unlock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,7 +33,8 @@ public class NavBlock extends HtmlBlock {
|
|||
h3("Cluster").
|
||||
ul().
|
||||
li().a(url("cluster"), "About")._().
|
||||
li().a(url("nodes"), "Nodes")._();
|
||||
li().a(url("nodes"), "Nodes")._().
|
||||
li().a(url("nodelabels"), "Node Labels")._();
|
||||
UL<LI<UL<DIV<Hamlet>>>> subAppsList = mainList.
|
||||
li().a(url("apps"), "Applications").
|
||||
ul();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
||||
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
|
||||
|
||||
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||
import org.apache.hadoop.yarn.webapp.SubView;
|
||||
import org.apache.hadoop.yarn.webapp.YarnWebParams;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
|
||||
public class NodeLabelsPage extends RmView {
|
||||
static class NodeLabelsBlock extends HtmlBlock {
|
||||
final ResourceManager rm;
|
||||
|
||||
@Inject
|
||||
NodeLabelsBlock(ResourceManager rm, ViewContext ctx) {
|
||||
super(ctx);
|
||||
this.rm = rm;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void render(Block html) {
|
||||
TBODY<TABLE<Hamlet>> tbody = html.table("#nodelabels").
|
||||
thead().
|
||||
tr().
|
||||
th(".name", "Label Name").
|
||||
th(".numOfActiveNMs", "Num Of Active NMs").
|
||||
th(".totalResource", "Total Resource").
|
||||
_()._().
|
||||
tbody();
|
||||
|
||||
RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager();
|
||||
for (NodeLabel info : nlm.pullRMNodeLabelsInfo()) {
|
||||
TR<TBODY<TABLE<Hamlet>>> row =
|
||||
tbody.tr().td(
|
||||
info.getLabelName().isEmpty() ? "<NO_LABEL>" : info
|
||||
.getLabelName());
|
||||
int nActiveNMs = info.getNumActiveNMs();
|
||||
if (nActiveNMs > 0) {
|
||||
row = row.td()
|
||||
.a(url("nodes",
|
||||
"?" + YarnWebParams.NODE_LABEL + "=" + info.getLabelName()),
|
||||
String.valueOf(nActiveNMs))
|
||||
._();
|
||||
} else {
|
||||
row = row.td(String.valueOf(nActiveNMs));
|
||||
}
|
||||
row.td(info.getResource().toString())._();
|
||||
}
|
||||
tbody._()._();
|
||||
}
|
||||
}
|
||||
|
||||
@Override protected void preHead(Page.HTML<_> html) {
|
||||
commonPreHead(html);
|
||||
String title = "Node labels of the cluster";
|
||||
setTitle(title);
|
||||
set(DATATABLES_ID, "nodelabels");
|
||||
setTableStyles(html, "nodelabels", ".healthStatus {width:10em}",
|
||||
".healthReport {width:10em}");
|
||||
}
|
||||
|
||||
@Override protected Class<? extends SubView> content() {
|
||||
return NodeLabelsBlock.class;
|
||||
}
|
||||
}
|
||||
|
|
@ -18,7 +18,8 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
||||
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp.NODE_STATE;
|
||||
import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
|
||||
import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
|
||||
|
|
@ -28,7 +29,9 @@ import java.util.Collection;
|
|||
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.api.records.NodeState;
|
||||
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
|
||||
|
|
@ -60,24 +63,18 @@ class NodesPage extends RmView {
|
|||
|
||||
ResourceScheduler sched = rm.getResourceScheduler();
|
||||
String type = $(NODE_STATE);
|
||||
TBODY<TABLE<Hamlet>> tbody = html.table("#nodes").
|
||||
thead().
|
||||
tr().
|
||||
th(".nodelabels", "Node Labels").
|
||||
th(".rack", "Rack").
|
||||
th(".state", "Node State").
|
||||
th(".nodeaddress", "Node Address").
|
||||
th(".nodehttpaddress", "Node HTTP Address").
|
||||
th(".lastHealthUpdate", "Last health-update").
|
||||
th(".healthReport", "Health-report").
|
||||
th(".containers", "Containers").
|
||||
th(".mem", "Mem Used").
|
||||
th(".mem", "Mem Avail").
|
||||
th(".vcores", "VCores Used").
|
||||
th(".vcores", "VCores Avail").
|
||||
th(".nodeManagerVersion", "Version").
|
||||
_()._().
|
||||
tbody();
|
||||
String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim();
|
||||
TBODY<TABLE<Hamlet>> tbody =
|
||||
html.table("#nodes").thead().tr().th(".nodelabels", "Node Labels")
|
||||
.th(".rack", "Rack").th(".state", "Node State")
|
||||
.th(".nodeaddress", "Node Address")
|
||||
.th(".nodehttpaddress", "Node HTTP Address")
|
||||
.th(".lastHealthUpdate", "Last health-update")
|
||||
.th(".healthReport", "Health-report")
|
||||
.th(".containers", "Containers").th(".mem", "Mem Used")
|
||||
.th(".mem", "Mem Avail").th(".vcores", "VCores Used")
|
||||
.th(".vcores", "VCores Avail")
|
||||
.th(".nodeManagerVersion", "Version")._()._().tbody();
|
||||
NodeState stateFilter = null;
|
||||
if (type != null && !type.isEmpty()) {
|
||||
stateFilter = NodeState.valueOf(type.toUpperCase());
|
||||
|
|
@ -109,39 +106,48 @@ class NodesPage extends RmView {
|
|||
continue;
|
||||
}
|
||||
}
|
||||
// Besides state, we need to filter label as well.
|
||||
if (!labelFilter.equals(RMNodeLabelsManager.ANY)) {
|
||||
if (labelFilter.isEmpty()) {
|
||||
// Empty label filter means only shows nodes without label
|
||||
if (!ni.getNodeLabels().isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
} else if (!ni.getNodeLabels().contains(labelFilter)) {
|
||||
// Only nodes have given label can show on web page.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
NodeInfo info = new NodeInfo(ni, sched);
|
||||
int usedMemory = (int) info.getUsedMemory();
|
||||
int availableMemory = (int) info.getAvailableMemory();
|
||||
TR<TBODY<TABLE<Hamlet>>> row = tbody.tr().
|
||||
td(StringUtils.join(",", info.getNodeLabels())).
|
||||
td(info.getRack()).
|
||||
td(info.getState()).
|
||||
td(info.getNodeId());
|
||||
TR<TBODY<TABLE<Hamlet>>> row =
|
||||
tbody.tr().td(StringUtils.join(",", info.getNodeLabels()))
|
||||
.td(info.getRack()).td(info.getState()).td(info.getNodeId());
|
||||
if (isInactive) {
|
||||
row.td()._("N/A")._();
|
||||
} else {
|
||||
String httpAddress = info.getNodeHTTPAddress();
|
||||
row.td().a("//" + httpAddress,
|
||||
httpAddress)._();
|
||||
row.td().a("//" + httpAddress, httpAddress)._();
|
||||
}
|
||||
row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._().
|
||||
_(Times.format(info.getLastHealthUpdate()))._().
|
||||
td(info.getHealthReport()).
|
||||
td(String.valueOf(info.getNumContainers())).
|
||||
td().br().$title(String.valueOf(usedMemory))._().
|
||||
_(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
|
||||
td().br().$title(String.valueOf(availableMemory))._().
|
||||
_(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
|
||||
td(String.valueOf(info.getUsedVirtualCores())).
|
||||
td(String.valueOf(info.getAvailableVirtualCores())).
|
||||
td(ni.getNodeManagerVersion()).
|
||||
_();
|
||||
row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._()
|
||||
._(Times.format(info.getLastHealthUpdate()))._()
|
||||
.td(info.getHealthReport())
|
||||
.td(String.valueOf(info.getNumContainers())).td().br()
|
||||
.$title(String.valueOf(usedMemory))._()
|
||||
._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br()
|
||||
.$title(String.valueOf(availableMemory))._()
|
||||
._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._()
|
||||
.td(String.valueOf(info.getUsedVirtualCores()))
|
||||
.td(String.valueOf(info.getAvailableVirtualCores()))
|
||||
.td(ni.getNodeManagerVersion())._();
|
||||
}
|
||||
tbody._()._();
|
||||
}
|
||||
}
|
||||
|
||||
@Override protected void preHead(Page.HTML<_> html) {
|
||||
@Override
|
||||
protected void preHead(Page.HTML<_> html) {
|
||||
commonPreHead(html);
|
||||
String type = $(NODE_STATE);
|
||||
String title = "Nodes of the cluster";
|
||||
|
|
@ -155,15 +161,16 @@ class NodesPage extends RmView {
|
|||
".healthReport {width:10em}");
|
||||
}
|
||||
|
||||
@Override protected Class<? extends SubView> content() {
|
||||
@Override
|
||||
protected Class<? extends SubView> content() {
|
||||
return NodesBlock.class;
|
||||
}
|
||||
|
||||
private String nodesTableInit() {
|
||||
StringBuilder b = tableInit().append(", aoColumnDefs: [");
|
||||
b.append("{'bSearchable': false, 'aTargets': [ 6 ]}");
|
||||
b.append(", {'sType': 'title-numeric', 'bSearchable': false, " +
|
||||
"'aTargets': [ 7, 8 ] }");
|
||||
b.append(", {'sType': 'title-numeric', 'bSearchable': false, "
|
||||
+ "'aTargets': [ 7, 8 ] }");
|
||||
b.append(", {'sType': 'title-numeric', 'aTargets': [ 4 ]}");
|
||||
b.append("]}");
|
||||
return b.toString();
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ public class RMWebApp extends WebApp implements YarnWebParams {
|
|||
route(pajoin("/app", APPLICATION_ID), RmController.class, "app");
|
||||
route("/scheduler", RmController.class, "scheduler");
|
||||
route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue");
|
||||
route("/nodelabels", RmController.class, "nodelabels");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||
import org.apache.hadoop.yarn.util.StringHelper;
|
||||
import org.apache.hadoop.yarn.webapp.Controller;
|
||||
import org.apache.hadoop.yarn.webapp.WebAppException;
|
||||
import org.apache.hadoop.yarn.webapp.YarnWebParams;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
|
|
@ -93,4 +92,9 @@ public class RmController extends Controller {
|
|||
public void submit() {
|
||||
setTitle("Application Submission Not Allowed");
|
||||
}
|
||||
|
||||
public void nodelabels() {
|
||||
setTitle("Node Labels");
|
||||
render(NodeLabelsPage.class);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,11 +30,13 @@ import org.apache.hadoop.yarn.api.records.NodeState;
|
|||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
||||
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
/**
|
||||
|
|
@ -53,9 +55,14 @@ public class MockNodes {
|
|||
// One unhealthy node per rack.
|
||||
list.add(nodeInfo(i, perNode, NodeState.UNHEALTHY));
|
||||
}
|
||||
if (j == 0) {
|
||||
// One node with label
|
||||
list.add(nodeInfo(i, perNode, NodeState.RUNNING, ImmutableSet.of("x")));
|
||||
} else {
|
||||
list.add(newNodeInfo(i, perNode));
|
||||
}
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
|
|
@ -100,10 +107,12 @@ public class MockNodes {
|
|||
private String healthReport;
|
||||
private long lastHealthReportTime;
|
||||
private NodeState state;
|
||||
private Set<String> labels;
|
||||
|
||||
public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
|
||||
Resource perNode, String rackName, String healthReport,
|
||||
long lastHealthReportTime, int cmdPort, String hostName, NodeState state) {
|
||||
long lastHealthReportTime, int cmdPort, String hostName, NodeState state,
|
||||
Set<String> labels) {
|
||||
this.nodeId = nodeId;
|
||||
this.nodeAddr = nodeAddr;
|
||||
this.httpAddress = httpAddress;
|
||||
|
|
@ -114,6 +123,7 @@ public class MockNodes {
|
|||
this.cmdPort = cmdPort;
|
||||
this.hostName = hostName;
|
||||
this.state = state;
|
||||
this.labels = labels;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -207,16 +217,33 @@ public class MockNodes {
|
|||
|
||||
@Override
|
||||
public Set<String> getNodeLabels() {
|
||||
return RMNodeLabelsManager.EMPTY_STRING_SET;
|
||||
if (labels != null) {
|
||||
return labels;
|
||||
}
|
||||
return CommonNodeLabelsManager.EMPTY_STRING_SET;
|
||||
}
|
||||
};
|
||||
|
||||
private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) {
|
||||
return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123);
|
||||
private static RMNode buildRMNode(int rack, final Resource perNode,
|
||||
NodeState state, String httpAddr) {
|
||||
return buildRMNode(rack, perNode, state, httpAddr, null);
|
||||
}
|
||||
|
||||
private static RMNode buildRMNode(int rack, final Resource perNode,
|
||||
NodeState state, String httpAddr, Set<String> labels) {
|
||||
return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123,
|
||||
labels);
|
||||
}
|
||||
|
||||
private static RMNode buildRMNode(int rack, final Resource perNode,
|
||||
NodeState state, String httpAddr, int hostnum, String hostName, int port) {
|
||||
return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port,
|
||||
null);
|
||||
}
|
||||
|
||||
private static RMNode buildRMNode(int rack, final Resource perNode,
|
||||
NodeState state, String httpAddr, int hostnum, String hostName, int port,
|
||||
Set<String> labels) {
|
||||
final String rackName = "rack"+ rack;
|
||||
final int nid = hostnum;
|
||||
final String nodeAddr = hostName + ":" + nid;
|
||||
|
|
@ -228,7 +255,7 @@ public class MockNodes {
|
|||
final String httpAddress = httpAddr;
|
||||
String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
|
||||
return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode,
|
||||
rackName, healthReport, 0, nid, hostName, state);
|
||||
rackName, healthReport, 0, nid, hostName, state, labels);
|
||||
}
|
||||
|
||||
public static RMNode nodeInfo(int rack, final Resource perNode,
|
||||
|
|
@ -236,6 +263,11 @@ public class MockNodes {
|
|||
return buildRMNode(rack, perNode, state, "N/A");
|
||||
}
|
||||
|
||||
public static RMNode nodeInfo(int rack, final Resource perNode,
|
||||
NodeState state, Set<String> labels) {
|
||||
return buildRMNode(rack, perNode, state, "N/A", labels);
|
||||
}
|
||||
|
||||
public static RMNode newNodeInfo(int rack, final Resource perNode) {
|
||||
return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -839,7 +839,7 @@ public class TestWorkPreservingRMRestart {
|
|||
// Test if RM on recovery receives the container release request from AM
|
||||
// before it receives the container status reported by NM for recovery. this
|
||||
// container should not be recovered.
|
||||
@Test (timeout = 30000)
|
||||
@Test (timeout = 50000)
|
||||
public void testReleasedContainerNotRecovered() throws Exception {
|
||||
MemoryRMStateStore memStore = new MemoryRMStateStore();
|
||||
memStore.init(conf);
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
|
|
@ -27,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
|
|||
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
||||
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
|
||||
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
|
||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||
import org.junit.After;
|
||||
|
|
@ -428,4 +430,35 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
|
|||
Assert.fail("IOException from removeLabelsFromNode " + e);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkNodeLabelInfo(List<NodeLabel> infos, String labelName, int activeNMs, int memory) {
|
||||
for (NodeLabel info : infos) {
|
||||
if (info.getLabelName().equals(labelName)) {
|
||||
Assert.assertEquals(activeNMs, info.getNumActiveNMs());
|
||||
Assert.assertEquals(memory, info.getResource().getMemory());
|
||||
return;
|
||||
}
|
||||
}
|
||||
Assert.fail("Failed to find info has label=" + labelName);
|
||||
}
|
||||
|
||||
@Test(timeout = 5000)
|
||||
public void testPullRMNodeLabelsInfo() throws IOException {
|
||||
mgr.addToCluserNodeLabels(toSet("x", "y", "z"));
|
||||
mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0));
|
||||
mgr.activateNode(NodeId.newInstance("n2", 1), Resource.newInstance(10, 0));
|
||||
mgr.activateNode(NodeId.newInstance("n3", 1), Resource.newInstance(10, 0));
|
||||
mgr.activateNode(NodeId.newInstance("n4", 1), Resource.newInstance(10, 0));
|
||||
mgr.activateNode(NodeId.newInstance("n5", 1), Resource.newInstance(10, 0));
|
||||
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("x"),
|
||||
toNodeId("n2"), toSet("x"), toNodeId("n3"), toSet("y")));
|
||||
|
||||
// x, y, z and ""
|
||||
List<NodeLabel> infos = mgr.pullRMNodeLabelsInfo();
|
||||
Assert.assertEquals(4, infos.size());
|
||||
checkNodeLabelInfo(infos, RMNodeLabelsManager.NO_LABEL, 2, 20);
|
||||
checkNodeLabelInfo(infos, "x", 2, 20);
|
||||
checkNodeLabelInfo(infos, "y", 1, 10);
|
||||
checkNodeLabelInfo(infos, "z", 0, 0);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -106,4 +106,49 @@ public class TestNodesPage {
|
|||
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
|
||||
"<td");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNodesBlockRenderForNodeLabelFilterWithNonEmptyLabel() {
|
||||
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
|
||||
nodesBlock.set("node.label", "x");
|
||||
nodesBlock.render();
|
||||
PrintWriter writer = injector.getInstance(PrintWriter.class);
|
||||
WebAppTests.flushOutput(injector);
|
||||
|
||||
Mockito.verify(
|
||||
writer,
|
||||
Mockito.times(numberOfRacks
|
||||
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
|
||||
"<td");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNodesBlockRenderForNodeLabelFilterWithEmptyLabel() {
|
||||
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
|
||||
nodesBlock.set("node.label", "");
|
||||
nodesBlock.render();
|
||||
PrintWriter writer = injector.getInstance(PrintWriter.class);
|
||||
WebAppTests.flushOutput(injector);
|
||||
|
||||
Mockito.verify(
|
||||
writer,
|
||||
Mockito.times(numberOfRacks * (numberOfNodesPerRack - 1)
|
||||
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
|
||||
"<td");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNodesBlockRenderForNodeLabelFilterWithAnyLabel() {
|
||||
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);
|
||||
nodesBlock.set("node.label", "*");
|
||||
nodesBlock.render();
|
||||
PrintWriter writer = injector.getInstance(PrintWriter.class);
|
||||
WebAppTests.flushOutput(injector);
|
||||
|
||||
Mockito.verify(
|
||||
writer,
|
||||
Mockito.times(numberOfRacks * numberOfNodesPerRack
|
||||
* numberOfActualTableHeaders + numberOfThInMetricsTable)).print(
|
||||
"<td");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue