MAPREDUCE-2690. Web-page for FifoScheduler. Contributed by Eric Payne.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1166958 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy
parent
fafe8cd28e
commit
035937bee8
|
|
@ -255,6 +255,8 @@ Release 0.23.0 - Unreleased
|
|||
MAPREDUCE-2864. Normalize configuration variable names for YARN. (Robert
|
||||
Evans via acmurthy)
|
||||
|
||||
MAPREDUCE-2690. Web-page for FifoScheduler. (Eric Payne via acmurthy)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateS
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
|
||||
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
|
||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
||||
import org.apache.hadoop.yarn.api.records.QueueState;
|
||||
|
||||
@LimitedPrivate("yarn")
|
||||
@Evolving
|
||||
|
|
@ -144,6 +145,7 @@ public class FifoScheduler implements ResourceScheduler {
|
|||
queueInfo.setCapacity(100.0f);
|
||||
queueInfo.setMaximumCapacity(100.0f);
|
||||
queueInfo.setChildQueues(new ArrayList<QueueInfo>());
|
||||
queueInfo.setQueueState(QueueState.RUNNING);
|
||||
return queueInfo;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,20 +18,177 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
||||
|
||||
import org.apache.hadoop.yarn.webapp.SubView;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
import com.google.inject.Inject;
|
||||
import com.google.inject.servlet.RequestScoped;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
|
||||
import org.apache.hadoop.yarn.webapp.SubView;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.*;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
||||
import org.apache.hadoop.yarn.api.records.QueueState;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
|
||||
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
|
||||
|
||||
import static org.apache.hadoop.yarn.util.StringHelper.*;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
|
||||
|
||||
class DefaultSchedulerPage extends RmView {
|
||||
static final String _Q = ".ui-state-default.ui-corner-all";
|
||||
static final float WIDTH_F = 0.8f;
|
||||
static final String Q_END = "left:101%";
|
||||
static final String OVER = "font-size:1px;background:rgba(255, 140, 0, 0.8)";
|
||||
static final String UNDER = "font-size:1px;background:rgba(50, 205, 50, 0.8)";
|
||||
static final float EPSILON = 1e-8f;
|
||||
|
||||
static class QueueBlock extends HtmlBlock {
|
||||
@Override public void render(Block html) {
|
||||
html.h2("Under construction");
|
||||
static class QueueInfoBlock extends HtmlBlock {
|
||||
final RMContext rmContext;
|
||||
final FifoScheduler fs;
|
||||
final String qName;
|
||||
final QueueInfo qInfo;
|
||||
|
||||
@Inject QueueInfoBlock(RMContext context, ViewContext ctx, ResourceManager rm) {
|
||||
super(ctx);
|
||||
this.rmContext = context;
|
||||
|
||||
fs = (FifoScheduler) rm.getResourceScheduler();
|
||||
qName = fs.getQueueInfo("",false,false).getQueueName();
|
||||
qInfo = fs.getQueueInfo(qName,true,true);
|
||||
}
|
||||
|
||||
@Override public void render(Block html) {
|
||||
String minmemoryresource =
|
||||
Integer.toString(fs.getMinimumResourceCapability().getMemory());
|
||||
String maxmemoryresource =
|
||||
Integer.toString(fs.getMaximumResourceCapability().getMemory());
|
||||
String qstate = (qInfo.getQueueState() == QueueState.RUNNING) ?
|
||||
"Running" :
|
||||
(qInfo.getQueueState() == QueueState.STOPPED) ?
|
||||
"Stopped" : "Unknown";
|
||||
|
||||
int usedNodeMem = 0;
|
||||
int availNodeMem = 0;
|
||||
int totNodeMem = 0;
|
||||
int nodeContainers = 0;
|
||||
|
||||
for (RMNode ni : this.rmContext.getRMNodes().values()) {
|
||||
usedNodeMem += fs.getUsedResource(ni.getNodeID()).getMemory();
|
||||
availNodeMem += fs.getAvailableResource(ni.getNodeID()).getMemory();
|
||||
totNodeMem += ni.getTotalCapability().getMemory();
|
||||
nodeContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
|
||||
}
|
||||
|
||||
info("\'" + qName + "\' Queue Status").
|
||||
_("Queue State:" , qstate).
|
||||
_("Minimum Queue Memory Capacity:" , minmemoryresource).
|
||||
_("Maximum Queue Memory Capacity:" , maxmemoryresource).
|
||||
_("Number of Nodes:" , Integer.toString(this.rmContext.getRMNodes().size())).
|
||||
_("Used Node Capacity:" , Integer.toString(usedNodeMem)).
|
||||
_("Available Node Capacity:" , Integer.toString(availNodeMem)).
|
||||
_("Total Node Capacity:" , Integer.toString(totNodeMem)).
|
||||
_("Number of Node Containers:" , Integer.toString(nodeContainers));
|
||||
|
||||
html._(InfoBlock.class);
|
||||
}
|
||||
}
|
||||
|
||||
static class QueuesBlock extends HtmlBlock {
|
||||
final FifoScheduler fs;
|
||||
final String qName;
|
||||
final QueueInfo qInfo;
|
||||
|
||||
@Inject QueuesBlock(ResourceManager rm) {
|
||||
fs = (FifoScheduler) rm.getResourceScheduler();
|
||||
qName = fs.getQueueInfo("",false,false).getQueueName();
|
||||
qInfo = fs.getQueueInfo(qName,false,false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void render(Block html) {
|
||||
UL<DIV<DIV<Hamlet>>> ul = html.
|
||||
div("#cs-wrapper.ui-widget").
|
||||
div(".ui-widget-header.ui-corner-top").
|
||||
_("FifoScheduler Queue")._().
|
||||
div("#cs.ui-widget-content.ui-corner-bottom").
|
||||
ul();
|
||||
|
||||
if (fs == null) {
|
||||
ul.
|
||||
li().
|
||||
a(_Q).$style(width(WIDTH_F)).
|
||||
span().$style(Q_END)._("100% ")._().
|
||||
span(".q", "default")._()._();
|
||||
} else {
|
||||
float used = qInfo.getCurrentCapacity() / 100.0f;
|
||||
float set = qInfo.getCapacity() / 100.0f;
|
||||
float delta = Math.abs(set - used) + 0.001f;
|
||||
ul.
|
||||
li().
|
||||
a(_Q).$style(width(WIDTH_F)).
|
||||
$title(join("used:", percent(used))).
|
||||
span().$style(Q_END)._("100%")._().
|
||||
span().$style(join(width(delta), ';', used > set ? OVER : UNDER,
|
||||
';', used > set ? left(set) : left(used)))._(".")._().
|
||||
span(".q", qName)._().
|
||||
_(QueueInfoBlock.class)._();
|
||||
}
|
||||
|
||||
ul._()._().
|
||||
script().$type("text/javascript").
|
||||
_("$('#cs').hide();")._()._().
|
||||
_(AppsBlock.class);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override protected void postHead(Page.HTML<_> html) {
|
||||
html.
|
||||
style().$type("text/css").
|
||||
_("#cs { padding: 0.5em 0 1em 0; margin-bottom: 1em; position: relative }",
|
||||
"#cs ul { list-style: none }",
|
||||
"#cs a { font-weight: normal; margin: 2px; position: relative }",
|
||||
"#cs a span { font-weight: normal; font-size: 80% }",
|
||||
"#cs-wrapper .ui-widget-header { padding: 0.2em 0.5em }",
|
||||
"table.info tr th {width: 50%}")._(). // to center info table
|
||||
script("/static/jt/jquery.jstree.js").
|
||||
script().$type("text/javascript").
|
||||
_("$(function() {",
|
||||
" $('#cs a span').addClass('ui-corner-all').css('position', 'absolute');",
|
||||
" $('#cs').bind('loaded.jstree', function (e, data) {",
|
||||
" data.inst.open_all(); }).",
|
||||
" jstree({",
|
||||
" core: { animation: 188, html_titles: true },",
|
||||
" plugins: ['themeroller', 'html_data', 'ui'],",
|
||||
" themeroller: { item_open: 'ui-icon-minus',",
|
||||
" item_clsd: 'ui-icon-plus', item_leaf: 'ui-icon-gear'",
|
||||
" }",
|
||||
" });",
|
||||
" $('#cs').bind('select_node.jstree', function(e, data) {",
|
||||
" var q = $('.q', data.rslt.obj).first().text();",
|
||||
" if (q == 'root') q = '';",
|
||||
" $('#apps').dataTable().fnFilter(q, 3);",
|
||||
" });",
|
||||
" $('#cs').show();",
|
||||
"});")._();
|
||||
}
|
||||
|
||||
@Override protected Class<? extends SubView> content() {
|
||||
return QueueBlock.class;
|
||||
return QueuesBlock.class;
|
||||
}
|
||||
|
||||
static String percent(float f) {
|
||||
return String.format("%.1f%%", f * 100);
|
||||
}
|
||||
|
||||
static String width(float f) {
|
||||
return String.format("width:%.1f%%", f * 100);
|
||||
}
|
||||
|
||||
static String left(float f) {
|
||||
return String.format("left:%.1f%%", f * 100);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
|
||||
import org.apache.hadoop.yarn.webapp.WebApps;
|
||||
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
|
||||
import org.junit.Test;
|
||||
|
|
@ -168,9 +169,38 @@ public class TestRMWebApp {
|
|||
conf.setCapacity(C13, 40);
|
||||
}
|
||||
|
||||
public static ResourceManager mockFifoRm(int apps, int racks, int nodes,
|
||||
int mbsPerNode)
|
||||
throws Exception {
|
||||
ResourceManager rm = mock(ResourceManager.class);
|
||||
RMContext rmContext = mockRMContext(apps, racks, nodes,
|
||||
mbsPerNode);
|
||||
ResourceScheduler rs = mockFifoScheduler();
|
||||
when(rm.getResourceScheduler()).thenReturn(rs);
|
||||
when(rm.getRMContext()).thenReturn(rmContext);
|
||||
return rm;
|
||||
}
|
||||
|
||||
public static FifoScheduler mockFifoScheduler() throws Exception {
|
||||
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
|
||||
setupFifoQueueConfiguration(conf);
|
||||
|
||||
FifoScheduler fs = new FifoScheduler();
|
||||
fs.reinitialize(conf, null, null);
|
||||
return fs;
|
||||
}
|
||||
|
||||
static void setupFifoQueueConfiguration(CapacitySchedulerConfiguration conf) {
|
||||
// Define default queue
|
||||
conf.setQueues("default", new String[] {"default"});
|
||||
conf.setCapacity("default", 100);
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
// For manual testing
|
||||
WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode().
|
||||
start(new RMWebApp(mockRm(101, 8, 8, 8*GiB))).joinThread();
|
||||
WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode().
|
||||
start(new RMWebApp(mockFifoRm(10, 1, 4, 8*GiB))).joinThread();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue