MAPREDUCE-2690. Web-page for FifoScheduler. Contributed by Eric Payne.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1166958 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
fafe8cd28e
commit
035937bee8
|
@ -255,6 +255,8 @@ Release 0.23.0 - Unreleased
|
||||||
MAPREDUCE-2864. Normalize configuration variable names for YARN. (Robert
|
MAPREDUCE-2864. Normalize configuration variable names for YARN. (Robert
|
||||||
Evans via acmurthy)
|
Evans via acmurthy)
|
||||||
|
|
||||||
|
MAPREDUCE-2690. Web-page for FifoScheduler. (Eric Payne via acmurthy)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
|
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
|
||||||
|
|
|
@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateS
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
|
||||||
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
|
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
|
||||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
import org.apache.hadoop.yarn.util.BuilderUtils;
|
||||||
|
import org.apache.hadoop.yarn.api.records.QueueState;
|
||||||
|
|
||||||
@LimitedPrivate("yarn")
|
@LimitedPrivate("yarn")
|
||||||
@Evolving
|
@Evolving
|
||||||
|
@ -144,6 +145,7 @@ public class FifoScheduler implements ResourceScheduler {
|
||||||
queueInfo.setCapacity(100.0f);
|
queueInfo.setCapacity(100.0f);
|
||||||
queueInfo.setMaximumCapacity(100.0f);
|
queueInfo.setMaximumCapacity(100.0f);
|
||||||
queueInfo.setChildQueues(new ArrayList<QueueInfo>());
|
queueInfo.setChildQueues(new ArrayList<QueueInfo>());
|
||||||
|
queueInfo.setQueueState(QueueState.RUNNING);
|
||||||
return queueInfo;
|
return queueInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -18,20 +18,177 @@
|
||||||
|
|
||||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
||||||
|
|
||||||
import org.apache.hadoop.yarn.webapp.SubView;
|
import com.google.inject.Inject;
|
||||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
import com.google.inject.servlet.RequestScoped;
|
||||||
|
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
|
||||||
|
import org.apache.hadoop.yarn.webapp.SubView;
|
||||||
|
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
|
||||||
|
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.*;
|
||||||
|
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||||
|
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
||||||
|
import org.apache.hadoop.yarn.api.records.QueueState;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
|
||||||
|
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.yarn.util.StringHelper.*;
|
||||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
|
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
|
||||||
|
|
||||||
class DefaultSchedulerPage extends RmView {
|
class DefaultSchedulerPage extends RmView {
|
||||||
|
static final String _Q = ".ui-state-default.ui-corner-all";
|
||||||
|
static final float WIDTH_F = 0.8f;
|
||||||
|
static final String Q_END = "left:101%";
|
||||||
|
static final String OVER = "font-size:1px;background:rgba(255, 140, 0, 0.8)";
|
||||||
|
static final String UNDER = "font-size:1px;background:rgba(50, 205, 50, 0.8)";
|
||||||
|
static final float EPSILON = 1e-8f;
|
||||||
|
|
||||||
static class QueueBlock extends HtmlBlock {
|
static class QueueInfoBlock extends HtmlBlock {
|
||||||
@Override public void render(Block html) {
|
final RMContext rmContext;
|
||||||
html.h2("Under construction");
|
final FifoScheduler fs;
|
||||||
|
final String qName;
|
||||||
|
final QueueInfo qInfo;
|
||||||
|
|
||||||
|
@Inject QueueInfoBlock(RMContext context, ViewContext ctx, ResourceManager rm) {
|
||||||
|
super(ctx);
|
||||||
|
this.rmContext = context;
|
||||||
|
|
||||||
|
fs = (FifoScheduler) rm.getResourceScheduler();
|
||||||
|
qName = fs.getQueueInfo("",false,false).getQueueName();
|
||||||
|
qInfo = fs.getQueueInfo(qName,true,true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override public void render(Block html) {
|
||||||
|
String minmemoryresource =
|
||||||
|
Integer.toString(fs.getMinimumResourceCapability().getMemory());
|
||||||
|
String maxmemoryresource =
|
||||||
|
Integer.toString(fs.getMaximumResourceCapability().getMemory());
|
||||||
|
String qstate = (qInfo.getQueueState() == QueueState.RUNNING) ?
|
||||||
|
"Running" :
|
||||||
|
(qInfo.getQueueState() == QueueState.STOPPED) ?
|
||||||
|
"Stopped" : "Unknown";
|
||||||
|
|
||||||
|
int usedNodeMem = 0;
|
||||||
|
int availNodeMem = 0;
|
||||||
|
int totNodeMem = 0;
|
||||||
|
int nodeContainers = 0;
|
||||||
|
|
||||||
|
for (RMNode ni : this.rmContext.getRMNodes().values()) {
|
||||||
|
usedNodeMem += fs.getUsedResource(ni.getNodeID()).getMemory();
|
||||||
|
availNodeMem += fs.getAvailableResource(ni.getNodeID()).getMemory();
|
||||||
|
totNodeMem += ni.getTotalCapability().getMemory();
|
||||||
|
nodeContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
|
||||||
|
}
|
||||||
|
|
||||||
|
info("\'" + qName + "\' Queue Status").
|
||||||
|
_("Queue State:" , qstate).
|
||||||
|
_("Minimum Queue Memory Capacity:" , minmemoryresource).
|
||||||
|
_("Maximum Queue Memory Capacity:" , maxmemoryresource).
|
||||||
|
_("Number of Nodes:" , Integer.toString(this.rmContext.getRMNodes().size())).
|
||||||
|
_("Used Node Capacity:" , Integer.toString(usedNodeMem)).
|
||||||
|
_("Available Node Capacity:" , Integer.toString(availNodeMem)).
|
||||||
|
_("Total Node Capacity:" , Integer.toString(totNodeMem)).
|
||||||
|
_("Number of Node Containers:" , Integer.toString(nodeContainers));
|
||||||
|
|
||||||
|
html._(InfoBlock.class);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static class QueuesBlock extends HtmlBlock {
|
||||||
|
final FifoScheduler fs;
|
||||||
|
final String qName;
|
||||||
|
final QueueInfo qInfo;
|
||||||
|
|
||||||
|
@Inject QueuesBlock(ResourceManager rm) {
|
||||||
|
fs = (FifoScheduler) rm.getResourceScheduler();
|
||||||
|
qName = fs.getQueueInfo("",false,false).getQueueName();
|
||||||
|
qInfo = fs.getQueueInfo(qName,false,false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void render(Block html) {
|
||||||
|
UL<DIV<DIV<Hamlet>>> ul = html.
|
||||||
|
div("#cs-wrapper.ui-widget").
|
||||||
|
div(".ui-widget-header.ui-corner-top").
|
||||||
|
_("FifoScheduler Queue")._().
|
||||||
|
div("#cs.ui-widget-content.ui-corner-bottom").
|
||||||
|
ul();
|
||||||
|
|
||||||
|
if (fs == null) {
|
||||||
|
ul.
|
||||||
|
li().
|
||||||
|
a(_Q).$style(width(WIDTH_F)).
|
||||||
|
span().$style(Q_END)._("100% ")._().
|
||||||
|
span(".q", "default")._()._();
|
||||||
|
} else {
|
||||||
|
float used = qInfo.getCurrentCapacity() / 100.0f;
|
||||||
|
float set = qInfo.getCapacity() / 100.0f;
|
||||||
|
float delta = Math.abs(set - used) + 0.001f;
|
||||||
|
ul.
|
||||||
|
li().
|
||||||
|
a(_Q).$style(width(WIDTH_F)).
|
||||||
|
$title(join("used:", percent(used))).
|
||||||
|
span().$style(Q_END)._("100%")._().
|
||||||
|
span().$style(join(width(delta), ';', used > set ? OVER : UNDER,
|
||||||
|
';', used > set ? left(set) : left(used)))._(".")._().
|
||||||
|
span(".q", qName)._().
|
||||||
|
_(QueueInfoBlock.class)._();
|
||||||
|
}
|
||||||
|
|
||||||
|
ul._()._().
|
||||||
|
script().$type("text/javascript").
|
||||||
|
_("$('#cs').hide();")._()._().
|
||||||
|
_(AppsBlock.class);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override protected void postHead(Page.HTML<_> html) {
|
||||||
|
html.
|
||||||
|
style().$type("text/css").
|
||||||
|
_("#cs { padding: 0.5em 0 1em 0; margin-bottom: 1em; position: relative }",
|
||||||
|
"#cs ul { list-style: none }",
|
||||||
|
"#cs a { font-weight: normal; margin: 2px; position: relative }",
|
||||||
|
"#cs a span { font-weight: normal; font-size: 80% }",
|
||||||
|
"#cs-wrapper .ui-widget-header { padding: 0.2em 0.5em }",
|
||||||
|
"table.info tr th {width: 50%}")._(). // to center info table
|
||||||
|
script("/static/jt/jquery.jstree.js").
|
||||||
|
script().$type("text/javascript").
|
||||||
|
_("$(function() {",
|
||||||
|
" $('#cs a span').addClass('ui-corner-all').css('position', 'absolute');",
|
||||||
|
" $('#cs').bind('loaded.jstree', function (e, data) {",
|
||||||
|
" data.inst.open_all(); }).",
|
||||||
|
" jstree({",
|
||||||
|
" core: { animation: 188, html_titles: true },",
|
||||||
|
" plugins: ['themeroller', 'html_data', 'ui'],",
|
||||||
|
" themeroller: { item_open: 'ui-icon-minus',",
|
||||||
|
" item_clsd: 'ui-icon-plus', item_leaf: 'ui-icon-gear'",
|
||||||
|
" }",
|
||||||
|
" });",
|
||||||
|
" $('#cs').bind('select_node.jstree', function(e, data) {",
|
||||||
|
" var q = $('.q', data.rslt.obj).first().text();",
|
||||||
|
" if (q == 'root') q = '';",
|
||||||
|
" $('#apps').dataTable().fnFilter(q, 3);",
|
||||||
|
" });",
|
||||||
|
" $('#cs').show();",
|
||||||
|
"});")._();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override protected Class<? extends SubView> content() {
|
@Override protected Class<? extends SubView> content() {
|
||||||
return QueueBlock.class;
|
return QueuesBlock.class;
|
||||||
|
}
|
||||||
|
|
||||||
|
static String percent(float f) {
|
||||||
|
return String.format("%.1f%%", f * 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
static String width(float f) {
|
||||||
|
return String.format("width:%.1f%%", f * 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
static String left(float f) {
|
||||||
|
return String.format("left:%.1f%%", f * 100);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
|
||||||
import org.apache.hadoop.yarn.webapp.WebApps;
|
import org.apache.hadoop.yarn.webapp.WebApps;
|
||||||
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
|
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
@ -168,9 +169,38 @@ public class TestRMWebApp {
|
||||||
conf.setCapacity(C13, 40);
|
conf.setCapacity(C13, 40);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static ResourceManager mockFifoRm(int apps, int racks, int nodes,
|
||||||
|
int mbsPerNode)
|
||||||
|
throws Exception {
|
||||||
|
ResourceManager rm = mock(ResourceManager.class);
|
||||||
|
RMContext rmContext = mockRMContext(apps, racks, nodes,
|
||||||
|
mbsPerNode);
|
||||||
|
ResourceScheduler rs = mockFifoScheduler();
|
||||||
|
when(rm.getResourceScheduler()).thenReturn(rs);
|
||||||
|
when(rm.getRMContext()).thenReturn(rmContext);
|
||||||
|
return rm;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static FifoScheduler mockFifoScheduler() throws Exception {
|
||||||
|
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
|
||||||
|
setupFifoQueueConfiguration(conf);
|
||||||
|
|
||||||
|
FifoScheduler fs = new FifoScheduler();
|
||||||
|
fs.reinitialize(conf, null, null);
|
||||||
|
return fs;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void setupFifoQueueConfiguration(CapacitySchedulerConfiguration conf) {
|
||||||
|
// Define default queue
|
||||||
|
conf.setQueues("default", new String[] {"default"});
|
||||||
|
conf.setCapacity("default", 100);
|
||||||
|
}
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
public static void main(String[] args) throws Exception {
|
||||||
// For manual testing
|
// For manual testing
|
||||||
WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode().
|
WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode().
|
||||||
start(new RMWebApp(mockRm(101, 8, 8, 8*GiB))).joinThread();
|
start(new RMWebApp(mockRm(101, 8, 8, 8*GiB))).joinThread();
|
||||||
|
WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode().
|
||||||
|
start(new RMWebApp(mockFifoRm(10, 1, 4, 8*GiB))).joinThread();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue