YARN-254. Update fair scheduler web UI for hierarchical queues. (sandyr via tucu)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1423743 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
026873c155
commit
7b7fc7b788
|
@ -56,6 +56,9 @@ Release 2.0.3-alpha - Unreleased
|
|||
|
||||
YARN-129. Simplify classpath construction for mini YARN tests. (tomwhite)
|
||||
|
||||
YARN-254. Update fair scheduler web UI for hierarchical queues.
|
||||
(sandyr via tucu)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
@ -20,21 +20,21 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
|||
|
||||
import static org.apache.hadoop.yarn.util.StringHelper.join;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Collection;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerLeafQueueInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerQueueInfo;
|
||||
import org.apache.hadoop.yarn.webapp.ResponseInfo;
|
||||
import org.apache.hadoop.yarn.webapp.SubView;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.LI;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlPage.Page;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlPage._;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
import com.google.inject.servlet.RequestScoped;
|
||||
|
@ -50,16 +50,15 @@ public class FairSchedulerPage extends RmView {
|
|||
|
||||
@RequestScoped
|
||||
static class FSQInfo {
|
||||
FairSchedulerInfo fsinfo;
|
||||
FairSchedulerQueueInfo qinfo;
|
||||
}
|
||||
|
||||
static class QueueInfoBlock extends HtmlBlock {
|
||||
final FairSchedulerQueueInfo qinfo;
|
||||
static class LeafQueueBlock extends HtmlBlock {
|
||||
final FairSchedulerLeafQueueInfo qinfo;
|
||||
|
||||
@Inject QueueInfoBlock(ViewContext ctx, FSQInfo info) {
|
||||
@Inject LeafQueueBlock(ViewContext ctx, FSQInfo info) {
|
||||
super(ctx);
|
||||
qinfo = (FairSchedulerQueueInfo) info.qinfo;
|
||||
qinfo = (FairSchedulerLeafQueueInfo)info.qinfo;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -83,6 +82,47 @@ public class FairSchedulerPage extends RmView {
|
|||
}
|
||||
}
|
||||
|
||||
static class QueueBlock extends HtmlBlock {
|
||||
final FSQInfo fsqinfo;
|
||||
|
||||
@Inject QueueBlock(FSQInfo info) {
|
||||
fsqinfo = info;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void render(Block html) {
|
||||
Collection<FairSchedulerQueueInfo> subQueues = fsqinfo.qinfo.getChildQueues();
|
||||
UL<Hamlet> ul = html.ul("#pq");
|
||||
for (FairSchedulerQueueInfo info : subQueues) {
|
||||
float capacity = info.getMaxResourcesFraction();
|
||||
float fairShare = info.getFairShareFraction();
|
||||
float used = info.getUsedFraction();
|
||||
LI<UL<Hamlet>> li = ul.
|
||||
li().
|
||||
a(_Q).$style(width(capacity * Q_MAX_WIDTH)).
|
||||
$title(join("Fair Share:", percent(fairShare))).
|
||||
span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))).
|
||||
_('.')._().
|
||||
span().$style(join(width(used/capacity),
|
||||
";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)).
|
||||
_('.')._().
|
||||
span(".q", info.getQueueName())._().
|
||||
span().$class("qstats").$style(left(Q_STATS_POS)).
|
||||
_(join(percent(used), " used"))._();
|
||||
|
||||
fsqinfo.qinfo = info;
|
||||
if (info instanceof FairSchedulerLeafQueueInfo) {
|
||||
li.ul("#lq").li()._(LeafQueueBlock.class)._()._();
|
||||
} else {
|
||||
li._(QueueBlock.class);
|
||||
}
|
||||
li._();
|
||||
}
|
||||
|
||||
ul._();
|
||||
}
|
||||
}
|
||||
|
||||
static class QueuesBlock extends HtmlBlock {
|
||||
final FairScheduler fs;
|
||||
final FSQInfo fsqinfo;
|
||||
|
@ -91,8 +131,9 @@ public class FairSchedulerPage extends RmView {
|
|||
fs = (FairScheduler)rm.getResourceScheduler();
|
||||
fsqinfo = info;
|
||||
}
|
||||
|
||||
@Override public void render(Block html) {
|
||||
|
||||
@Override
|
||||
public void render(Block html) {
|
||||
html._(MetricsOverviewTable.class);
|
||||
UL<DIV<DIV<Hamlet>>> ul = html.
|
||||
div("#cs-wrapper.ui-widget").
|
||||
|
@ -108,8 +149,8 @@ public class FairSchedulerPage extends RmView {
|
|||
span(".q", "default")._()._();
|
||||
} else {
|
||||
FairSchedulerInfo sinfo = new FairSchedulerInfo(fs);
|
||||
fsqinfo.fsinfo = sinfo;
|
||||
fsqinfo.qinfo = null;
|
||||
fsqinfo.qinfo = sinfo.getRootQueueInfo();
|
||||
float used = fsqinfo.qinfo.getUsedFraction();
|
||||
|
||||
ul.
|
||||
li().$style("margin-bottom: 1em").
|
||||
|
@ -122,29 +163,15 @@ public class FairSchedulerPage extends RmView {
|
|||
_("Used (over fair share)")._().
|
||||
span().$class("qlegend ui-corner-all ui-state-default").
|
||||
_("Max Capacity")._().
|
||||
_();
|
||||
|
||||
List<FairSchedulerQueueInfo> subQueues = fsqinfo.fsinfo.getQueueInfos();
|
||||
for (FairSchedulerQueueInfo info : subQueues) {
|
||||
fsqinfo.qinfo = info;
|
||||
float capacity = info.getMaxResourcesFraction();
|
||||
float fairShare = info.getFairShareFraction();
|
||||
float used = info.getUsedFraction();
|
||||
ul.
|
||||
li().
|
||||
a(_Q).$style(width(capacity * Q_MAX_WIDTH)).
|
||||
$title(join("Fair Share:", percent(fairShare))).
|
||||
span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))).
|
||||
_('.')._().
|
||||
span().$style(join(width(used/capacity),
|
||||
";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)).
|
||||
_('.')._().
|
||||
span(".q", info.getQueueName())._().
|
||||
span().$class("qstats").$style(left(Q_STATS_POS)).
|
||||
_(join(percent(used), " used"))._().
|
||||
ul("#lq").li()._(QueueInfoBlock.class)._()._().
|
||||
_();
|
||||
}
|
||||
_().
|
||||
li().
|
||||
a(_Q).$style(width(Q_MAX_WIDTH)).
|
||||
span().$style(join(width(used), ";left:0%;",
|
||||
used > 1 ? Q_OVER : Q_UNDER))._(".")._().
|
||||
span(".q", "root")._().
|
||||
span().$class("qstats").$style(left(Q_STATS_POS)).
|
||||
_(join(percent(used), " used"))._().
|
||||
_(QueueBlock.class)._();
|
||||
}
|
||||
ul._()._().
|
||||
script().$type("text/javascript").
|
||||
|
|
|
@ -18,33 +18,23 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||
|
||||
public class FairSchedulerInfo {
|
||||
private List<FairSchedulerQueueInfo> queueInfos;
|
||||
private FairScheduler scheduler;
|
||||
|
||||
public FairSchedulerInfo(FairScheduler fs) {
|
||||
scheduler = fs;
|
||||
Collection<FSLeafQueue> queues = fs.getQueueManager().getLeafQueues();
|
||||
queueInfos = new ArrayList<FairSchedulerQueueInfo>();
|
||||
for (FSLeafQueue queue : queues) {
|
||||
queueInfos.add(new FairSchedulerQueueInfo(queue, fs));
|
||||
}
|
||||
}
|
||||
|
||||
public List<FairSchedulerQueueInfo> getQueueInfos() {
|
||||
return queueInfos;
|
||||
}
|
||||
|
||||
public int getAppFairShare(ApplicationAttemptId appAttemptId) {
|
||||
return scheduler.getSchedulerApp(appAttemptId).
|
||||
getAppSchedulable().getFairShare().getMemory();
|
||||
}
|
||||
|
||||
public FairSchedulerQueueInfo getRootQueueInfo() {
|
||||
return new FairSchedulerQueueInfo(scheduler.getQueueManager().
|
||||
getRootQueue(), scheduler);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,32 @@
|
|||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
|
||||
|
||||
public class FairSchedulerLeafQueueInfo extends FairSchedulerQueueInfo {
|
||||
private int numPendingApps;
|
||||
private int numActiveApps;
|
||||
|
||||
public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
|
||||
super(queue, scheduler);
|
||||
Collection<AppSchedulable> apps = queue.getAppSchedulables();
|
||||
for (AppSchedulable app : apps) {
|
||||
if (app.getApp().isPending()) {
|
||||
numPendingApps++;
|
||||
} else {
|
||||
numActiveApps++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public int getNumActiveApplications() {
|
||||
return numPendingApps;
|
||||
}
|
||||
|
||||
public int getNumPendingApplications() {
|
||||
return numActiveApps;
|
||||
}
|
||||
}
|
|
@ -18,19 +18,18 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
|
||||
|
||||
public class FairSchedulerQueueInfo {
|
||||
private int numPendingApps;
|
||||
private int numActiveApps;
|
||||
|
||||
public class FairSchedulerQueueInfo {
|
||||
private int fairShare;
|
||||
private int minShare;
|
||||
private int maxShare;
|
||||
|
@ -48,16 +47,9 @@ public class FairSchedulerQueueInfo {
|
|||
|
||||
private String queueName;
|
||||
|
||||
public FairSchedulerQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
|
||||
Collection<AppSchedulable> apps = queue.getAppSchedulables();
|
||||
for (AppSchedulable app : apps) {
|
||||
if (app.getApp().isPending()) {
|
||||
numPendingApps++;
|
||||
} else {
|
||||
numActiveApps++;
|
||||
}
|
||||
}
|
||||
|
||||
private Collection<FairSchedulerQueueInfo> childInfos;
|
||||
|
||||
public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) {
|
||||
QueueManager manager = scheduler.getQueueManager();
|
||||
|
||||
queueName = queue.getName();
|
||||
|
@ -81,6 +73,16 @@ public class FairSchedulerQueueInfo {
|
|||
fractionMinShare = (float)minShare / clusterMaxMem;
|
||||
|
||||
maxApps = manager.getQueueMaxApps(queueName);
|
||||
|
||||
Collection<FSQueue> childQueues = queue.getChildQueues();
|
||||
childInfos = new ArrayList<FairSchedulerQueueInfo>();
|
||||
for (FSQueue child : childQueues) {
|
||||
if (child instanceof FSLeafQueue) {
|
||||
childInfos.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
|
||||
} else {
|
||||
childInfos.add(new FairSchedulerQueueInfo(child, scheduler));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -96,15 +98,7 @@ public class FairSchedulerQueueInfo {
|
|||
public int getFairShare() {
|
||||
return fairShare;
|
||||
}
|
||||
|
||||
public int getNumActiveApplications() {
|
||||
return numPendingApps;
|
||||
}
|
||||
|
||||
public int getNumPendingApplications() {
|
||||
return numActiveApps;
|
||||
}
|
||||
|
||||
|
||||
public Resource getMinResources() {
|
||||
return minResources;
|
||||
}
|
||||
|
@ -148,4 +142,8 @@ public class FairSchedulerQueueInfo {
|
|||
public float getMaxResourcesFraction() {
|
||||
return (float)maxShare / clusterMaxMem;
|
||||
}
|
||||
|
||||
public Collection<FairSchedulerQueueInfo> getChildQueues() {
|
||||
return childInfos;
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue