From b54e794fb059fa68b115c2ca5e58f11a0d7f3985 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Fri, 26 Oct 2012 20:55:57 +0000 Subject: [PATCH] YARN-145. Add a Web UI to the fair share scheduler. Contributed by Sandy Ryza. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1402657 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../scheduler/fair/FSQueue.java | 4 +- .../scheduler/fair/FSSchedulerApp.java | 9 + .../scheduler/fair/FairScheduler.java | 6 +- .../scheduler/fair/QueueManager.java | 2 +- .../webapp/FairSchedulerAppsBlock.java | 136 ++++++++++++ .../webapp/FairSchedulerPage.java | 200 ++++++++++++++++++ .../resourcemanager/webapp/RmController.java | 5 +- .../webapp/dao/FairSchedulerInfo.java | 50 +++++ .../webapp/dao/FairSchedulerQueueInfo.java | 153 ++++++++++++++ 10 files changed, 562 insertions(+), 5 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5ca37e253f4..064d869d511 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -23,6 +23,8 @@ Release 2.0.3-alpha - Unreleased NEW FEATURES + YARN-145. Add a Web UI to the fair share scheduler. (Sandy Ryza via tomwhite) + IMPROVEMENTS YARN-78. Changed UnManagedAM application to use YarnClient. (Bikas Saha via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 100fbf2af0c..31508d3d162 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -56,7 +56,9 @@ public Collection getApplications() { public void addApp(FSSchedulerApp app) { applications.add(app); - queueSchedulable.addApp(new AppSchedulable(scheduler, app, this)); + AppSchedulable appSchedulable = new AppSchedulable(scheduler, app, this); + app.setAppSchedulable(appSchedulable); + queueSchedulable.addApp(appSchedulable); } public void removeJob(FSSchedulerApp app) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index b9931df86bf..e2a385f26c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -70,6 +70,7 @@ public class FSSchedulerApp extends SchedulerApplication { .getRecordFactory(null); private final AppSchedulingInfo appSchedulingInfo; + private AppSchedulable appSchedulable; private final Queue queue; private final Resource currentConsumption = recordFactory @@ -118,6 +119,14 @@ public ApplicationId getApplicationId() { public ApplicationAttemptId getApplicationAttemptId() { return this.appSchedulingInfo.getApplicationAttemptId(); } + + public void setAppSchedulable(AppSchedulable appSchedulable) { + this.appSchedulable = appSchedulable; + } + + public AppSchedulable getAppSchedulable() { + return appSchedulable; + } public String getUser() { return this.appSchedulingInfo.getUser(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index e102622d2b6..1d2412e5ffb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -788,7 +788,11 @@ public SchedulerNodeReport getNodeReport(NodeId nodeId) { FSSchedulerNode node = nodes.get(nodeId); return node == null ? null : new SchedulerNodeReport(node); } - + + public FSSchedulerApp getSchedulerApp(ApplicationAttemptId appAttemptId) { + return applications.get(appAttemptId); + } + @Override public SchedulerAppReport getSchedulerAppInfo( ApplicationAttemptId appAttemptId) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index 38e57b40688..c765e7f7dab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -446,7 +446,7 @@ private void setMinResources(Map resources) { * Get the maximum resource allocation for the given queue. * @return the cap set on this queue, or Integer.MAX_VALUE if not set. */ - Resource getMaxResources(String queueName) { + public Resource getMaxResources(String queueName) { synchronized (maxQueueResourcesMO) { if (maxQueueResources.containsKey(queueName)) { return maxQueueResources.get(queueName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java new file mode 100644 index 00000000000..efbe64a5b78 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -0,0 +1,136 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE; + +import java.util.Collection; +import java.util.HashSet; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; +import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.webapp.view.JQueryUI.Render; + +import com.google.inject.Inject; + +/** + * Shows application information specific to the fair + * scheduler as part of the fair scheduler page. + */ +public class FairSchedulerAppsBlock extends HtmlBlock { + final AppsList list; + final FairSchedulerInfo fsinfo; + + @Inject public FairSchedulerAppsBlock(AppsList list, + ResourceManager rm, ViewContext ctx) { + super(ctx); + this.list = list; + FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler(); + fsinfo = new FairSchedulerInfo(scheduler); + } + + @Override public void render(Block html) { + TBODY> tbody = html. + table("#apps"). + thead(). + tr(). + th(".id", "ID"). + th(".user", "User"). + th(".name", "Name"). + th(".queue", "Queue"). + th(".fairshare", "Fair Share"). + th(".starttime", "StartTime"). + th(".finishtime", "FinishTime"). + th(".state", "State"). + th(".finalstatus", "FinalStatus"). + th(".progress", "Progress"). + th(".ui", "Tracking UI")._()._(). + tbody(); + int i = 0; + Collection reqAppStates = null; + String reqStateString = $(APP_STATE); + if (reqStateString != null && !reqStateString.isEmpty()) { + String[] appStateStrings = reqStateString.split(","); + reqAppStates = new HashSet(appStateStrings.length); + for(String stateString : appStateStrings) { + reqAppStates.add(RMAppState.valueOf(stateString)); + } + } + for (RMApp app : list.apps.values()) { + if (reqAppStates != null && !reqAppStates.contains(app.getState())) { + continue; + } + AppInfo appInfo = new AppInfo(app, true); + String percent = String.format("%.1f", appInfo.getProgress()); + String startTime = Times.format(appInfo.getStartTime()); + String finishTime = Times.format(appInfo.getFinishTime()); + ApplicationAttemptId attemptId = app.getCurrentAppAttempt().getAppAttemptId(); + int fairShare = fsinfo.getAppFairShare(attemptId); + + tbody. + tr(). + td(). + br().$title(appInfo.getAppIdNum())._(). // for sorting + a(url("app", appInfo.getAppId()), appInfo.getAppId())._(). + td(appInfo.getUser()). + td(appInfo.getName()). + td(appInfo.getQueue()). + td("" + fairShare). + td(). + br().$title(String.valueOf(appInfo.getStartTime()))._(). + _(startTime)._(). + td(). + br().$title(String.valueOf(appInfo.getFinishTime()))._(). + _(finishTime)._(). + td(appInfo.getState()). + td(appInfo.getFinalStatus()). + td(). + br().$title(percent)._(). // for sorting + div(_PROGRESSBAR). + $title(join(percent, '%')). // tooltip + div(_PROGRESSBAR_VALUE). + $style(join("width:", percent, '%'))._()._()._(). + td(). + a(!appInfo.isTrackingUrlReady()? + "#" : appInfo.getTrackingUrlPretty(), appInfo.getTrackingUI())._()._(); + if (list.rendering != Render.HTML && ++i >= 20) break; + } + tbody._()._(); + + if (list.rendering == Render.JS_ARRAY) { + echo("\n"); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java new file mode 100644 index 00000000000..3a560161402 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.apache.hadoop.yarn.util.StringHelper.join; + +import java.util.List; + +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerQueueInfo; +import org.apache.hadoop.yarn.webapp.ResponseInfo; +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.webapp.view.InfoBlock; + +import com.google.inject.Inject; +import com.google.inject.servlet.RequestScoped; + +public class FairSchedulerPage extends RmView { + static final String _Q = ".ui-state-default.ui-corner-all"; + static final float Q_MAX_WIDTH = 0.8f; + static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f; + static final String Q_END = "left:101%"; + static final String Q_GIVEN = "left:0%;background:none;border:1px dashed rgba(0,0,0,0.25)"; + static final String Q_OVER = "background:rgba(255, 140, 0, 0.8)"; + static final String Q_UNDER = "background:rgba(50, 205, 50, 0.8)"; + + @RequestScoped + static class FSQInfo { + FairSchedulerInfo fsinfo; + FairSchedulerQueueInfo qinfo; + } + + static class QueueInfoBlock extends HtmlBlock { + final FairSchedulerQueueInfo qinfo; + + @Inject QueueInfoBlock(ViewContext ctx, FSQInfo info) { + super(ctx); + qinfo = (FairSchedulerQueueInfo) info.qinfo; + } + + @Override + protected void render(Block html) { + ResponseInfo ri = info("\'" + qinfo.getQueueName() + "\' Queue Status"). + _("Used Resources:", qinfo.getUsedResources().toString()). + _("Num Active Applications:", qinfo.getNumActiveApplications()). + _("Num Pending Applications:", qinfo.getNumPendingApplications()). + _("Min Resources:", qinfo.getMinResources().toString()). + _("Max Resources:", qinfo.getMaxResources().toString()); + int maxApps = qinfo.getMaxApplications(); + if (maxApps < Integer.MAX_VALUE) { + ri._("Max Running Applications:", qinfo.getMaxApplications()); + } + ri._("Fair Share:", qinfo.getFairShare()); + + html._(InfoBlock.class); + + // clear the info contents so this queue's info doesn't accumulate into another queue's info + ri.clear(); + } + } + + static class QueuesBlock extends HtmlBlock { + final FairScheduler fs; + final FSQInfo fsqinfo; + + @Inject QueuesBlock(ResourceManager rm, FSQInfo info) { + fs = (FairScheduler)rm.getResourceScheduler(); + fsqinfo = info; + } + + @Override public void render(Block html) { + html._(MetricsOverviewTable.class); + UL>> ul = html. + div("#cs-wrapper.ui-widget"). + div(".ui-widget-header.ui-corner-top"). + _("Application Queues")._(). + div("#cs.ui-widget-content.ui-corner-bottom"). + ul(); + if (fs == null) { + ul. + li(). + a(_Q).$style(width(Q_MAX_WIDTH)). + span().$style(Q_END)._("100% ")._(). + span(".q", "default")._()._(); + } else { + FairSchedulerInfo sinfo = new FairSchedulerInfo(fs); + fsqinfo.fsinfo = sinfo; + fsqinfo.qinfo = null; + + ul. + li().$style("margin-bottom: 1em"). + span().$style("font-weight: bold")._("Legend:")._(). + span().$class("qlegend ui-corner-all").$style(Q_GIVEN). + _("Fair Share")._(). + span().$class("qlegend ui-corner-all").$style(Q_UNDER). + _("Used")._(). + span().$class("qlegend ui-corner-all").$style(Q_OVER). + _("Used (over fair share)")._(). + span().$class("qlegend ui-corner-all ui-state-default"). + _("Max Capacity")._(). + _(); + + List subQueues = fsqinfo.fsinfo.getQueueInfos(); + for (FairSchedulerQueueInfo info : subQueues) { + fsqinfo.qinfo = info; + float capacity = info.getMaxResourcesFraction(); + float fairShare = info.getFairShareFraction(); + float used = info.getUsedFraction(); + ul. + li(). + a(_Q).$style(width(capacity * Q_MAX_WIDTH)). + $title(join("Fair Share:", percent(fairShare))). + span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))). + _('.')._(). + span().$style(join(width(used/capacity), + ";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)). + _('.')._(). + span(".q", info.getQueueName())._(). + span().$class("qstats").$style(left(Q_STATS_POS)). + _(join(percent(used), " used"))._(). + ul("#lq").li()._(QueueInfoBlock.class)._()._(). + _(); + } + } + ul._()._(). + script().$type("text/javascript"). + _("$('#cs').hide();")._()._(). + _(FairSchedulerAppsBlock.class); + } + } + + @Override protected void postHead(Page.HTML<_> html) { + html. + style().$type("text/css"). + _("#cs { padding: 0.5em 0 1em 0; margin-bottom: 1em; position: relative }", + "#cs ul { list-style: none }", + "#cs a { font-weight: normal; margin: 2px; position: relative }", + "#cs a span { font-weight: normal; font-size: 80% }", + "#cs-wrapper .ui-widget-header { padding: 0.2em 0.5em }", + "table.info tr th {width: 50%}")._(). // to center info table + script("/static/jt/jquery.jstree.js"). + script().$type("text/javascript"). + _("$(function() {", + " $('#cs a span').addClass('ui-corner-all').css('position', 'absolute');", + " $('#cs').bind('loaded.jstree', function (e, data) {", + " data.inst.open_all(); }).", + " jstree({", + " core: { animation: 188, html_titles: true },", + " plugins: ['themeroller', 'html_data', 'ui'],", + " themeroller: { item_open: 'ui-icon-minus',", + " item_clsd: 'ui-icon-plus', item_leaf: 'ui-icon-gear'", + " }", + " });", + " $('#cs').bind('select_node.jstree', function(e, data) {", + " var q = $('.q', data.rslt.obj).first().text();", + " if (q == 'root') q = '';", + " $('#apps').dataTable().fnFilter(q, 3);", + " });", + " $('#cs').show();", + "});")._(); + } + + @Override protected Class content() { + return QueuesBlock.class; + } + + static String percent(float f) { + return String.format("%.1f%%", f * 100); + } + + static String width(float f) { + return String.format("width:%.1f%%", f * 100); + } + + static String left(float f) { + return String.format("left:%.1f%%", f * 100); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java index 20a6ebe5cbd..753e197af01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java @@ -77,8 +77,9 @@ public void scheduler() { } if (rs instanceof FairScheduler) { - context().setStatus(404); - throw new WebAppException("Fair Scheduler UI not yet supported"); + setTitle("Fair Scheduler"); + render(FairSchedulerPage.class); + return; } setTitle("Default Scheduler"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java new file mode 100644 index 00000000000..0591683bd30 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; + +public class FairSchedulerInfo { + private List queueInfos; + private FairScheduler scheduler; + + public FairSchedulerInfo(FairScheduler fs) { + scheduler = fs; + Collection queues = fs.getQueueManager().getQueues(); + queueInfos = new ArrayList(); + for (FSQueue queue : queues) { + queueInfos.add(new FairSchedulerQueueInfo(queue, fs)); + } + } + + public List getQueueInfos() { + return queueInfos; + } + + public int getAppFairShare(ApplicationAttemptId appAttemptId) { + return scheduler.getSchedulerApp(appAttemptId). + getAppSchedulable().getFairShare().getMemory(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java new file mode 100644 index 00000000000..ab224087fee --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.Collection; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueSchedulable; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; + +public class FairSchedulerQueueInfo { + private int numPendingApps; + private int numActiveApps; + + private int fairShare; + private int minShare; + private int maxShare; + private int clusterMaxMem; + + private int maxApps; + + private float fractionUsed; + private float fractionFairShare; + private float fractionMinShare; + + private Resource minResources; + private Resource maxResources; + private Resource usedResources; + + private String queueName; + + public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { + Collection apps = queue.getApplications(); + for (FSSchedulerApp app : apps) { + if (app.isPending()) { + numPendingApps++; + } else { + numActiveApps++; + } + } + + FSQueueSchedulable schedulable = queue.getQueueSchedulable(); + QueueManager manager = scheduler.getQueueManager(); + + queueName = queue.getName(); + + Resource clusterMax = scheduler.getClusterCapacity(); + clusterMaxMem = clusterMax.getMemory(); + + usedResources = schedulable.getResourceUsage(); + fractionUsed = (float)usedResources.getMemory() / clusterMaxMem; + + fairShare = schedulable.getFairShare().getMemory(); + minResources = schedulable.getMinShare(); + minShare = minResources.getMemory(); + maxResources = scheduler.getQueueManager().getMaxResources(queueName); + if (maxResources.getMemory() > clusterMaxMem) { + maxResources = Resources.createResource(clusterMaxMem); + } + maxShare = maxResources.getMemory(); + + fractionFairShare = (float)fairShare / clusterMaxMem; + fractionMinShare = (float)minShare / clusterMaxMem; + + maxApps = manager.getQueueMaxApps(queueName); + } + + /** + * Returns the fair share as a fraction of the entire cluster capacity. + */ + public float getFairShareFraction() { + return fractionFairShare; + } + + /** + * Returns the fair share of this queue in megabytes. + */ + public int getFairShare() { + return fairShare; + } + + public int getNumActiveApplications() { + return numPendingApps; + } + + public int getNumPendingApplications() { + return numActiveApps; + } + + public Resource getMinResources() { + return minResources; + } + + public Resource getMaxResources() { + return maxResources; + } + + public int getMaxApplications() { + return maxApps; + } + + public String getQueueName() { + return queueName; + } + + public Resource getUsedResources() { + return usedResources; + } + + /** + * Returns the queue's min share in as a fraction of the entire + * cluster capacity. + */ + public float getMinShareFraction() { + return fractionMinShare; + } + + /** + * Returns the memory used by this queue as a fraction of the entire + * cluster capacity. + */ + public float getUsedFraction() { + return fractionUsed; + } + + /** + * Returns the capacity of this queue as a fraction of the entire cluster + * capacity. + */ + public float getMaxResourcesFraction() { + return (float)maxShare / clusterMaxMem; + } +}