From ffdf980b2056b2a1b31ccb19746f23c31f7d08ef Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Tue, 28 Feb 2012 20:06:53 +0000 Subject: [PATCH] MAPREDUCE-3816 capacity scheduler web ui bar graphs for used capacity wrong (tgraves via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1294808 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 5 +- .../scheduler/capacity/CSQueue.java | 35 ++++++------- .../scheduler/capacity/CSQueueUtils.java | 30 ++++++------ .../scheduler/capacity/CapacityScheduler.java | 4 +- .../scheduler/capacity/LeafQueue.java | 49 +++++++++++-------- .../scheduler/capacity/ParentQueue.java | 37 ++++++++------ .../webapp/CapacitySchedulerPage.java | 31 ++++++------ .../dao/CapacitySchedulerQueueInfo.java | 8 +-- .../scheduler/capacity/TestParentQueue.java | 11 ++--- .../TestRMWebServicesCapacitySched.java | 10 ++-- .../src/site/apt/ResourceManagerRest.apt.vm | 46 ++++++++--------- 11 files changed, 140 insertions(+), 126 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index fd3064752df..9e6f6dff42f 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -197,9 +197,12 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-3922. Fixed build to not compile 32bit container-executor binary by default on all platforms. (Hitesh Shah via vinodkv) - MAPREDUCE-3790 Broken pipe on streaming job can lead to truncated output for + MAPREDUCE-3790. Broken pipe on streaming job can lead to truncated output for a successful job (Jason Lowe via bobby) + MAPREDUCE-3816. capacity scheduler web ui bar graphs for used capacity wrong + (tgraves via bobby) + Release 0.23.1 - 2012-02-17 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 0730cfc25d6..176a948e439 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -91,10 +91,16 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue { public float getAbsoluteMaximumCapacity(); /** - * Get the currently utilized capacity of the queue - * relative to it's parent queue. - * @return the currently utilized capacity of the queue - * relative to it's parent queue + * Get the current absolute used capacity of the queue + * relative to the entire cluster. + * @return queue absolute used capacity + */ + public float getAbsoluteUsedCapacity(); + + /** + * Get the current used capacity of the queue + * and it's children (if any). + * @return queue used capacity */ public float getUsedCapacity(); @@ -104,6 +110,12 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue { */ public void setUsedCapacity(float usedCapacity); + /** + * Set absolute used capacity of the queue. + * @param absUsedCapacity absolute used capacity of the queue + */ + public void setAbsoluteUsedCapacity(float absUsedCapacity); + /** * Get the currently utilized resources in the cluster * by the queue and children (if any). @@ -111,21 +123,6 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue { */ public Resource getUsedResources(); - /** - * Get the current utilization of the queue - * and it's children (if any). - * Utilization is defined as the ratio of - * used-capacity over configured-capacity of the queue. - * @return queue utilization - */ - public float getUtilization(); - - /** - * Get the current utilization of the queue. - * @param utilization queue utilization - */ - public void setUtilization(float utilization); - /** * Get the current run-state of the queue * @return current run-state diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 01f14ebc53a..48c04916c4c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -23,21 +23,25 @@ import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; class CSQueueUtils { + final static float EPSILON = 0.0001f; + public static void checkMaxCapacity(String queueName, float capacity, float maximumCapacity) { - if (maximumCapacity < 0.0f || maximumCapacity > 1.0f || - maximumCapacity < capacity) { + if (maximumCapacity < 0.0f || maximumCapacity > 1.0f) { throw new IllegalArgumentException( "Illegal value of maximumCapacity " + maximumCapacity + " used in call to setMaxCapacity for queue " + queueName); } - if (maximumCapacity < capacity) { - throw new IllegalArgumentException( - "Illegal call to setMaxCapacity. " + - "Queue '" + queueName + "' has " + - "capacity (" + capacity + ") greater than " + - "maximumCapacity (" + maximumCapacity + ")" ); } + + public static void checkAbsoluteCapacities(String queueName, + float absCapacity, float absMaxCapacity) { + if (absMaxCapacity < (absCapacity - EPSILON)) { + throw new IllegalArgumentException("Illegal call to setMaxCapacity. " + + "Queue '" + queueName + "' has " + "an absolute capacity (" + absCapacity + + ") greater than " + "its absolute maximumCapacity (" + absMaxCapacity + + ")"); + } } public static float computeAbsoluteMaximumCapacity( @@ -75,18 +79,16 @@ class CSQueueUtils { final int usedMemory = childQueue.getUsedResources().getMemory(); float queueLimit = 0.0f; - float utilization = 0.0f; + float absoluteUsedCapacity = 0.0f; float usedCapacity = 0.0f; if (clusterMemory > 0) { queueLimit = clusterMemory * childQueue.getAbsoluteCapacity(); - final float parentAbsoluteCapacity = - (parentQueue == null) ? 1.0f : parentQueue.getAbsoluteCapacity(); - utilization = (usedMemory / queueLimit); - usedCapacity = (usedMemory / (clusterMemory * parentAbsoluteCapacity)); + absoluteUsedCapacity = ((float)usedMemory / (float)clusterMemory); + usedCapacity = (usedMemory / queueLimit); } - childQueue.setUtilization(utilization); childQueue.setUsedCapacity(usedCapacity); + childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity); int available = Math.max((roundUp(minimumAllocation, (int)queueLimit) - usedMemory), 0); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 0f90f6c4e2c..47e7a3dbe67 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -91,9 +91,9 @@ implements ResourceScheduler, CapacitySchedulerContext { static final Comparator queueComparator = new Comparator() { @Override public int compare(CSQueue q1, CSQueue q2) { - if (q1.getUtilization() < q2.getUtilization()) { + if (q1.getUsedCapacity() < q2.getUsedCapacity()) { return -1; - } else if (q1.getUtilization() > q2.getUtilization()) { + } else if (q1.getUsedCapacity() > q2.getUsedCapacity()) { return 1; } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 91f1c8ea97b..0e3d747523a 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -80,6 +80,7 @@ public class LeafQueue implements CSQueue { private float absoluteCapacity; private float maximumCapacity; private float absoluteMaxCapacity; + private float absoluteUsedCapacity = 0.0f; private int userLimit; private float userLimitFactor; @@ -91,7 +92,6 @@ public class LeafQueue implements CSQueue { private int maxActiveApplicationsPerUser; private Resource usedResources = Resources.createResource(0); - private float utilization = 0.0f; private float usedCapacity = 0.0f; private volatile int numContainers; @@ -210,9 +210,11 @@ public class LeafQueue implements CSQueue { { // Sanity check CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); + float absCapacity = parent.getAbsoluteCapacity() * capacity; + CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absCapacity, absoluteMaxCapacity); this.capacity = capacity; - this.absoluteCapacity = parent.getAbsoluteCapacity() * capacity; + this.absoluteCapacity = absCapacity; this.maximumCapacity = maximumCapacity; this.absoluteMaxCapacity = absoluteMaxCapacity; @@ -274,12 +276,11 @@ public class LeafQueue implements CSQueue { "(int)(maxActiveApplications * (userLimit / 100.0f) * " + "userLimitFactor)," + "1) ]" + "\n" + - "utilization = " + utilization + - " [= usedResourcesMemory / " + - "(clusterResourceMemory * absoluteCapacity)]" + "\n" + "usedCapacity = " + usedCapacity + " [= usedResourcesMemory / " + - "(clusterResourceMemory * parent.absoluteCapacity)]" + "\n" + + "(clusterResourceMemory * absoluteCapacity)]" + "\n" + + "absoluteUsedCapacity = " + absoluteUsedCapacity + + " [= usedResourcesMemory / clusterResourceMemory]" + "\n" + "maxAMResourcePercent = " + maxAMResourcePercent + " [= configuredMaximumAMResourcePercent ]" + "\n" + "minimumAllocationFactor = " + minimumAllocationFactor + @@ -313,6 +314,11 @@ public class LeafQueue implements CSQueue { return absoluteMaxCapacity; } + @Override + public synchronized float getAbsoluteUsedCapacity() { + return absoluteUsedCapacity; + } + @Override public CSQueue getParent() { return parent; @@ -383,24 +389,21 @@ public class LeafQueue implements CSQueue { return usedResources; } - @Override - public synchronized float getUtilization() { - return utilization; - } - @Override public List getChildQueues() { return null; } - public synchronized void setUtilization(float utilization) { - this.utilization = utilization; - } - + @Override public synchronized void setUsedCapacity(float usedCapacity) { this.usedCapacity = usedCapacity; } + @Override + public synchronized void setAbsoluteUsedCapacity(float absUsedCapacity) { + this.absoluteUsedCapacity = absUsedCapacity; + } + /** * Set maximum capacity - used only for testing. * @param maximumCapacity new max capacity @@ -408,10 +411,11 @@ public class LeafQueue implements CSQueue { synchronized void setMaxCapacity(float maximumCapacity) { // Sanity check CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); + float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent); + CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absoluteCapacity, absMaxCapacity); this.maximumCapacity = maximumCapacity; - this.absoluteMaxCapacity = - CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent); + this.absoluteMaxCapacity = absMaxCapacity; } /** @@ -516,7 +520,7 @@ public class LeafQueue implements CSQueue { "absoluteCapacity=" + absoluteCapacity + ", " + "usedResources=" + usedResources.getMemory() + "MB, " + "usedCapacity=" + getUsedCapacity() + ", " + - "utilization=" + getUtilization() + ", " + + "absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + ", " + "numApps=" + getNumApplications() + ", " + "numContainers=" + getNumContainers(); } @@ -1228,7 +1232,8 @@ public class LeafQueue implements CSQueue { " container=" + container + " containerId=" + container.getId() + " queue=" + this + - " util=" + getUtilization() + + " usedCapacity=" + getUsedCapacity() + + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster=" + clusterResource); @@ -1241,7 +1246,8 @@ public class LeafQueue implements CSQueue { " application=" + application.getApplicationId() + " resource=" + request.getCapability() + " queue=" + this.toString() + - " util=" + getUtilization() + + " usedCapacity=" + getUsedCapacity() + + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster=" + clusterResource); @@ -1307,7 +1313,8 @@ public class LeafQueue implements CSQueue { " container=" + container + " resource=" + container.getResource() + " queue=" + this + - " util=" + getUtilization() + + " usedCapacity=" + getUsedCapacity() + + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster=" + clusterResource); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index da7b195889d..2b8147f3dc3 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -67,9 +67,9 @@ public class ParentQueue implements CSQueue { private float maximumCapacity; private float absoluteCapacity; private float absoluteMaxCapacity; + private float absoluteUsedCapacity = 0.0f; private float usedCapacity = 0.0f; - private float utilization = 0.0f; private final Set childQueues; private final Comparator queueComparator; @@ -158,9 +158,11 @@ public class ParentQueue implements CSQueue { ) { // Sanity check CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); + CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absoluteCapacity, absoluteMaxCapacity); this.capacity = capacity; this.absoluteCapacity = absoluteCapacity; + this.maximumCapacity = maximumCapacity; this.absoluteMaxCapacity = absoluteMaxCapacity; @@ -243,6 +245,11 @@ public class ParentQueue implements CSQueue { return absoluteMaxCapacity; } + @Override + public synchronized float getAbsoluteUsedCapacity() { + return absoluteUsedCapacity; + } + @Override public float getMaximumCapacity() { return maximumCapacity; @@ -264,11 +271,6 @@ public class ParentQueue implements CSQueue { return usedResources; } - @Override - public synchronized float getUtilization() { - return utilization; - } - @Override public synchronized List getChildQueues() { return new ArrayList(childQueues); @@ -351,7 +353,6 @@ public class ParentQueue implements CSQueue { "absoluteCapacity=" + absoluteCapacity + ", " + "usedResources=" + usedResources.getMemory() + "MB, " + "usedCapacity=" + getUsedCapacity() + ", " + - "utilization=" + getUtilization() + ", " + "numApps=" + getNumApplications() + ", " + "numContainers=" + getNumContainers(); } @@ -490,12 +491,14 @@ public class ParentQueue implements CSQueue { " #applications: " + getNumApplications()); } + @Override public synchronized void setUsedCapacity(float usedCapacity) { this.usedCapacity = usedCapacity; } - public synchronized void setUtilization(float utilization) { - this.utilization = utilization; + @Override + public synchronized void setAbsoluteUsedCapacity(float absUsedCapacity) { + this.absoluteUsedCapacity = absUsedCapacity; } /** @@ -505,10 +508,11 @@ public class ParentQueue implements CSQueue { synchronized void setMaxCapacity(float maximumCapacity) { // Sanity check CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); + float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent); + CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absoluteCapacity, absMaxCapacity); this.maximumCapacity = maximumCapacity; - this.absoluteMaxCapacity = - CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent); + this.absoluteMaxCapacity = absMaxCapacity; } @Override @@ -545,7 +549,8 @@ public class ParentQueue implements CSQueue { LOG.info("assignedContainer" + " queue=" + getQueueName() + - " util=" + getUtilization() + + " usedCapacity=" + getUsedCapacity() + + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster=" + clusterResource); @@ -556,7 +561,8 @@ public class ParentQueue implements CSQueue { if (LOG.isDebugEnabled()) { LOG.debug("ParentQ=" + getQueueName() + " assignedSoFarInThisIteration=" + assignment.getResource() - + " utilization=" + getUtilization()); + + " usedCapacity=" + getUsedCapacity() + + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity()); } // Do not assign more than one container if this isn't the root queue @@ -639,7 +645,7 @@ public class ParentQueue implements CSQueue { String getChildQueuesToPrint() { StringBuilder sb = new StringBuilder(); for (CSQueue q : childQueues) { - sb.append(q.getQueuePath() + "(" + q.getUtilization() + "), "); + sb.append(q.getQueuePath() + "(" + q.getUsedCapacity() + "), "); } return sb.toString(); } @@ -663,7 +669,8 @@ public class ParentQueue implements CSQueue { LOG.info("completedContainer" + " queue=" + getQueueName() + - " util=" + getUtilization() + + " usedCapacity=" + getUsedCapacity() + + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster=" + clusterResource); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index edf0231ae3e..2bfda630202 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -67,12 +67,9 @@ class CapacitySchedulerPage extends RmView { protected void render(Block html) { ResponseInfo ri = info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status"). _("Queue State:", lqinfo.getQueueState()). - _("Capacity:", percent(lqinfo.getCapacity() / 100)). - _("Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)). _("Used Capacity:", percent(lqinfo.getUsedCapacity() / 100)). _("Absolute Capacity:", percent(lqinfo.getAbsoluteCapacity() / 100)). _("Absolute Max Capacity:", percent(lqinfo.getAbsoluteMaxCapacity() / 100)). - _("Utilization:", percent(lqinfo.getUtilization() / 100)). _("Used Resources:", lqinfo.getUsedResources().toString()). _("Num Active Applications:", Integer.toString(lqinfo.getNumActiveApplications())). _("Num Pending Applications:", Integer.toString(lqinfo.getNumPendingApplications())). @@ -81,8 +78,10 @@ class CapacitySchedulerPage extends RmView { _("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())). _("Max Active Applications:", Integer.toString(lqinfo.getMaxActiveApplications())). _("Max Active Applications Per User:", Integer.toString(lqinfo.getMaxActiveApplicationsPerUser())). - _("User Limit:", Integer.toString(lqinfo.getUserLimit()) + "%"). - _("User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())); + _("Configured Capacity:", percent(lqinfo.getCapacity() / 100)). + _("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)). + _("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%"). + _("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())); html._(InfoBlock.class); @@ -103,20 +102,20 @@ class CapacitySchedulerPage extends RmView { ArrayList subQueues = (csqinfo.qinfo == null) ? csqinfo.csinfo.getSubQueues() : csqinfo.qinfo.getSubQueues(); - UL ul = html.ul(); + UL ul = html.ul("#pq"); for (CapacitySchedulerQueueInfo info : subQueues) { float used = info.getUsedCapacity() / 100; - float set = info.getCapacity() / 100; - float max = info.getMaxCapacity() / 100; + float absCap = info.getAbsoluteCapacity() / 100; + float absMaxCap = info.getAbsoluteMaxCapacity() / 100; + float absUsedCap = info.getAbsoluteUsedCapacity() / 100; LI> li = ul. li(). - a(_Q).$style(width(max * Q_MAX_WIDTH)). - $title(join("capacity:", percent(set), " used:", percent(used), - " max capacity:", percent(max))). - span().$style(join(Q_GIVEN, ";font-size:1px;", width(set/max))). + a(_Q).$style(width(absMaxCap * Q_MAX_WIDTH)). + $title(join("Absolute Capacity:", percent(absCap))). + span().$style(join(Q_GIVEN, ";font-size:1px;", width(absCap/absMaxCap))). _('.')._(). - span().$style(join(width(used*set/max), - ";font-size:1px;left:0%;", used > 1 ? Q_OVER : Q_UNDER)). + span().$style(join(width(absUsedCap/absMaxCap), + ";font-size:1px;left:0%;", absUsedCap > absCap ? Q_OVER : Q_UNDER)). _('.')._(). span(".q", info.getQueuePath().substring(5))._(). span().$class("qstats").$style(left(Q_STATS_POS)). @@ -180,7 +179,6 @@ class CapacitySchedulerPage extends RmView { _(). li(). a(_Q).$style(width(Q_MAX_WIDTH)). - $title(join("used:", percent(used))). span().$style(join(width(used), ";left:0%;", used > 1 ? Q_OVER : Q_UNDER))._(".")._(). span(".q", "root")._(). @@ -211,8 +209,7 @@ class CapacitySchedulerPage extends RmView { _("$(function() {", " $('#cs a span').addClass('ui-corner-all').css('position', 'absolute');", " $('#cs').bind('loaded.jstree', function (e, data) {", - " data.inst.open_all();", - " data.inst.close_node('#lq', true);", + " data.inst.open_node('#pq', true);", " }).", " jstree({", " core: { animation: 188, html_titles: true },", diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index edce4469c91..e3e81500105 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -44,7 +44,7 @@ public class CapacitySchedulerQueueInfo { protected float maxCapacity; protected float absoluteCapacity; protected float absoluteMaxCapacity; - protected float utilization; + protected float absoluteUsedCapacity; protected int numApplications; protected String usedResources; protected String queueName; @@ -66,7 +66,7 @@ public class CapacitySchedulerQueueInfo { absoluteCapacity = cap(q.getAbsoluteCapacity(), 0f, 1f) * 100; absoluteMaxCapacity = cap(q.getAbsoluteMaximumCapacity(), 0f, 1f) * 100; - utilization = q.getUtilization() * 100; + absoluteUsedCapacity = cap(q.getAbsoluteUsedCapacity(), 0f, 1f) * 100; numApplications = q.getNumApplications(); usedResources = q.getUsedResources().toString(); queueName = q.getQueueName(); @@ -93,8 +93,8 @@ public class CapacitySchedulerQueueInfo { return absoluteMaxCapacity; } - public float getUtilization() { - return utilization; + public float getAbsoluteUsedCapacity() { + return absoluteUsedCapacity; } public int getNumApplications() { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index f3092b0637e..4b943524bbe 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -138,15 +138,14 @@ public class TestParentQueue { when(queue).assignContainers(eq(clusterResource), eq(node)); } - private float computeQueueUsedCapacity(CSQueue queue, + private float computeQueueAbsoluteUsedCapacity(CSQueue queue, int expectedMemory, Resource clusterResource) { return ( - ((float)expectedMemory / clusterResource.getMemory()) * - queue.getParent().getAbsoluteCapacity() + ((float)expectedMemory / (float)clusterResource.getMemory()) ); } - private float computeQueueUtilization(CSQueue queue, + private float computeQueueUsedCapacity(CSQueue queue, int expectedMemory, Resource clusterResource) { return (expectedMemory / (clusterResource.getMemory() * queue.getAbsoluteCapacity())); @@ -156,8 +155,8 @@ public class TestParentQueue { private void verifyQueueMetrics(CSQueue queue, int expectedMemory, Resource clusterResource) { assertEquals( - computeQueueUtilization(queue, expectedMemory, clusterResource), - queue.getUtilization(), + computeQueueAbsoluteUsedCapacity(queue, expectedMemory, clusterResource), + queue.getAbsoluteUsedCapacity(), DELTA); assertEquals( computeQueueUsedCapacity(queue, expectedMemory, clusterResource), diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index b82171e1eae..bd44f9c3e3d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -68,7 +68,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTest { float maxCapacity; float absoluteCapacity; float absoluteMaxCapacity; - float utilization; + float absoluteUsedCapacity; int numApplications; String usedResources; String queueName; @@ -252,7 +252,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTest { qi.absoluteCapacity = WebServicesTestUtils.getXmlFloat(qElem, "absoluteCapacity"); qi.absoluteMaxCapacity = WebServicesTestUtils.getXmlFloat(qElem, "absoluteMaxCapacity"); - qi.utilization = WebServicesTestUtils.getXmlFloat(qElem, "utilization"); + qi.absoluteUsedCapacity = + WebServicesTestUtils.getXmlFloat(qElem, "absoluteUsedCapacity"); qi.numApplications = WebServicesTestUtils.getXmlInt(qElem, "numApplications"); qi.usedResources = @@ -342,7 +343,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTest { qi.maxCapacity = (float) info.getDouble("maxCapacity"); qi.absoluteCapacity = (float) info.getDouble("absoluteCapacity"); qi.absoluteMaxCapacity = (float) info.getDouble("absoluteMaxCapacity"); - qi.utilization = (float) info.getDouble("utilization"); + qi.absoluteUsedCapacity = (float) info.getDouble("absoluteUsedCapacity"); qi.numApplications = info.getInt("numApplications"); qi.usedResources = info.getString("usedResources"); qi.queueName = info.getString("queueName"); @@ -394,7 +395,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTest { parentAbsCapacity * (info.capacity/100), info.absoluteCapacity, 1e-3f); assertEquals("absoluteMaxCapacity doesn't match", expectAbsMaxCapacity, info.absoluteMaxCapacity, 1e-3f); - assertEquals("utilization doesn't match", 0, info.utilization, 1e-3f); + assertEquals("absoluteUsedCapacity doesn't match", + 0, info.absoluteUsedCapacity, 1e-3f); assertEquals("numApplications doesn't match", 0, info.numApplications); assertTrue("usedResources doesn't match", info.usedResources.matches("memory: 0")); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index e762594af8e..4b26d2e469f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -313,11 +313,11 @@ ResourceManager REST API's. *---------------+--------------+-------------------------------+ | type | string | Scheduler type - capacityScheduler| *---------------+--------------+-------------------------------+ -| capacity | float | Queue capacity in percentage relative to its parent queue | +| capacity | float | Configured queue capacity in percentage relative to its parent queue | *---------------+--------------+-------------------------------+ -| usedCapacity | float | Used queue capacity in percentage relative its to parent queue | +| usedCapacity | float | Used queue capacity in percentage | *---------------+--------------+-------------------------------+ -| maxCapacity | float | Maximum queue capacity in percentage relative to its parent queue | +| maxCapacity | float | Configured maximum queue capacity in percentage relative to its parent queue| *---------------+--------------+-------------------------------+ | queueName | string | Name of the queue | *---------------+--------------+-------------------------------+ @@ -329,17 +329,17 @@ ResourceManager REST API's. *---------------+--------------+-------------------------------+ || Item || Data Type || Description | *---------------+--------------+-------------------------------+ -| capacity | float | Queue capacity in percentage relative to its parent queue | +| capacity | float | Configured queue capacity in percentage relative to its parent queue | *---------------+--------------+-------------------------------+ -| usedCapacity | float | Used queue capacity in percentage relative its to parent queue | +| usedCapacity | float | Used queue capacity in percentage | *---------------+--------------+-------------------------------+ -| maxCapacity | float | Maximum queue capacity in percentage relative to its parent queue | +| maxCapacity | float | Configured maximum queue capacity in percentage relative to its parent queue | *---------------+--------------+-------------------------------+ | absoluteCapacity | float | Absolute capacity percentage this queue can use of entire cluster | *---------------+--------------+-------------------------------+ | absoluteMaxCapacity | float | Absolute maximum capacity percentage this queue can use of the entire cluster | *---------------+--------------+-------------------------------+ -| utilization | float | Queue utilization percentage relative to the entire cluster | +| absoluteUsedCapacity | float | Absolute used capacity percentage this queue is using of the entire cluster | *---------------+--------------+-------------------------------+ | numApplications | int | The number of applications currently in the queue | *---------------+--------------+-------------------------------+ @@ -421,7 +421,7 @@ ResourceManager REST API's. "absoluteMaxCapacity" : 90, "maxActiveApplications" : 1, "numActiveApplications" : 0, - "utilization" : 0, + "absoluteUsedCapacity" : 0, "userLimit" : 100, "absoluteCapacity" : 70, "maxActiveApplicationsPerUser" : 1, @@ -431,7 +431,7 @@ ResourceManager REST API's. }, { "queueName" : "test", - "utilization" : 0, + "absoluteUsedCapacity" : 0, "absoluteCapacity" : 20, "usedCapacity" : 0, "capacity" : 20, @@ -450,7 +450,7 @@ ResourceManager REST API's. "absoluteMaxCapacity" : 16.000002, "maxActiveApplications" : 1, "numActiveApplications" : 0, - "utilization" : 0, + "absoluteUsedCapacity" : 0, "userLimit" : 100, "absoluteCapacity" : 12, "maxActiveApplicationsPerUser" : 1, @@ -472,7 +472,7 @@ ResourceManager REST API's. "absoluteMaxCapacity" : 100, "maxActiveApplications" : 1, "numActiveApplications" : 0, - "utilization" : 0, + "absoluteUsedCapacity" : 0, "userLimit" : 100, "absoluteCapacity" : 8.000001, "maxActiveApplicationsPerUser" : 1, @@ -489,7 +489,7 @@ ResourceManager REST API's. }, { "queueName" : "test2", - "utilization" : 0, + "absoluteUsedCapacity" : 0, "absoluteCapacity" : 10, "usedCapacity" : 0, "capacity" : 10, @@ -508,7 +508,7 @@ ResourceManager REST API's. "absoluteMaxCapacity" : 100, "maxActiveApplications" : 1, "numActiveApplications" : 0, - "utilization" : 0, + "absoluteUsedCapacity" : 0, "userLimit" : 100, "absoluteCapacity" : 5, "maxActiveApplicationsPerUser" : 1, @@ -530,7 +530,7 @@ ResourceManager REST API's. "absoluteMaxCapacity" : 100, "maxActiveApplications" : 1, "numActiveApplications" : 0, - "utilization" : 0, + "absoluteUsedCapacity" : 0, "userLimit" : 100, "absoluteCapacity" : 4.0000005, "maxActiveApplicationsPerUser" : 1, @@ -552,7 +552,7 @@ ResourceManager REST API's. "absoluteMaxCapacity" : 100, "maxActiveApplications" : 1, "numActiveApplications" : 0, - "utilization" : 0, + "absoluteUsedCapacity" : 0, "userLimit" : 100, "absoluteCapacity" : 1.0000001, "maxActiveApplicationsPerUser" : 1, @@ -609,7 +609,7 @@ ResourceManager REST API's. 90.0 70.0 90.0 - 0.0 + 0.0 0 memory: 0 default @@ -630,7 +630,7 @@ ResourceManager REST API's. 80.0 20.0 80.0 - 0.0 + 0.0 0 memory: 0 test @@ -641,7 +641,7 @@ ResourceManager REST API's. 80.0 12.0 16.000002 - 0.0 + 0.0 0 memory: 0 a1 @@ -662,7 +662,7 @@ ResourceManager REST API's. 100.0 8.000001 100.0 - 0.0 + 0.0 0 memory: 0 a2 @@ -684,7 +684,7 @@ ResourceManager REST API's. 15.000001 10.0 15.000001 - 0.0 + 0.0 0 memory: 0 test2 @@ -695,7 +695,7 @@ ResourceManager REST API's. 100.0 5.0 100.0 - 0.0 + 0.0 0 memory: 0 A4 @@ -716,7 +716,7 @@ ResourceManager REST API's. 100.0 4.0000005 100.0 - 0.0 + 0.0 0 memory: 0 a3 @@ -737,7 +737,7 @@ ResourceManager REST API's. 100.0 1.0000001 100.0 - 0.0 + 0.0 0 memory: 0 a4