MAPREDUCE-3816 capacity scheduler web ui bar graphs for used capacity wrong (tgraves via bobby)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1294808 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-02-28 20:06:53 +00:00
parent f83be7cb21
commit ffdf980b20
11 changed files with 140 additions and 126 deletions

View File

@ -197,9 +197,12 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-3922. Fixed build to not compile 32bit container-executor binary MAPREDUCE-3922. Fixed build to not compile 32bit container-executor binary
by default on all platforms. (Hitesh Shah via vinodkv) by default on all platforms. (Hitesh Shah via vinodkv)
MAPREDUCE-3790 Broken pipe on streaming job can lead to truncated output for MAPREDUCE-3790. Broken pipe on streaming job can lead to truncated output for
a successful job (Jason Lowe via bobby) a successful job (Jason Lowe via bobby)
MAPREDUCE-3816. capacity scheduler web ui bar graphs for used capacity wrong
(tgraves via bobby)
Release 0.23.1 - 2012-02-17 Release 0.23.1 - 2012-02-17
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

View File

@ -91,10 +91,16 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
public float getAbsoluteMaximumCapacity(); public float getAbsoluteMaximumCapacity();
/** /**
* Get the currently utilized capacity of the queue * Get the current absolute used capacity of the queue
* relative to it's parent queue. * relative to the entire cluster.
* @return the currently utilized capacity of the queue * @return queue absolute used capacity
* relative to it's parent queue */
public float getAbsoluteUsedCapacity();
/**
* Get the current used capacity of the queue
* and it's children (if any).
* @return queue used capacity
*/ */
public float getUsedCapacity(); public float getUsedCapacity();
@ -104,6 +110,12 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
*/ */
public void setUsedCapacity(float usedCapacity); public void setUsedCapacity(float usedCapacity);
/**
* Set absolute used capacity of the queue.
* @param absUsedCapacity absolute used capacity of the queue
*/
public void setAbsoluteUsedCapacity(float absUsedCapacity);
/** /**
* Get the currently utilized resources in the cluster * Get the currently utilized resources in the cluster
* by the queue and children (if any). * by the queue and children (if any).
@ -111,21 +123,6 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
*/ */
public Resource getUsedResources(); public Resource getUsedResources();
/**
* Get the current <em>utilization</em> of the queue
* and it's children (if any).
* Utilization is defined as the ratio of
* <em>used-capacity over configured-capacity</em> of the queue.
* @return queue utilization
*/
public float getUtilization();
/**
* Get the current <em>utilization</em> of the queue.
* @param utilization queue utilization
*/
public void setUtilization(float utilization);
/** /**
* Get the current run-state of the queue * Get the current run-state of the queue
* @return current run-state * @return current run-state

View File

@ -23,21 +23,25 @@ import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
class CSQueueUtils { class CSQueueUtils {
final static float EPSILON = 0.0001f;
public static void checkMaxCapacity(String queueName, public static void checkMaxCapacity(String queueName,
float capacity, float maximumCapacity) { float capacity, float maximumCapacity) {
if (maximumCapacity < 0.0f || maximumCapacity > 1.0f || if (maximumCapacity < 0.0f || maximumCapacity > 1.0f) {
maximumCapacity < capacity) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"Illegal value of maximumCapacity " + maximumCapacity + "Illegal value of maximumCapacity " + maximumCapacity +
" used in call to setMaxCapacity for queue " + queueName); " used in call to setMaxCapacity for queue " + queueName);
} }
if (maximumCapacity < capacity) {
throw new IllegalArgumentException(
"Illegal call to setMaxCapacity. " +
"Queue '" + queueName + "' has " +
"capacity (" + capacity + ") greater than " +
"maximumCapacity (" + maximumCapacity + ")" );
} }
public static void checkAbsoluteCapacities(String queueName,
float absCapacity, float absMaxCapacity) {
if (absMaxCapacity < (absCapacity - EPSILON)) {
throw new IllegalArgumentException("Illegal call to setMaxCapacity. "
+ "Queue '" + queueName + "' has " + "an absolute capacity (" + absCapacity
+ ") greater than " + "its absolute maximumCapacity (" + absMaxCapacity
+ ")");
}
} }
public static float computeAbsoluteMaximumCapacity( public static float computeAbsoluteMaximumCapacity(
@ -75,18 +79,16 @@ class CSQueueUtils {
final int usedMemory = childQueue.getUsedResources().getMemory(); final int usedMemory = childQueue.getUsedResources().getMemory();
float queueLimit = 0.0f; float queueLimit = 0.0f;
float utilization = 0.0f; float absoluteUsedCapacity = 0.0f;
float usedCapacity = 0.0f; float usedCapacity = 0.0f;
if (clusterMemory > 0) { if (clusterMemory > 0) {
queueLimit = clusterMemory * childQueue.getAbsoluteCapacity(); queueLimit = clusterMemory * childQueue.getAbsoluteCapacity();
final float parentAbsoluteCapacity = absoluteUsedCapacity = ((float)usedMemory / (float)clusterMemory);
(parentQueue == null) ? 1.0f : parentQueue.getAbsoluteCapacity(); usedCapacity = (usedMemory / queueLimit);
utilization = (usedMemory / queueLimit);
usedCapacity = (usedMemory / (clusterMemory * parentAbsoluteCapacity));
} }
childQueue.setUtilization(utilization);
childQueue.setUsedCapacity(usedCapacity); childQueue.setUsedCapacity(usedCapacity);
childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
int available = int available =
Math.max((roundUp(minimumAllocation, (int)queueLimit) - usedMemory), 0); Math.max((roundUp(minimumAllocation, (int)queueLimit) - usedMemory), 0);

View File

@ -91,9 +91,9 @@ implements ResourceScheduler, CapacitySchedulerContext {
static final Comparator<CSQueue> queueComparator = new Comparator<CSQueue>() { static final Comparator<CSQueue> queueComparator = new Comparator<CSQueue>() {
@Override @Override
public int compare(CSQueue q1, CSQueue q2) { public int compare(CSQueue q1, CSQueue q2) {
if (q1.getUtilization() < q2.getUtilization()) { if (q1.getUsedCapacity() < q2.getUsedCapacity()) {
return -1; return -1;
} else if (q1.getUtilization() > q2.getUtilization()) { } else if (q1.getUsedCapacity() > q2.getUsedCapacity()) {
return 1; return 1;
} }

View File

@ -80,6 +80,7 @@ public class LeafQueue implements CSQueue {
private float absoluteCapacity; private float absoluteCapacity;
private float maximumCapacity; private float maximumCapacity;
private float absoluteMaxCapacity; private float absoluteMaxCapacity;
private float absoluteUsedCapacity = 0.0f;
private int userLimit; private int userLimit;
private float userLimitFactor; private float userLimitFactor;
@ -91,7 +92,6 @@ public class LeafQueue implements CSQueue {
private int maxActiveApplicationsPerUser; private int maxActiveApplicationsPerUser;
private Resource usedResources = Resources.createResource(0); private Resource usedResources = Resources.createResource(0);
private float utilization = 0.0f;
private float usedCapacity = 0.0f; private float usedCapacity = 0.0f;
private volatile int numContainers; private volatile int numContainers;
@ -210,9 +210,11 @@ public class LeafQueue implements CSQueue {
{ {
// Sanity check // Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
float absCapacity = parent.getAbsoluteCapacity() * capacity;
CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absCapacity, absoluteMaxCapacity);
this.capacity = capacity; this.capacity = capacity;
this.absoluteCapacity = parent.getAbsoluteCapacity() * capacity; this.absoluteCapacity = absCapacity;
this.maximumCapacity = maximumCapacity; this.maximumCapacity = maximumCapacity;
this.absoluteMaxCapacity = absoluteMaxCapacity; this.absoluteMaxCapacity = absoluteMaxCapacity;
@ -274,12 +276,11 @@ public class LeafQueue implements CSQueue {
"(int)(maxActiveApplications * (userLimit / 100.0f) * " + "(int)(maxActiveApplications * (userLimit / 100.0f) * " +
"userLimitFactor)," + "userLimitFactor)," +
"1) ]" + "\n" + "1) ]" + "\n" +
"utilization = " + utilization +
" [= usedResourcesMemory / " +
"(clusterResourceMemory * absoluteCapacity)]" + "\n" +
"usedCapacity = " + usedCapacity + "usedCapacity = " + usedCapacity +
" [= usedResourcesMemory / " + " [= usedResourcesMemory / " +
"(clusterResourceMemory * parent.absoluteCapacity)]" + "\n" + "(clusterResourceMemory * absoluteCapacity)]" + "\n" +
"absoluteUsedCapacity = " + absoluteUsedCapacity +
" [= usedResourcesMemory / clusterResourceMemory]" + "\n" +
"maxAMResourcePercent = " + maxAMResourcePercent + "maxAMResourcePercent = " + maxAMResourcePercent +
" [= configuredMaximumAMResourcePercent ]" + "\n" + " [= configuredMaximumAMResourcePercent ]" + "\n" +
"minimumAllocationFactor = " + minimumAllocationFactor + "minimumAllocationFactor = " + minimumAllocationFactor +
@ -313,6 +314,11 @@ public class LeafQueue implements CSQueue {
return absoluteMaxCapacity; return absoluteMaxCapacity;
} }
@Override
public synchronized float getAbsoluteUsedCapacity() {
return absoluteUsedCapacity;
}
@Override @Override
public CSQueue getParent() { public CSQueue getParent() {
return parent; return parent;
@ -383,24 +389,21 @@ public class LeafQueue implements CSQueue {
return usedResources; return usedResources;
} }
@Override
public synchronized float getUtilization() {
return utilization;
}
@Override @Override
public List<CSQueue> getChildQueues() { public List<CSQueue> getChildQueues() {
return null; return null;
} }
public synchronized void setUtilization(float utilization) { @Override
this.utilization = utilization;
}
public synchronized void setUsedCapacity(float usedCapacity) { public synchronized void setUsedCapacity(float usedCapacity) {
this.usedCapacity = usedCapacity; this.usedCapacity = usedCapacity;
} }
@Override
public synchronized void setAbsoluteUsedCapacity(float absUsedCapacity) {
this.absoluteUsedCapacity = absUsedCapacity;
}
/** /**
* Set maximum capacity - used only for testing. * Set maximum capacity - used only for testing.
* @param maximumCapacity new max capacity * @param maximumCapacity new max capacity
@ -408,10 +411,11 @@ public class LeafQueue implements CSQueue {
synchronized void setMaxCapacity(float maximumCapacity) { synchronized void setMaxCapacity(float maximumCapacity) {
// Sanity check // Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absoluteCapacity, absMaxCapacity);
this.maximumCapacity = maximumCapacity; this.maximumCapacity = maximumCapacity;
this.absoluteMaxCapacity = this.absoluteMaxCapacity = absMaxCapacity;
CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
} }
/** /**
@ -516,7 +520,7 @@ public class LeafQueue implements CSQueue {
"absoluteCapacity=" + absoluteCapacity + ", " + "absoluteCapacity=" + absoluteCapacity + ", " +
"usedResources=" + usedResources.getMemory() + "MB, " + "usedResources=" + usedResources.getMemory() + "MB, " +
"usedCapacity=" + getUsedCapacity() + ", " + "usedCapacity=" + getUsedCapacity() + ", " +
"utilization=" + getUtilization() + ", " + "absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + ", " +
"numApps=" + getNumApplications() + ", " + "numApps=" + getNumApplications() + ", " +
"numContainers=" + getNumContainers(); "numContainers=" + getNumContainers();
} }
@ -1228,7 +1232,8 @@ public class LeafQueue implements CSQueue {
" container=" + container + " container=" + container +
" containerId=" + container.getId() + " containerId=" + container.getId() +
" queue=" + this + " queue=" + this +
" util=" + getUtilization() + " usedCapacity=" + getUsedCapacity() +
" absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
" used=" + usedResources + " used=" + usedResources +
" cluster=" + clusterResource); " cluster=" + clusterResource);
@ -1241,7 +1246,8 @@ public class LeafQueue implements CSQueue {
" application=" + application.getApplicationId() + " application=" + application.getApplicationId() +
" resource=" + request.getCapability() + " resource=" + request.getCapability() +
" queue=" + this.toString() + " queue=" + this.toString() +
" util=" + getUtilization() + " usedCapacity=" + getUsedCapacity() +
" absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
" used=" + usedResources + " used=" + usedResources +
" cluster=" + clusterResource); " cluster=" + clusterResource);
@ -1307,7 +1313,8 @@ public class LeafQueue implements CSQueue {
" container=" + container + " container=" + container +
" resource=" + container.getResource() + " resource=" + container.getResource() +
" queue=" + this + " queue=" + this +
" util=" + getUtilization() + " usedCapacity=" + getUsedCapacity() +
" absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
" used=" + usedResources + " used=" + usedResources +
" cluster=" + clusterResource); " cluster=" + clusterResource);
} }

View File

@ -67,9 +67,9 @@ public class ParentQueue implements CSQueue {
private float maximumCapacity; private float maximumCapacity;
private float absoluteCapacity; private float absoluteCapacity;
private float absoluteMaxCapacity; private float absoluteMaxCapacity;
private float absoluteUsedCapacity = 0.0f;
private float usedCapacity = 0.0f; private float usedCapacity = 0.0f;
private float utilization = 0.0f;
private final Set<CSQueue> childQueues; private final Set<CSQueue> childQueues;
private final Comparator<CSQueue> queueComparator; private final Comparator<CSQueue> queueComparator;
@ -158,9 +158,11 @@ public class ParentQueue implements CSQueue {
) { ) {
// Sanity check // Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absoluteCapacity, absoluteMaxCapacity);
this.capacity = capacity; this.capacity = capacity;
this.absoluteCapacity = absoluteCapacity; this.absoluteCapacity = absoluteCapacity;
this.maximumCapacity = maximumCapacity; this.maximumCapacity = maximumCapacity;
this.absoluteMaxCapacity = absoluteMaxCapacity; this.absoluteMaxCapacity = absoluteMaxCapacity;
@ -243,6 +245,11 @@ public class ParentQueue implements CSQueue {
return absoluteMaxCapacity; return absoluteMaxCapacity;
} }
@Override
public synchronized float getAbsoluteUsedCapacity() {
return absoluteUsedCapacity;
}
@Override @Override
public float getMaximumCapacity() { public float getMaximumCapacity() {
return maximumCapacity; return maximumCapacity;
@ -264,11 +271,6 @@ public class ParentQueue implements CSQueue {
return usedResources; return usedResources;
} }
@Override
public synchronized float getUtilization() {
return utilization;
}
@Override @Override
public synchronized List<CSQueue> getChildQueues() { public synchronized List<CSQueue> getChildQueues() {
return new ArrayList<CSQueue>(childQueues); return new ArrayList<CSQueue>(childQueues);
@ -351,7 +353,6 @@ public class ParentQueue implements CSQueue {
"absoluteCapacity=" + absoluteCapacity + ", " + "absoluteCapacity=" + absoluteCapacity + ", " +
"usedResources=" + usedResources.getMemory() + "MB, " + "usedResources=" + usedResources.getMemory() + "MB, " +
"usedCapacity=" + getUsedCapacity() + ", " + "usedCapacity=" + getUsedCapacity() + ", " +
"utilization=" + getUtilization() + ", " +
"numApps=" + getNumApplications() + ", " + "numApps=" + getNumApplications() + ", " +
"numContainers=" + getNumContainers(); "numContainers=" + getNumContainers();
} }
@ -490,12 +491,14 @@ public class ParentQueue implements CSQueue {
" #applications: " + getNumApplications()); " #applications: " + getNumApplications());
} }
@Override
public synchronized void setUsedCapacity(float usedCapacity) { public synchronized void setUsedCapacity(float usedCapacity) {
this.usedCapacity = usedCapacity; this.usedCapacity = usedCapacity;
} }
public synchronized void setUtilization(float utilization) { @Override
this.utilization = utilization; public synchronized void setAbsoluteUsedCapacity(float absUsedCapacity) {
this.absoluteUsedCapacity = absUsedCapacity;
} }
/** /**
@ -505,10 +508,11 @@ public class ParentQueue implements CSQueue {
synchronized void setMaxCapacity(float maximumCapacity) { synchronized void setMaxCapacity(float maximumCapacity) {
// Sanity check // Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
CSQueueUtils.checkAbsoluteCapacities(getQueueName(), absoluteCapacity, absMaxCapacity);
this.maximumCapacity = maximumCapacity; this.maximumCapacity = maximumCapacity;
this.absoluteMaxCapacity = this.absoluteMaxCapacity = absMaxCapacity;
CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
} }
@Override @Override
@ -545,7 +549,8 @@ public class ParentQueue implements CSQueue {
LOG.info("assignedContainer" + LOG.info("assignedContainer" +
" queue=" + getQueueName() + " queue=" + getQueueName() +
" util=" + getUtilization() + " usedCapacity=" + getUsedCapacity() +
" absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
" used=" + usedResources + " used=" + usedResources +
" cluster=" + clusterResource); " cluster=" + clusterResource);
@ -556,7 +561,8 @@ public class ParentQueue implements CSQueue {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("ParentQ=" + getQueueName() LOG.debug("ParentQ=" + getQueueName()
+ " assignedSoFarInThisIteration=" + assignment.getResource() + " assignedSoFarInThisIteration=" + assignment.getResource()
+ " utilization=" + getUtilization()); + " usedCapacity=" + getUsedCapacity()
+ " absoluteUsedCapacity=" + getAbsoluteUsedCapacity());
} }
// Do not assign more than one container if this isn't the root queue // Do not assign more than one container if this isn't the root queue
@ -639,7 +645,7 @@ public class ParentQueue implements CSQueue {
String getChildQueuesToPrint() { String getChildQueuesToPrint() {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
for (CSQueue q : childQueues) { for (CSQueue q : childQueues) {
sb.append(q.getQueuePath() + "(" + q.getUtilization() + "), "); sb.append(q.getQueuePath() + "(" + q.getUsedCapacity() + "), ");
} }
return sb.toString(); return sb.toString();
} }
@ -663,7 +669,8 @@ public class ParentQueue implements CSQueue {
LOG.info("completedContainer" + LOG.info("completedContainer" +
" queue=" + getQueueName() + " queue=" + getQueueName() +
" util=" + getUtilization() + " usedCapacity=" + getUsedCapacity() +
" absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
" used=" + usedResources + " used=" + usedResources +
" cluster=" + clusterResource); " cluster=" + clusterResource);
} }

View File

@ -67,12 +67,9 @@ class CapacitySchedulerPage extends RmView {
protected void render(Block html) { protected void render(Block html) {
ResponseInfo ri = info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status"). ResponseInfo ri = info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status").
_("Queue State:", lqinfo.getQueueState()). _("Queue State:", lqinfo.getQueueState()).
_("Capacity:", percent(lqinfo.getCapacity() / 100)).
_("Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)).
_("Used Capacity:", percent(lqinfo.getUsedCapacity() / 100)). _("Used Capacity:", percent(lqinfo.getUsedCapacity() / 100)).
_("Absolute Capacity:", percent(lqinfo.getAbsoluteCapacity() / 100)). _("Absolute Capacity:", percent(lqinfo.getAbsoluteCapacity() / 100)).
_("Absolute Max Capacity:", percent(lqinfo.getAbsoluteMaxCapacity() / 100)). _("Absolute Max Capacity:", percent(lqinfo.getAbsoluteMaxCapacity() / 100)).
_("Utilization:", percent(lqinfo.getUtilization() / 100)).
_("Used Resources:", lqinfo.getUsedResources().toString()). _("Used Resources:", lqinfo.getUsedResources().toString()).
_("Num Active Applications:", Integer.toString(lqinfo.getNumActiveApplications())). _("Num Active Applications:", Integer.toString(lqinfo.getNumActiveApplications())).
_("Num Pending Applications:", Integer.toString(lqinfo.getNumPendingApplications())). _("Num Pending Applications:", Integer.toString(lqinfo.getNumPendingApplications())).
@ -81,8 +78,10 @@ class CapacitySchedulerPage extends RmView {
_("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())). _("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())).
_("Max Active Applications:", Integer.toString(lqinfo.getMaxActiveApplications())). _("Max Active Applications:", Integer.toString(lqinfo.getMaxActiveApplications())).
_("Max Active Applications Per User:", Integer.toString(lqinfo.getMaxActiveApplicationsPerUser())). _("Max Active Applications Per User:", Integer.toString(lqinfo.getMaxActiveApplicationsPerUser())).
_("User Limit:", Integer.toString(lqinfo.getUserLimit()) + "%"). _("Configured Capacity:", percent(lqinfo.getCapacity() / 100)).
_("User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())); _("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)).
_("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%").
_("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor()));
html._(InfoBlock.class); html._(InfoBlock.class);
@ -103,20 +102,20 @@ class CapacitySchedulerPage extends RmView {
ArrayList<CapacitySchedulerQueueInfo> subQueues = ArrayList<CapacitySchedulerQueueInfo> subQueues =
(csqinfo.qinfo == null) ? csqinfo.csinfo.getSubQueues() (csqinfo.qinfo == null) ? csqinfo.csinfo.getSubQueues()
: csqinfo.qinfo.getSubQueues(); : csqinfo.qinfo.getSubQueues();
UL<Hamlet> ul = html.ul(); UL<Hamlet> ul = html.ul("#pq");
for (CapacitySchedulerQueueInfo info : subQueues) { for (CapacitySchedulerQueueInfo info : subQueues) {
float used = info.getUsedCapacity() / 100; float used = info.getUsedCapacity() / 100;
float set = info.getCapacity() / 100; float absCap = info.getAbsoluteCapacity() / 100;
float max = info.getMaxCapacity() / 100; float absMaxCap = info.getAbsoluteMaxCapacity() / 100;
float absUsedCap = info.getAbsoluteUsedCapacity() / 100;
LI<UL<Hamlet>> li = ul. LI<UL<Hamlet>> li = ul.
li(). li().
a(_Q).$style(width(max * Q_MAX_WIDTH)). a(_Q).$style(width(absMaxCap * Q_MAX_WIDTH)).
$title(join("capacity:", percent(set), " used:", percent(used), $title(join("Absolute Capacity:", percent(absCap))).
" max capacity:", percent(max))). span().$style(join(Q_GIVEN, ";font-size:1px;", width(absCap/absMaxCap))).
span().$style(join(Q_GIVEN, ";font-size:1px;", width(set/max))).
_('.')._(). _('.')._().
span().$style(join(width(used*set/max), span().$style(join(width(absUsedCap/absMaxCap),
";font-size:1px;left:0%;", used > 1 ? Q_OVER : Q_UNDER)). ";font-size:1px;left:0%;", absUsedCap > absCap ? Q_OVER : Q_UNDER)).
_('.')._(). _('.')._().
span(".q", info.getQueuePath().substring(5))._(). span(".q", info.getQueuePath().substring(5))._().
span().$class("qstats").$style(left(Q_STATS_POS)). span().$class("qstats").$style(left(Q_STATS_POS)).
@ -180,7 +179,6 @@ class CapacitySchedulerPage extends RmView {
_(). _().
li(). li().
a(_Q).$style(width(Q_MAX_WIDTH)). a(_Q).$style(width(Q_MAX_WIDTH)).
$title(join("used:", percent(used))).
span().$style(join(width(used), ";left:0%;", span().$style(join(width(used), ";left:0%;",
used > 1 ? Q_OVER : Q_UNDER))._(".")._(). used > 1 ? Q_OVER : Q_UNDER))._(".")._().
span(".q", "root")._(). span(".q", "root")._().
@ -211,8 +209,7 @@ class CapacitySchedulerPage extends RmView {
_("$(function() {", _("$(function() {",
" $('#cs a span').addClass('ui-corner-all').css('position', 'absolute');", " $('#cs a span').addClass('ui-corner-all').css('position', 'absolute');",
" $('#cs').bind('loaded.jstree', function (e, data) {", " $('#cs').bind('loaded.jstree', function (e, data) {",
" data.inst.open_all();", " data.inst.open_node('#pq', true);",
" data.inst.close_node('#lq', true);",
" }).", " }).",
" jstree({", " jstree({",
" core: { animation: 188, html_titles: true },", " core: { animation: 188, html_titles: true },",

View File

@ -44,7 +44,7 @@ public class CapacitySchedulerQueueInfo {
protected float maxCapacity; protected float maxCapacity;
protected float absoluteCapacity; protected float absoluteCapacity;
protected float absoluteMaxCapacity; protected float absoluteMaxCapacity;
protected float utilization; protected float absoluteUsedCapacity;
protected int numApplications; protected int numApplications;
protected String usedResources; protected String usedResources;
protected String queueName; protected String queueName;
@ -66,7 +66,7 @@ public class CapacitySchedulerQueueInfo {
absoluteCapacity = cap(q.getAbsoluteCapacity(), 0f, 1f) * 100; absoluteCapacity = cap(q.getAbsoluteCapacity(), 0f, 1f) * 100;
absoluteMaxCapacity = cap(q.getAbsoluteMaximumCapacity(), 0f, 1f) * 100; absoluteMaxCapacity = cap(q.getAbsoluteMaximumCapacity(), 0f, 1f) * 100;
utilization = q.getUtilization() * 100; absoluteUsedCapacity = cap(q.getAbsoluteUsedCapacity(), 0f, 1f) * 100;
numApplications = q.getNumApplications(); numApplications = q.getNumApplications();
usedResources = q.getUsedResources().toString(); usedResources = q.getUsedResources().toString();
queueName = q.getQueueName(); queueName = q.getQueueName();
@ -93,8 +93,8 @@ public class CapacitySchedulerQueueInfo {
return absoluteMaxCapacity; return absoluteMaxCapacity;
} }
public float getUtilization() { public float getAbsoluteUsedCapacity() {
return utilization; return absoluteUsedCapacity;
} }
public int getNumApplications() { public int getNumApplications() {

View File

@ -138,15 +138,14 @@ public class TestParentQueue {
when(queue).assignContainers(eq(clusterResource), eq(node)); when(queue).assignContainers(eq(clusterResource), eq(node));
} }
private float computeQueueUsedCapacity(CSQueue queue, private float computeQueueAbsoluteUsedCapacity(CSQueue queue,
int expectedMemory, Resource clusterResource) { int expectedMemory, Resource clusterResource) {
return ( return (
((float)expectedMemory / clusterResource.getMemory()) * ((float)expectedMemory / (float)clusterResource.getMemory())
queue.getParent().getAbsoluteCapacity()
); );
} }
private float computeQueueUtilization(CSQueue queue, private float computeQueueUsedCapacity(CSQueue queue,
int expectedMemory, Resource clusterResource) { int expectedMemory, Resource clusterResource) {
return (expectedMemory / return (expectedMemory /
(clusterResource.getMemory() * queue.getAbsoluteCapacity())); (clusterResource.getMemory() * queue.getAbsoluteCapacity()));
@ -156,8 +155,8 @@ public class TestParentQueue {
private void verifyQueueMetrics(CSQueue queue, private void verifyQueueMetrics(CSQueue queue,
int expectedMemory, Resource clusterResource) { int expectedMemory, Resource clusterResource) {
assertEquals( assertEquals(
computeQueueUtilization(queue, expectedMemory, clusterResource), computeQueueAbsoluteUsedCapacity(queue, expectedMemory, clusterResource),
queue.getUtilization(), queue.getAbsoluteUsedCapacity(),
DELTA); DELTA);
assertEquals( assertEquals(
computeQueueUsedCapacity(queue, expectedMemory, clusterResource), computeQueueUsedCapacity(queue, expectedMemory, clusterResource),

View File

@ -68,7 +68,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
float maxCapacity; float maxCapacity;
float absoluteCapacity; float absoluteCapacity;
float absoluteMaxCapacity; float absoluteMaxCapacity;
float utilization; float absoluteUsedCapacity;
int numApplications; int numApplications;
String usedResources; String usedResources;
String queueName; String queueName;
@ -252,7 +252,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
qi.absoluteCapacity = WebServicesTestUtils.getXmlFloat(qElem, "absoluteCapacity"); qi.absoluteCapacity = WebServicesTestUtils.getXmlFloat(qElem, "absoluteCapacity");
qi.absoluteMaxCapacity = qi.absoluteMaxCapacity =
WebServicesTestUtils.getXmlFloat(qElem, "absoluteMaxCapacity"); WebServicesTestUtils.getXmlFloat(qElem, "absoluteMaxCapacity");
qi.utilization = WebServicesTestUtils.getXmlFloat(qElem, "utilization"); qi.absoluteUsedCapacity =
WebServicesTestUtils.getXmlFloat(qElem, "absoluteUsedCapacity");
qi.numApplications = qi.numApplications =
WebServicesTestUtils.getXmlInt(qElem, "numApplications"); WebServicesTestUtils.getXmlInt(qElem, "numApplications");
qi.usedResources = qi.usedResources =
@ -342,7 +343,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
qi.maxCapacity = (float) info.getDouble("maxCapacity"); qi.maxCapacity = (float) info.getDouble("maxCapacity");
qi.absoluteCapacity = (float) info.getDouble("absoluteCapacity"); qi.absoluteCapacity = (float) info.getDouble("absoluteCapacity");
qi.absoluteMaxCapacity = (float) info.getDouble("absoluteMaxCapacity"); qi.absoluteMaxCapacity = (float) info.getDouble("absoluteMaxCapacity");
qi.utilization = (float) info.getDouble("utilization"); qi.absoluteUsedCapacity = (float) info.getDouble("absoluteUsedCapacity");
qi.numApplications = info.getInt("numApplications"); qi.numApplications = info.getInt("numApplications");
qi.usedResources = info.getString("usedResources"); qi.usedResources = info.getString("usedResources");
qi.queueName = info.getString("queueName"); qi.queueName = info.getString("queueName");
@ -394,7 +395,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
parentAbsCapacity * (info.capacity/100), info.absoluteCapacity, 1e-3f); parentAbsCapacity * (info.capacity/100), info.absoluteCapacity, 1e-3f);
assertEquals("absoluteMaxCapacity doesn't match", assertEquals("absoluteMaxCapacity doesn't match",
expectAbsMaxCapacity, info.absoluteMaxCapacity, 1e-3f); expectAbsMaxCapacity, info.absoluteMaxCapacity, 1e-3f);
assertEquals("utilization doesn't match", 0, info.utilization, 1e-3f); assertEquals("absoluteUsedCapacity doesn't match",
0, info.absoluteUsedCapacity, 1e-3f);
assertEquals("numApplications doesn't match", 0, info.numApplications); assertEquals("numApplications doesn't match", 0, info.numApplications);
assertTrue("usedResources doesn't match", assertTrue("usedResources doesn't match",
info.usedResources.matches("memory: 0")); info.usedResources.matches("memory: 0"));

View File

@ -313,11 +313,11 @@ ResourceManager REST API's.
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| type | string | Scheduler type - capacityScheduler| | type | string | Scheduler type - capacityScheduler|
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| capacity | float | Queue capacity in percentage relative to its parent queue | | capacity | float | Configured queue capacity in percentage relative to its parent queue |
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| usedCapacity | float | Used queue capacity in percentage relative its to parent queue | | usedCapacity | float | Used queue capacity in percentage |
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| maxCapacity | float | Maximum queue capacity in percentage relative to its parent queue | | maxCapacity | float | Configured maximum queue capacity in percentage relative to its parent queue|
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| queueName | string | Name of the queue | | queueName | string | Name of the queue |
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
@ -329,17 +329,17 @@ ResourceManager REST API's.
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
|| Item || Data Type || Description | || Item || Data Type || Description |
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| capacity | float | Queue capacity in percentage relative to its parent queue | | capacity | float | Configured queue capacity in percentage relative to its parent queue |
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| usedCapacity | float | Used queue capacity in percentage relative its to parent queue | | usedCapacity | float | Used queue capacity in percentage |
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| maxCapacity | float | Maximum queue capacity in percentage relative to its parent queue | | maxCapacity | float | Configured maximum queue capacity in percentage relative to its parent queue |
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| absoluteCapacity | float | Absolute capacity percentage this queue can use of entire cluster | | absoluteCapacity | float | Absolute capacity percentage this queue can use of entire cluster |
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| absoluteMaxCapacity | float | Absolute maximum capacity percentage this queue can use of the entire cluster | | absoluteMaxCapacity | float | Absolute maximum capacity percentage this queue can use of the entire cluster |
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| utilization | float | Queue utilization percentage relative to the entire cluster | | absoluteUsedCapacity | float | Absolute used capacity percentage this queue is using of the entire cluster |
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
| numApplications | int | The number of applications currently in the queue | | numApplications | int | The number of applications currently in the queue |
*---------------+--------------+-------------------------------+ *---------------+--------------+-------------------------------+
@ -421,7 +421,7 @@ ResourceManager REST API's.
"absoluteMaxCapacity" : 90, "absoluteMaxCapacity" : 90,
"maxActiveApplications" : 1, "maxActiveApplications" : 1,
"numActiveApplications" : 0, "numActiveApplications" : 0,
"utilization" : 0, "absoluteUsedCapacity" : 0,
"userLimit" : 100, "userLimit" : 100,
"absoluteCapacity" : 70, "absoluteCapacity" : 70,
"maxActiveApplicationsPerUser" : 1, "maxActiveApplicationsPerUser" : 1,
@ -431,7 +431,7 @@ ResourceManager REST API's.
}, },
{ {
"queueName" : "test", "queueName" : "test",
"utilization" : 0, "absoluteUsedCapacity" : 0,
"absoluteCapacity" : 20, "absoluteCapacity" : 20,
"usedCapacity" : 0, "usedCapacity" : 0,
"capacity" : 20, "capacity" : 20,
@ -450,7 +450,7 @@ ResourceManager REST API's.
"absoluteMaxCapacity" : 16.000002, "absoluteMaxCapacity" : 16.000002,
"maxActiveApplications" : 1, "maxActiveApplications" : 1,
"numActiveApplications" : 0, "numActiveApplications" : 0,
"utilization" : 0, "absoluteUsedCapacity" : 0,
"userLimit" : 100, "userLimit" : 100,
"absoluteCapacity" : 12, "absoluteCapacity" : 12,
"maxActiveApplicationsPerUser" : 1, "maxActiveApplicationsPerUser" : 1,
@ -472,7 +472,7 @@ ResourceManager REST API's.
"absoluteMaxCapacity" : 100, "absoluteMaxCapacity" : 100,
"maxActiveApplications" : 1, "maxActiveApplications" : 1,
"numActiveApplications" : 0, "numActiveApplications" : 0,
"utilization" : 0, "absoluteUsedCapacity" : 0,
"userLimit" : 100, "userLimit" : 100,
"absoluteCapacity" : 8.000001, "absoluteCapacity" : 8.000001,
"maxActiveApplicationsPerUser" : 1, "maxActiveApplicationsPerUser" : 1,
@ -489,7 +489,7 @@ ResourceManager REST API's.
}, },
{ {
"queueName" : "test2", "queueName" : "test2",
"utilization" : 0, "absoluteUsedCapacity" : 0,
"absoluteCapacity" : 10, "absoluteCapacity" : 10,
"usedCapacity" : 0, "usedCapacity" : 0,
"capacity" : 10, "capacity" : 10,
@ -508,7 +508,7 @@ ResourceManager REST API's.
"absoluteMaxCapacity" : 100, "absoluteMaxCapacity" : 100,
"maxActiveApplications" : 1, "maxActiveApplications" : 1,
"numActiveApplications" : 0, "numActiveApplications" : 0,
"utilization" : 0, "absoluteUsedCapacity" : 0,
"userLimit" : 100, "userLimit" : 100,
"absoluteCapacity" : 5, "absoluteCapacity" : 5,
"maxActiveApplicationsPerUser" : 1, "maxActiveApplicationsPerUser" : 1,
@ -530,7 +530,7 @@ ResourceManager REST API's.
"absoluteMaxCapacity" : 100, "absoluteMaxCapacity" : 100,
"maxActiveApplications" : 1, "maxActiveApplications" : 1,
"numActiveApplications" : 0, "numActiveApplications" : 0,
"utilization" : 0, "absoluteUsedCapacity" : 0,
"userLimit" : 100, "userLimit" : 100,
"absoluteCapacity" : 4.0000005, "absoluteCapacity" : 4.0000005,
"maxActiveApplicationsPerUser" : 1, "maxActiveApplicationsPerUser" : 1,
@ -552,7 +552,7 @@ ResourceManager REST API's.
"absoluteMaxCapacity" : 100, "absoluteMaxCapacity" : 100,
"maxActiveApplications" : 1, "maxActiveApplications" : 1,
"numActiveApplications" : 0, "numActiveApplications" : 0,
"utilization" : 0, "absoluteUsedCapacity" : 0,
"userLimit" : 100, "userLimit" : 100,
"absoluteCapacity" : 1.0000001, "absoluteCapacity" : 1.0000001,
"maxActiveApplicationsPerUser" : 1, "maxActiveApplicationsPerUser" : 1,
@ -609,7 +609,7 @@ ResourceManager REST API's.
<maxCapacity>90.0</maxCapacity> <maxCapacity>90.0</maxCapacity>
<absoluteCapacity>70.0</absoluteCapacity> <absoluteCapacity>70.0</absoluteCapacity>
<absoluteMaxCapacity>90.0</absoluteMaxCapacity> <absoluteMaxCapacity>90.0</absoluteMaxCapacity>
<utilization>0.0</utilization> <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
<numApplications>0</numApplications> <numApplications>0</numApplications>
<usedResources>memory: 0</usedResources> <usedResources>memory: 0</usedResources>
<queueName>default</queueName> <queueName>default</queueName>
@ -630,7 +630,7 @@ ResourceManager REST API's.
<maxCapacity>80.0</maxCapacity> <maxCapacity>80.0</maxCapacity>
<absoluteCapacity>20.0</absoluteCapacity> <absoluteCapacity>20.0</absoluteCapacity>
<absoluteMaxCapacity>80.0</absoluteMaxCapacity> <absoluteMaxCapacity>80.0</absoluteMaxCapacity>
<utilization>0.0</utilization> <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
<numApplications>0</numApplications> <numApplications>0</numApplications>
<usedResources>memory: 0</usedResources> <usedResources>memory: 0</usedResources>
<queueName>test</queueName> <queueName>test</queueName>
@ -641,7 +641,7 @@ ResourceManager REST API's.
<maxCapacity>80.0</maxCapacity> <maxCapacity>80.0</maxCapacity>
<absoluteCapacity>12.0</absoluteCapacity> <absoluteCapacity>12.0</absoluteCapacity>
<absoluteMaxCapacity>16.000002</absoluteMaxCapacity> <absoluteMaxCapacity>16.000002</absoluteMaxCapacity>
<utilization>0.0</utilization> <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
<numApplications>0</numApplications> <numApplications>0</numApplications>
<usedResources>memory: 0</usedResources> <usedResources>memory: 0</usedResources>
<queueName>a1</queueName> <queueName>a1</queueName>
@ -662,7 +662,7 @@ ResourceManager REST API's.
<maxCapacity>100.0</maxCapacity> <maxCapacity>100.0</maxCapacity>
<absoluteCapacity>8.000001</absoluteCapacity> <absoluteCapacity>8.000001</absoluteCapacity>
<absoluteMaxCapacity>100.0</absoluteMaxCapacity> <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
<utilization>0.0</utilization> <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
<numApplications>0</numApplications> <numApplications>0</numApplications>
<usedResources>memory: 0</usedResources> <usedResources>memory: 0</usedResources>
<queueName>a2</queueName> <queueName>a2</queueName>
@ -684,7 +684,7 @@ ResourceManager REST API's.
<maxCapacity>15.000001</maxCapacity> <maxCapacity>15.000001</maxCapacity>
<absoluteCapacity>10.0</absoluteCapacity> <absoluteCapacity>10.0</absoluteCapacity>
<absoluteMaxCapacity>15.000001</absoluteMaxCapacity> <absoluteMaxCapacity>15.000001</absoluteMaxCapacity>
<utilization>0.0</utilization> <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
<numApplications>0</numApplications> <numApplications>0</numApplications>
<usedResources>memory: 0</usedResources> <usedResources>memory: 0</usedResources>
<queueName>test2</queueName> <queueName>test2</queueName>
@ -695,7 +695,7 @@ ResourceManager REST API's.
<maxCapacity>100.0</maxCapacity> <maxCapacity>100.0</maxCapacity>
<absoluteCapacity>5.0</absoluteCapacity> <absoluteCapacity>5.0</absoluteCapacity>
<absoluteMaxCapacity>100.0</absoluteMaxCapacity> <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
<utilization>0.0</utilization> <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
<numApplications>0</numApplications> <numApplications>0</numApplications>
<usedResources>memory: 0</usedResources> <usedResources>memory: 0</usedResources>
<queueName>A4</queueName> <queueName>A4</queueName>
@ -716,7 +716,7 @@ ResourceManager REST API's.
<maxCapacity>100.0</maxCapacity> <maxCapacity>100.0</maxCapacity>
<absoluteCapacity>4.0000005</absoluteCapacity> <absoluteCapacity>4.0000005</absoluteCapacity>
<absoluteMaxCapacity>100.0</absoluteMaxCapacity> <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
<utilization>0.0</utilization> <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
<numApplications>0</numApplications> <numApplications>0</numApplications>
<usedResources>memory: 0</usedResources> <usedResources>memory: 0</usedResources>
<queueName>a3</queueName> <queueName>a3</queueName>
@ -737,7 +737,7 @@ ResourceManager REST API's.
<maxCapacity>100.0</maxCapacity> <maxCapacity>100.0</maxCapacity>
<absoluteCapacity>1.0000001</absoluteCapacity> <absoluteCapacity>1.0000001</absoluteCapacity>
<absoluteMaxCapacity>100.0</absoluteMaxCapacity> <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
<utilization>0.0</utilization> <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
<numApplications>0</numApplications> <numApplications>0</numApplications>
<usedResources>memory: 0</usedResources> <usedResources>memory: 0</usedResources>
<queueName>a4</queueName> <queueName>a4</queueName>