MAPREDUCE-3625. CapacityScheduler web-ui display of queue's used capacity is broken. (Jason Lowe via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1230336 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -473,6 +473,9 @@ Release 0.23.1 - Unreleased
     MAPREDUCE-3652. org.apache.hadoop.mapred.TestWebUIAuthorization.testWebUIAuthorization 
     fails. (Thomas Graves via mahadev)
 
+    MAPREDUCE-3625. CapacityScheduler web-ui display of queue's used capacity is broken.
+    (Jason Lowe via mahadev)
+
 Release 0.23.0 - 2011-11-01 
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java

@@ -42,11 +42,12 @@ import com.google.inject.servlet.RequestScoped;
 
 class CapacitySchedulerPage extends RmView {
   static final String _Q = ".ui-state-default.ui-corner-all";
-  static final float WIDTH_F = 0.8f;
+  static final float Q_MAX_WIDTH = 0.8f;
+  static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f;
   static final String Q_END = "left:101%";
-  static final String OVER = "font-size:1px;background:rgba(255, 140, 0, 0.8)";
+  static final String Q_GIVEN = "left:0%;background:none;border:1px dashed rgba(0,0,0,0.25)";
-  static final String UNDER = "font-size:1px;background:rgba(50, 205, 50, 0.8)";
+  static final String Q_OVER = "background:rgba(255, 140, 0, 0.8)";
-  static final float EPSILON = 1e-8f;
+  static final String Q_UNDER = "background:rgba(50, 205, 50, 0.8)";
 
   @RequestScoped
   static class CSQInfo {
@@ -106,18 +107,20 @@ class CapacitySchedulerPage extends RmView {
       for (CapacitySchedulerQueueInfo info : subQueues) {
         float used = info.getUsedCapacity() / 100;
         float set = info.getCapacity() / 100;
-        float delta = Math.abs(set - used) + 0.001f;
         float max = info.getMaxCapacity() / 100;
         LI<UL<Hamlet>> li = ul.
           li().
-            a(_Q).$style(width(max * WIDTH_F)).
+            a(_Q).$style(width(max * Q_MAX_WIDTH)).
-              $title(join("used:", percent(used), " set:", percent(set),
+              $title(join("capacity:", percent(set), " used:", percent(used),
-                          " max:", percent(max))).
+                          " max capacity:", percent(max))).
-              //span().$style(Q_END)._(absMaxPct)._().
+              span().$style(join(Q_GIVEN, ";font-size:1px;", width(set/max))).
-              span().$style(join(width(delta/max), ';',
+                _('.')._().
-                used > set ? OVER : UNDER, ';',
+              span().$style(join(width(used*set/max),
-                used > set ? left(set/max) : left(used/max)))._('.')._().
+                ";font-size:1px;left:0%;", used > 1 ? Q_OVER : Q_UNDER)).
-              span(".q", info.getQueuePath().substring(5))._();
+                _('.')._().
+              span(".q", info.getQueuePath().substring(5))._().
+            span().$class("qstats").$style(left(Q_STATS_POS)).
+              _(join(percent(used), " used"))._();
 
         csqinfo.qinfo = info;
         if (info.getSubQueues() == null) {
@@ -153,7 +156,7 @@ class CapacitySchedulerPage extends RmView {
       if (cs == null) {
         ul.
           li().
-            a(_Q).$style(width(WIDTH_F)).
+            a(_Q).$style(width(Q_MAX_WIDTH)).
               span().$style(Q_END)._("100% ")._().
               span(".q", "default")._()._();
       } else {
@@ -163,16 +166,26 @@ class CapacitySchedulerPage extends RmView {
         csqinfo.qinfo = null;
 
         float used = sinfo.getUsedCapacity() / 100;
-        float set = sinfo.getCapacity() / 100;
-        float delta = Math.abs(set - used) + 0.001f;
         ul.
+          li().$style("margin-bottom: 1em").
+            span().$style("font-weight: bold")._("Legend:")._().
+            span().$class("qlegend ui-corner-all").$style(Q_GIVEN).
+              _("Capacity")._().
+            span().$class("qlegend ui-corner-all").$style(Q_UNDER).
+              _("Used")._().
+            span().$class("qlegend ui-corner-all").$style(Q_OVER).
+              _("Used (over capacity)")._().
+            span().$class("qlegend ui-corner-all ui-state-default").
+              _("Max Capacity")._().
+          _().
           li().
-            a(_Q).$style(width(WIDTH_F)).
+            a(_Q).$style(width(Q_MAX_WIDTH)).
               $title(join("used:", percent(used))).
-              span().$style(Q_END)._("100%")._().
+              span().$style(join(width(used), ";left:0%;",
-              span().$style(join(width(delta), ';', used > set ? OVER : UNDER,
+                  used > 1 ? Q_OVER : Q_UNDER))._(".")._().
-                ';', used > set ? left(set) : left(used)))._(".")._().
               span(".q", "root")._().
+            span().$class("qstats").$style(left(Q_STATS_POS)).
+              _(join(percent(used), " used"))._().
             _(QueueBlock.class)._();
       }
       ul._()._().
@@ -190,6 +203,8 @@ class CapacitySchedulerPage extends RmView {
           "#cs a { font-weight: normal; margin: 2px; position: relative }",
           "#cs a span { font-weight: normal; font-size: 80% }",
           "#cs-wrapper .ui-widget-header { padding: 0.2em 0.5em }",
+          ".qstats { font-weight: normal; font-size: 80%; position: absolute }",
+          ".qlegend { font-weight: normal; padding: 0 1em; margin: 1em }",
           "table.info tr th {width: 50%}")._(). // to center info table
       script("/static/jt/jquery.jstree.js").
       script().$type("text/javascript").

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java

@@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlSeeAlso;
 import javax.xml.bind.annotation.XmlTransient;
 
+import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 
 @XmlRootElement
@@ -47,7 +48,7 @@ public class CapacitySchedulerQueueInfo {
   protected int numApplications;
   protected String usedResources;
   protected String queueName;
-  protected String state;
+  protected QueueState state;
   protected ArrayList<CapacitySchedulerQueueInfo> subQueues;
 
   CapacitySchedulerQueueInfo() {
@@ -69,7 +70,7 @@ public class CapacitySchedulerQueueInfo {
     numApplications = q.getNumApplications();
     usedResources = q.getUsedResources().toString();
     queueName = q.getQueueName();
-    state = q.getState().toString();
+    state = q.getState();
   }
 
   public float getCapacity() {
@@ -109,7 +110,7 @@ public class CapacitySchedulerQueueInfo {
   }
 
   public String getQueueState() {
-    return this.state;
+    return this.state.toString();
   }
 
   public String getQueuePath() {

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java

@@ -62,6 +62,31 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
   private static MockRM rm;
   private CapacitySchedulerConfiguration csConf;
 
+  private class QueueInfo {
+    float capacity;
+    float usedCapacity;
+    float maxCapacity;
+    float absoluteCapacity;
+    float absoluteMaxCapacity;
+    float utilization;
+    int numApplications;
+    String usedResources;
+    String queueName;
+    String state;
+  }
+
+  private class LeafQueueInfo extends QueueInfo {
+    int numActiveApplications;
+    int numPendingApplications;
+    int numContainers;
+    int maxApplications;
+    int maxApplicationsPerUser;
+    int maxActiveApplications;
+    int maxActiveApplicationsPerUser;
+    int userLimit;
+    float userLimitFactor;
+  }
+
   private Injector injector = Guice.createInjector(new ServletModule() {
     @Override
     protected void configureServlets() {
@@ -217,29 +242,51 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
 
   public void verifySubQueueXML(Element qElem, String q, float parentAbsCapacity)
       throws Exception {
-    float absCapacity = WebServicesTestUtils.getXmlFloat(qElem, "absoluteCapacity");
-    verifySubQueueGeneric(q,
-        WebServicesTestUtils.getXmlFloat(qElem, "usedCapacity"),
-        WebServicesTestUtils.getXmlFloat(qElem, "capacity"),
-        WebServicesTestUtils.getXmlFloat(qElem, "maxCapacity"),
-        absCapacity,
-        WebServicesTestUtils.getXmlFloat(qElem, "absoluteMaxCapacity"),
-        parentAbsCapacity,
-        WebServicesTestUtils.getXmlString(qElem, "queueName"),
-        WebServicesTestUtils.getXmlString(qElem, "state"));
-
     NodeList queues = qElem.getElementsByTagName("subQueues");
+    QueueInfo qi = (queues != null) ? new QueueInfo() : new LeafQueueInfo();
+    qi.capacity = WebServicesTestUtils.getXmlFloat(qElem, "capacity");
+    qi.usedCapacity =
+        WebServicesTestUtils.getXmlFloat(qElem, "usedCapacity");
+    qi.maxCapacity = WebServicesTestUtils.getXmlFloat(qElem, "maxCapacity");
+    qi.absoluteCapacity = WebServicesTestUtils.getXmlFloat(qElem, "absoluteCapacity");
+    qi.absoluteMaxCapacity =
+        WebServicesTestUtils.getXmlFloat(qElem, "absoluteMaxCapacity");
+    qi.utilization = WebServicesTestUtils.getXmlFloat(qElem, "utilization");
+    qi.numApplications =
+        WebServicesTestUtils.getXmlInt(qElem, "numApplications");
+    qi.usedResources =
+        WebServicesTestUtils.getXmlString(qElem, "usedResources");
+    qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName");
+    qi.state = WebServicesTestUtils.getXmlString(qElem, "state");
+    verifySubQueueGeneric(q, qi, parentAbsCapacity);
+
     if (queues != null) {
       for (int j = 0; j < queues.getLength(); j++) {
         Element subqElem = (Element) queues.item(j);
         String qName = WebServicesTestUtils.getXmlString(subqElem, "queueName");
         String q2 = q + "." + qName;
-        verifySubQueueXML(subqElem, q2, absCapacity);
+        verifySubQueueXML(subqElem, q2, qi.absoluteCapacity);
       }
     } else {
-      verifyLeafQueueGeneric(q,
+      LeafQueueInfo lqi = (LeafQueueInfo) qi;
-          WebServicesTestUtils.getXmlInt(qElem, "userLimit"),
+      lqi.numActiveApplications =
-          WebServicesTestUtils.getXmlFloat(qElem, "userLimitFactor"));
+          WebServicesTestUtils.getXmlInt(qElem, "numActiveApplications");
+      lqi.numPendingApplications =
+          WebServicesTestUtils.getXmlInt(qElem, "numPendingApplications");
+      lqi.numContainers =
+          WebServicesTestUtils.getXmlInt(qElem, "numContainers");
+      lqi.maxApplications =
+          WebServicesTestUtils.getXmlInt(qElem, "maxApplications");
+      lqi.maxApplicationsPerUser =
+          WebServicesTestUtils.getXmlInt(qElem, "maxApplicationsPerUser");
+      lqi.maxActiveApplications =
+          WebServicesTestUtils.getXmlInt(qElem, "maxActiveApplications");
+      lqi.maxActiveApplicationsPerUser =
+          WebServicesTestUtils.getXmlInt(qElem, "maxActiveApplicationsPerUser");
+      lqi.userLimit = WebServicesTestUtils.getXmlInt(qElem, "userLimit");
+      lqi.userLimitFactor =
+          WebServicesTestUtils.getXmlFloat(qElem, "userLimitFactor");
+      verifyLeafQueueGeneric(q, lqi);
     }
   }
 
@@ -286,16 +333,19 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
     }
     assertEquals("incorrect number of elements", numExpectedElements, info.length());
 
-    float absCapacity = (float) info.getDouble("absoluteCapacity");
+    QueueInfo qi = isParentQueue ? new QueueInfo() : new LeafQueueInfo();
+    qi.capacity = (float) info.getDouble("capacity");
+    qi.usedCapacity = (float) info.getDouble("usedCapacity");
+    qi.maxCapacity = (float) info.getDouble("maxCapacity");
+    qi.absoluteCapacity = (float) info.getDouble("absoluteCapacity");
+    qi.absoluteMaxCapacity = (float) info.getDouble("absoluteMaxCapacity");
+    qi.utilization = (float) info.getDouble("utilization");
+    qi.numApplications = info.getInt("numApplications");
+    qi.usedResources = info.getString("usedResources");
+    qi.queueName = info.getString("queueName");
+    qi.state = info.getString("state");
 
-    verifySubQueueGeneric(q, (float) info.getDouble("usedCapacity"),
+    verifySubQueueGeneric(q, qi, parentAbsCapacity);
-        (float) info.getDouble("capacity"),
-        (float) info.getDouble("maxCapacity"),
-        absCapacity,
-        (float) info.getDouble("absoluteMaxCapacity"),
-        parentAbsCapacity,
-        info.getString("queueName"),
-        info.getString("state"));
 
     if (isParentQueue) {
       JSONArray arr = info.getJSONArray("subQueues");
@@ -303,50 +353,84 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
       for (int i = 0; i < arr.length(); i++) {
         JSONObject obj = arr.getJSONObject(i);
         String q2 = q + "." + obj.getString("queueName");
-        verifySubQueue(obj, q2, absCapacity);
+        verifySubQueue(obj, q2, qi.absoluteCapacity);
       }
     } else {
-      verifyLeafQueueGeneric(q, info.getInt("userLimit"),
+      LeafQueueInfo lqi = (LeafQueueInfo) qi;
-          (float) info.getDouble("userLimitFactor"));
+      lqi.numActiveApplications = info.getInt("numActiveApplications");
+      lqi.numPendingApplications = info.getInt("numPendingApplications");
+      lqi.numContainers = info.getInt("numContainers");
+      lqi.maxApplications = info.getInt("maxApplications");
+      lqi.maxApplicationsPerUser = info.getInt("maxApplicationsPerUser");
+      lqi.maxActiveApplications = info.getInt("maxActiveApplications");
+      lqi.maxActiveApplicationsPerUser = info.getInt("maxActiveApplicationsPerUser");
+      lqi.userLimit = info.getInt("userLimit");
+      lqi.userLimitFactor = (float) info.getDouble("userLimitFactor");
+      verifyLeafQueueGeneric(q, lqi);
     }
   }
 
-  private void verifySubQueueGeneric(String q, float usedCapacity,
+  private void verifySubQueueGeneric(String q, QueueInfo info,
-      float capacity, float maxCapacity,
+      float parentAbsCapacity) throws Exception {
-      float absCapacity, float absMaxCapacity,
-      float parentAbsCapacity,
-      String qname, String state)
-      throws Exception {
     String[] qArr = q.split("\\.");
     assertTrue("q name invalid: " + q, qArr.length > 1);
     String qshortName = qArr[qArr.length - 1];
 
-    assertEquals("usedCapacity doesn't match", 0, usedCapacity, 1e-3f);
+    assertEquals("usedCapacity doesn't match", 0, info.usedCapacity, 1e-3f);
-    assertEquals("capacity doesn't match", csConf.getCapacity(q), capacity,
+    assertEquals("capacity doesn't match", csConf.getCapacity(q),
-        1e-3f);
+        info.capacity, 1e-3f);
     float expectCapacity = csConf.getMaximumCapacity(q);
-    float expectAbsMaxCapacity = parentAbsCapacity * (maxCapacity/100);
+    float expectAbsMaxCapacity = parentAbsCapacity * (info.maxCapacity/100);
     if (CapacitySchedulerConfiguration.UNDEFINED == expectCapacity) {
       expectCapacity = 100;
       expectAbsMaxCapacity = 100;
     }
-    assertEquals("maxCapacity doesn't match", expectCapacity, maxCapacity,
+    assertEquals("maxCapacity doesn't match", expectCapacity,
-        1e-3f);
+        info.maxCapacity, 1e-3f);
     assertEquals("absoluteCapacity doesn't match",
-        parentAbsCapacity * (capacity/100), absCapacity, 1e-3f);
+        parentAbsCapacity * (info.capacity/100), info.absoluteCapacity, 1e-3f);
     assertEquals("absoluteMaxCapacity doesn't match",
-        expectAbsMaxCapacity, absMaxCapacity, 1e-3f);
+        expectAbsMaxCapacity, info.absoluteMaxCapacity, 1e-3f);
-    assertTrue("queueName doesn't match, got: " + qname + " expected: " + q,
+    assertEquals("utilization doesn't match", 0, info.utilization, 1e-3f);
-        qshortName.matches(qname));
+    assertEquals("numApplications doesn't match", 0, info.numApplications);
+    assertTrue("usedResources doesn't match",
+        info.usedResources.matches("memory: 0"));
+    assertTrue("queueName doesn't match, got: " + info.queueName
+        + " expected: " + q, qshortName.matches(info.queueName));
     assertTrue("state doesn't match",
-        (csConf.getState(q).toString()).matches(state));
+        (csConf.getState(q).toString()).matches(info.state));
 
   }
 
-  private void verifyLeafQueueGeneric(String q, int userLimit,
+  private void verifyLeafQueueGeneric(String q, LeafQueueInfo info)
-      float userLimitFactor) throws Exception {
+      throws Exception {
-    assertEquals("userLimit doesn't match", csConf.getUserLimit(q), userLimit);
+    assertEquals("numActiveApplications doesn't match",
+        0, info.numActiveApplications);
+    assertEquals("numPendingApplications doesn't match",
+        0, info.numPendingApplications);
+    assertEquals("numContainers doesn't match",
+        0, info.numContainers);
+
+    int maxSystemApps = csConf.getMaximumSystemApplications();
+    int expectedMaxApps = (int)(maxSystemApps * (info.absoluteCapacity/100));
+    int expectedMaxAppsPerUser =
+      (int)(expectedMaxApps * (info.userLimit/100.0f) * info.userLimitFactor);
+
+    // TODO: would like to use integer comparisons here but can't due to
+    //       roundoff errors in absolute capacity calculations
+    assertEquals("maxApplications doesn't match",
+        (float)expectedMaxApps, (float)info.maxApplications, 1.0f);
+    assertEquals("maxApplicationsPerUser doesn't match",
+        (float)expectedMaxAppsPerUser,
+        (float)info.maxApplicationsPerUser, info.userLimitFactor);
+
+    assertTrue("maxActiveApplications doesn't match",
+        info.maxActiveApplications > 0);
+    assertTrue("maxActiveApplicationsPerUser doesn't match",
+        info.maxActiveApplicationsPerUser > 0);
+    assertEquals("userLimit doesn't match", csConf.getUserLimit(q),
+        info.userLimit);
     assertEquals("userLimitFactor doesn't match",
-        csConf.getUserLimitFactor(q), userLimitFactor, 1e-3f);
+        csConf.getUserLimitFactor(q), info.userLimitFactor, 1e-3f);
   }
 }