svn merge -c 1404247 FIXES: YARN-166. capacity scheduler doesn't allow capacity < 1.0 (tgraves via bobby)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1404249 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -176,6 +176,9 @@ Release 0.23.5 - UNRELEASED
     YARN-159. RM web ui applications page should be sorted to display last app
     first (tgraves via bobby)
 
+    YARN-166. capacity scheduler doesn't allow capacity < 1.0 (tgraves via
+    bobby)
+
 Release 0.23.4 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java

@@ -84,7 +84,7 @@ class CSQueueUtils {
     if (clusterMemory > 0) {
       queueLimit = clusterMemory * childQueue.getAbsoluteCapacity();
       absoluteUsedCapacity = ((float)usedMemory / (float)clusterMemory);
-      usedCapacity = (usedMemory / queueLimit);
+      usedCapacity = (queueLimit == 0) ? 0 : (usedMemory / queueLimit);
     }
     
     childQueue.setUsedCapacity(usedCapacity);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java

@@ -91,7 +91,7 @@ public class CapacitySchedulerConfiguration extends Configuration {
   public static final float UNDEFINED = -1;
   
   @Private
-  public static final float MINIMUM_CAPACITY_VALUE = 1;
+  public static final float MINIMUM_CAPACITY_VALUE = 0;
   
   @Private
   public static final float MAXIMUM_CAPACITY_VALUE = 100;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

@@ -202,7 +202,9 @@ public class ParentQueue implements CSQueue {
       childCapacities += queue.getCapacity();
     }
     float delta = Math.abs(1.0f - childCapacities);  // crude way to check
-    if (delta > PRECISION) {
+    // allow capacities being set to 0, and enforce child 0 if parent is 0
+    if (((capacity > 0) && (delta > PRECISION)) || 
+        ((capacity == 0) && (childCapacities > 0))) {
       throw new IllegalArgumentException("Illegal" +
       		" capacity of " + childCapacities + 
       		" for children of queue " + queueName);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java

@@ -66,8 +66,8 @@ public class TestCapacityScheduler {
   private static float B_CAPACITY = 89.5f;
   private static float A1_CAPACITY = 30;
   private static float A2_CAPACITY = 70;
-  private static float B1_CAPACITY = 50;
+  private static float B1_CAPACITY = 79.2f;
-  private static float B2_CAPACITY = 30;
+  private static float B2_CAPACITY = 0.8f;
   private static float B3_CAPACITY = 20;
 
   private ResourceManager resourceManager = null;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doAnswer;
@@ -494,6 +495,72 @@ public class TestParentQueue {
     verifyQueueMetrics(c, 4*GB, clusterResource);
     reset(a); reset(b); reset(c);
   }
+  
+  @Test (expected=IllegalArgumentException.class)
+  public void testQueueCapacitySettingChildZero() throws Exception {
+    // Setup queue configs
+    setupMultiLevelQueues(csConf);
+    
+    // set child queues capacity to 0 when parents not 0
+    final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+    csConf.setCapacity(Q_B + "." + B1, 0);
+    csConf.setCapacity(Q_B + "." + B2, 0);
+    csConf.setCapacity(Q_B + "." + B3, 0);
+    
+    Map<String, CSQueue> queues = new HashMap<String, CSQueue>(); 
+    CapacityScheduler.parseQueue(csContext, csConf, null, 
+        CapacitySchedulerConfiguration.ROOT, queues, queues, 
+        CapacityScheduler.queueComparator, 
+        CapacityScheduler.applicationComparator,
+        TestUtils.spyHook);
+  }
+  
+  @Test (expected=IllegalArgumentException.class)
+  public void testQueueCapacitySettingParentZero() throws Exception {
+    // Setup queue configs
+    setupMultiLevelQueues(csConf);
+    
+    // set parent capacity to 0 when child not 0
+    final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+    csConf.setCapacity(Q_B, 0);
+    final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
+    csConf.setCapacity(Q_A, 60);
+
+    Map<String, CSQueue> queues = new HashMap<String, CSQueue>(); 
+    CapacityScheduler.parseQueue(csContext, csConf, null, 
+        CapacitySchedulerConfiguration.ROOT, queues, queues, 
+        CapacityScheduler.queueComparator, 
+        CapacityScheduler.applicationComparator,
+        TestUtils.spyHook);
+  }
+  
+  @Test
+  public void testQueueCapacityZero() throws Exception {
+    // Setup queue configs
+    setupMultiLevelQueues(csConf);
+    
+    // set parent and child capacity to 0
+    final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+    csConf.setCapacity(Q_B, 0);
+    csConf.setCapacity(Q_B + "." + B1, 0);
+    csConf.setCapacity(Q_B + "." + B2, 0);
+    csConf.setCapacity(Q_B + "." + B3, 0);
+    
+    final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
+    csConf.setCapacity(Q_A, 60);
+
+    Map<String, CSQueue> queues = new HashMap<String, CSQueue>(); 
+    try {
+      CapacityScheduler.parseQueue(csContext, csConf, null, 
+          CapacitySchedulerConfiguration.ROOT, queues, queues, 
+          CapacityScheduler.queueComparator, 
+          CapacityScheduler.applicationComparator,
+          TestUtils.spyHook);
+    } catch (IllegalArgumentException e) {
+      fail("Failed to create queues with 0 capacity: " + e);
+    }
+    assertTrue("Failed to create queues with 0 capacity", true);
+  }
 
   @Test
   public void testOffSwitchScheduling() throws Exception {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java

@@ -144,11 +144,11 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
     final String B2 = B + ".b2";
     final String B3 = B + ".b3";
     conf.setQueues(B, new String[] { "b1", "b2", "b3" });
-    conf.setCapacity(B1, 50);
+    conf.setCapacity(B1, 60);
     conf.setUserLimitFactor(B1, 100.0f);
-    conf.setCapacity(B2, 30);
+    conf.setCapacity(B2, 39.5f);
     conf.setUserLimitFactor(B2, 100.0f);
-    conf.setCapacity(B3, 20);
+    conf.setCapacity(B3, 0.5f);
     conf.setUserLimitFactor(B3, 100.0f);
     
     conf.setQueues(A1, new String[] {"a1a", "a1b"});