MAPREDUCE-3510. Capacity Scheduler inherited ACLs not displayed by mapred queue -showacls (Jonathan Eagles via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1213511 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -270,6 +270,9 @@ Release 0.23.1 - Unreleased
     MAPREDUCE-3328. mapred queue -list output inconsistent and missing child 
     queues. (Ravi Prakash via mahadev)
 
+    MAPREDUCE-3510. Capacity Scheduler inherited ACLs not displayed by mapred queue 
+    -showacls (Jonathan Eagles via mahadev)
+
 Release 0.23.0 - 2011-11-01 
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java

@@ -198,19 +198,33 @@ public QueueState getState(String queue) {
   private static String getAclKey(QueueACL acl) {
     return "acl_" + acl.toString().toLowerCase();
   }
-  
+
-  public Map<QueueACL, AccessControlList> getAcls(String queue) {
+  public AccessControlList getAcl(String queue, QueueACL acl) {
-    Map<QueueACL, AccessControlList> acls = 
-      new HashMap<QueueACL, AccessControlList>();
     String queuePrefix = getQueuePrefix(queue);
+    String aclString = get(queuePrefix + getAclKey(acl), DEFAULT_ACL);
+    return new AccessControlList(aclString);
+  }
+
+  public void setAcl(String queue, QueueACL acl, String aclString) {
+    String queuePrefix = getQueuePrefix(queue);
+    set(queuePrefix + getAclKey(acl), aclString);
+  }
+
+  public Map<QueueACL, AccessControlList> getAcls(String queue) {
+    Map<QueueACL, AccessControlList> acls =
+      new HashMap<QueueACL, AccessControlList>();
     for (QueueACL acl : QueueACL.values()) {
-      acls.put(acl, 
+      acls.put(acl, getAcl(queue, acl));
-          new AccessControlList(get(queuePrefix + getAclKey(acl), 
-              DEFAULT_ACL)));
     }
     return acls;
   }
 
+  public void setAcls(String queue, Map<QueueACL, AccessControlList> acls) {
+    for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
+      setAcl(queue, e.getKey(), e.getValue().getAclString());
+    }
+  }
+
   public String[] getQueues(String queue) {
     LOG.info("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue));
     String[] queues = getStrings(getQueuePrefix(queue) + QUEUES);

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java

@@ -492,11 +492,8 @@ public synchronized QueueInfo getQueueInfo(
     QueueUserACLInfo userAclInfo = 
       recordFactory.newRecordInstance(QueueUserACLInfo.class);
     List<QueueACL> operations = new ArrayList<QueueACL>();
-    for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
+    for (QueueACL operation : QueueACL.values()) {
-      QueueACL operation = e.getKey();
+      if (hasAccess(operation, user)) {
-      AccessControlList acl = e.getValue();
-
-      if (acl.isUserAllowed(user)) {
         operations.add(operation);
       }
     }

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java

@@ -21,6 +21,7 @@
 import static org.junit.Assert.*;
 import static org.mockito.Mockito.*;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -30,11 +31,14 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -102,20 +106,29 @@ public void setUp() throws Exception {
   
   private static final String A = "a";
   private static final String B = "b";
+  private static final String C = "c";
   private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
     
     // Define top-level queues
-    conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B});
+    conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B, C});
     conf.setCapacity(CapacityScheduler.ROOT, 100);
     conf.setMaximumCapacity(CapacityScheduler.ROOT, 100);
+    conf.setAcl(CapacityScheduler.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
     
     final String Q_A = CapacityScheduler.ROOT + "." + A;
-    conf.setCapacity(Q_A, 10);
+    conf.setCapacity(Q_A, 9);
     conf.setMaximumCapacity(Q_A, 20);
+    conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*");
     
     final String Q_B = CapacityScheduler.ROOT + "." + B;
     conf.setCapacity(Q_B, 90);
     conf.setMaximumCapacity(Q_B, 99);
+    conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*");
+
+    final String Q_C = CapacityScheduler.ROOT + "." + C;
+    conf.setCapacity(Q_C, 1);
+    conf.setMaximumCapacity(Q_C, 10);
+    conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " ");
     
     LOG.info("Setup top-level queues a and b");
   }
@@ -167,8 +180,8 @@ public void testInitializeQueue() throws Exception {
 	  //can add more sturdy test with 3-layer queues 
 	  //once MAPREDUCE:3410 is resolved
 	  LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
-	  assertEquals(0.1, a.getCapacity(), epsilon);
+	  assertEquals(0.09, a.getCapacity(), epsilon);
-	  assertEquals(0.1, a.getAbsoluteCapacity(), epsilon);
+	  assertEquals(0.09, a.getAbsoluteCapacity(), epsilon);
 	  assertEquals(0.2, a.getMaximumCapacity(), epsilon);
 	  assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
 	  
@@ -177,6 +190,12 @@ public void testInitializeQueue() throws Exception {
 	  assertEquals(0.9, b.getAbsoluteCapacity(), epsilon);
 	  assertEquals(0.99, b.getMaximumCapacity(), epsilon);
 	  assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
+
+	  LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C));
+	  assertEquals(0.01, c.getCapacity(), epsilon);
+	  assertEquals(0.01, c.getAbsoluteCapacity(), epsilon);
+	  assertEquals(0.1, c.getMaximumCapacity(), epsilon);
+	  assertEquals(0.1, c.getAbsoluteMaximumCapacity(), epsilon);
   }
  
   @Test
@@ -1082,6 +1101,37 @@ public void testSchedulingConstraints() throws Exception {
     assertEquals(0, app_0.getTotalRequiredResources(priority));
 
   }
+
+  public boolean hasQueueACL(List<QueueUserACLInfo> aclInfos, QueueACL acl) {
+    for (QueueUserACLInfo aclInfo : aclInfos) {
+      if (aclInfo.getUserAcls().contains(acl)) {
+        return true;
+      }
+    }    
+    return false;
+  }
+
+  @Test
+  public void testInheritedQueueAcls() throws IOException {
+    UserGroupInformation user = UserGroupInformation.getCurrentUser();
+
+    LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+    LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
+    LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C));
+
+    assertFalse(root.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+    assertTrue(a.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+    assertTrue(b.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+    assertFalse(c.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+
+    assertTrue(hasQueueACL(
+          a.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
+    assertTrue(hasQueueACL(
+          b.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
+    assertFalse(hasQueueACL(
+          c.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
+
+  }
   
   @After
   public void tearDown() throws Exception {