Reverting "YARN-1258: Move to 2.2.1 in CHANGES.txt" because it contained unintended changes

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1534308 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -61,6 +61,8 @@ Release 2.3.0 - UNRELEASED
 
     YARN-976. Document the meaning of a virtual core. (Sandy Ryza)
 
+    YARN-1258. Allow configuring the Fair Scheduler root queue (Sandy Ryza)
+
     YARN-1182. MiniYARNCluster creates and inits the RM/NM only on start()
     (Karthik Kambatla via Sandy Ryza)
 
@@ -93,8 +95,6 @@ Release 2.2.1 - UNRELEASED
     YARN-305. Fair scheduler logs too many "Node offered to app" messages.
     (Lohit Vijayarenu via Sandy Ryza)
 
-    YARN-1258. Allow configuring the Fair Scheduler root queue (Sandy Ryza)
-
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java

@@ -19,10 +19,12 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 
 import java.util.List;
+import java.util.Map;
 
 import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@@ -42,6 +44,12 @@ public interface Queue {
    */
   QueueMetrics getMetrics();
 
+  /**
+   * Get ACLs for the queue.
+   * @return ACLs for the queue
+   */
+  public Map<QueueACL, AccessControlList> getQueueAcls();
+  
   /**
    * Get queue information
    * @param includeChildQueues include child queues?

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java

@@ -526,6 +526,11 @@ public class LeafQueue implements CSQueue {
     return userLimitFactor;
   }
 
+  @Override
+  public synchronized Map<QueueACL, AccessControlList> getQueueAcls() {
+    return new HashMap<QueueACL, AccessControlList>(acls);
+  }
+
   @Override
   public synchronized QueueInfo getQueueInfo(
       boolean includeChildQueues, boolean recursive) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

@@ -299,6 +299,11 @@ public class ParentQueue implements CSQueue {
     return state;
   }
 
+  @Override
+  public synchronized Map<QueueACL, AccessControlList> getQueueAcls() {
+    return new HashMap<QueueACL, AccessControlList>(acls);
+  }
+
   @Override
   public synchronized QueueInfo getQueueInfo( 
       boolean includeChildQueues, boolean recursive) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java

@@ -24,12 +24,14 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -175,7 +177,8 @@ public class FSLeafQueue extends FSQueue {
       recordFactory.newRecordInstance(QueueUserACLInfo.class);
     List<QueueACL> operations = new ArrayList<QueueACL>();
     for (QueueACL operation : QueueACL.values()) {
-      if (hasAccess(operation, user)) {
+      Map<QueueACL, AccessControlList> acls = queueMgr.getQueueAcls(getName());
+      if (acls.get(operation).isUserAllowed(user)) {
         operations.add(operation);
       }
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java

@@ -20,10 +20,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -132,6 +135,12 @@ public abstract class FSQueue extends Schedulable implements Queue {
     return queueInfo;
   }
   
+  @Override
+  public Map<QueueACL, AccessControlList> getQueueAcls() {
+    Map<QueueACL, AccessControlList> acls = queueMgr.getQueueAcls(getName());
+    return new HashMap<QueueACL, AccessControlList>(acls);
+  }
+  
   @Override
   public FSQueueMetrics getMetrics() {
     return metrics;
@@ -145,7 +154,7 @@ public abstract class FSQueue extends Schedulable implements Queue {
   
   public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
     // Check if the leaf-queue allows access
-    if (queueMgr.getQueueAcl(getName(), acl).isUserAllowed(user)) {
+    if (queueMgr.getQueueAcls(getName()).get(acl).isUserAllowed(user)) {
       return true;
     }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java

@@ -72,9 +72,6 @@ public class QueueManager {
    * (this is done to prevent loading a file that hasn't been fully written).
    */
   public static final long ALLOC_RELOAD_WAIT = 5 * 1000;
-  
-  private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*");
-  private static final AccessControlList NOBODY_ACL = new AccessControlList(" ");
 
   private final FairScheduler scheduler;
 
@@ -384,6 +381,15 @@ public class QueueManager {
         queueMetrics.setMinShare(queue.getMinShare());
         queueMetrics.setMaxShare(queue.getMaxShare());
       }
+      
+      // Root queue should have empty ACLs.  As a queue's ACL is the union of
+      // its ACL and all its parents' ACLs, setting the roots' to empty will
+      // neither allow nor prohibit more access to its children.
+      Map<QueueACL, AccessControlList> rootAcls =
+          new HashMap<QueueACL, AccessControlList>();
+      rootAcls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(" "));
+      rootAcls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(" "));
+      queueAcls.put(ROOT_QUEUE, rootAcls);
  
       // Create all queus
       for (String name: queueNamesInAllocFile) {
@@ -448,10 +454,10 @@ public class QueueManager {
         policy.initialize(scheduler.getClusterCapacity());
         queuePolicies.put(queueName, policy);
       } else if ("aclSubmitApps".equals(field.getTagName())) {
-        String text = ((Text)field.getFirstChild()).getData();
+        String text = ((Text)field.getFirstChild()).getData().trim();
         acls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(text));
       } else if ("aclAdministerApps".equals(field.getTagName())) {
-        String text = ((Text)field.getFirstChild()).getData();
+        String text = ((Text)field.getFirstChild()).getData().trim();
         acls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(text));
       } else if ("queue".endsWith(field.getTagName()) || 
           "pool".equals(field.getTagName())) {
@@ -571,16 +577,21 @@ public class QueueManager {
 
   /**
    * Get the ACLs associated with this queue. If a given ACL is not explicitly
-   * configured, include the default value for that ACL.  The default for the
+   * configured, include the default value for that ACL.
-   * root queue is everybody ("*") and the default for all other queues is
-   * nobody ("")
    */
-  public AccessControlList getQueueAcl(String queue, QueueACL operation) {
+  public Map<QueueACL, AccessControlList> getQueueAcls(String queue) {
-    Map<QueueACL, AccessControlList> queueAcls = info.queueAcls.get(queue);
+    HashMap<QueueACL, AccessControlList> out = new HashMap<QueueACL, AccessControlList>();
-    if (queueAcls == null || !queueAcls.containsKey(operation)) {
+    Map<QueueACL, AccessControlList> queueAcl = info.queueAcls.get(queue);
-      return (queue.equals(ROOT_QUEUE)) ? EVERYBODY_ACL : NOBODY_ACL;
+    if (queueAcl != null) {
+      out.putAll(queueAcl);
     }
-    return queueAcls.get(operation);
+    if (!out.containsKey(QueueACL.ADMINISTER_QUEUE)) {
+      out.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList("*"));
+    }
+    if (!out.containsKey(QueueACL.SUBMIT_APPLICATIONS)) {
+      out.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList("*"));
+    }
+    return out;
   }
   
   static class QueueManagerInfo {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java

@@ -156,6 +156,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable {
       return queueInfo;
     }
 
+    @Override
     public Map<QueueACL, AccessControlList> getQueueAcls() {
       Map<QueueACL, AccessControlList> acls =
         new HashMap<QueueACL, AccessControlList>();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java

@@ -865,25 +865,22 @@ public class TestFairScheduler {
     assertEquals(10, queueManager.getUserMaxApps("user1"));
     assertEquals(5, queueManager.getUserMaxApps("user2"));
 
-    // Root should get * ACL
-    assertEquals("*",queueManager.getQueueAcl("root",
-        QueueACL.ADMINISTER_QUEUE).getAclString());
-    assertEquals("*", queueManager.getQueueAcl("root",
-        QueueACL.SUBMIT_APPLICATIONS).getAclString());
-
     // Unspecified queues should get default ACL
-    assertEquals(" ",queueManager.getQueueAcl("root.queueA",
+    Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("root.queueA");
-        QueueACL.ADMINISTER_QUEUE).getAclString());
+    assertTrue(aclsA.containsKey(QueueACL.ADMINISTER_QUEUE));
-    assertEquals(" ", queueManager.getQueueAcl("root.queueA",
+    assertEquals("*", aclsA.get(QueueACL.ADMINISTER_QUEUE).getAclString());
-        QueueACL.SUBMIT_APPLICATIONS).getAclString());
+    assertTrue(aclsA.containsKey(QueueACL.SUBMIT_APPLICATIONS));
+    assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
 
     // Queue B ACL
-    assertEquals("alice,bob admins",queueManager.getQueueAcl("root.queueB",
+    Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
-        QueueACL.ADMINISTER_QUEUE).getAclString());
+    assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
+    assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
 
-    // Queue C ACL
+    // Queue c ACL
-    assertEquals("alice,bob admins",queueManager.getQueueAcl("root.queueC",
+    Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
-        QueueACL.SUBMIT_APPLICATIONS).getAclString());
+    assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
+    assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
 
     assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." + 
         YarnConfiguration.DEFAULT_QUEUE_NAME));
@@ -1066,19 +1063,21 @@ public class TestFairScheduler {
     assertEquals(5, queueManager.getUserMaxApps("user2"));
 
     // Unspecified queues should get default ACL
-    assertEquals(" ", queueManager.getQueueAcl("root.queueA",
+    Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("queueA");
-        QueueACL.ADMINISTER_QUEUE).getAclString());
+    assertTrue(aclsA.containsKey(QueueACL.ADMINISTER_QUEUE));
-    assertEquals(" ", queueManager.getQueueAcl("root.queueA",
+    assertEquals("*", aclsA.get(QueueACL.ADMINISTER_QUEUE).getAclString());
-        QueueACL.SUBMIT_APPLICATIONS).getAclString());
+    assertTrue(aclsA.containsKey(QueueACL.SUBMIT_APPLICATIONS));
+    assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
 
     // Queue B ACL
-    assertEquals("alice,bob admins", queueManager.getQueueAcl("root.queueB",
+    Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
-        QueueACL.ADMINISTER_QUEUE).getAclString());
+    assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
-
+    assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
-    // Queue C ACL
-    assertEquals("alice,bob admins", queueManager.getQueueAcl("root.queueC",
-        QueueACL.SUBMIT_APPLICATIONS).getAclString());
 
+    // Queue c ACL
+    Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
+    assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
+    assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
 
     assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." +
         YarnConfiguration.DEFAULT_QUEUE_NAME));
@@ -1665,13 +1664,9 @@ public class TestFairScheduler {
     PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
     out.println("<?xml version=\"1.0\"?>");
     out.println("<allocations>");
-    out.println("<queue name=\"root\">");
+    out.println("<queue name=\"queue1\">");
-    out.println("  <aclSubmitApps> </aclSubmitApps>");
+    out.println("<aclSubmitApps>norealuserhasthisname</aclSubmitApps>");
-    out.println("  <aclAdministerApps> </aclAdministerApps>");
+    out.println("<aclAdministerApps>norealuserhasthisname</aclAdministerApps>");
-    out.println("  <queue name=\"queue1\">");
-    out.println("    <aclSubmitApps>norealuserhasthisname</aclSubmitApps>");
-    out.println("    <aclAdministerApps>norealuserhasthisname</aclAdministerApps>");
-    out.println("  </queue>");
     out.println("</queue>");
     out.println("</allocations>");
     out.close();
@@ -1898,13 +1893,9 @@ public class TestFairScheduler {
     PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
     out.println("<?xml version=\"1.0\"?>");
     out.println("<allocations>");
-    out.println("<queue name=\"root\">");
+    out.println("<queue name=\"queue1\">");
-    out.println("  <aclSubmitApps> </aclSubmitApps>");
+    out.println("<aclSubmitApps>userallow</aclSubmitApps>");
-    out.println("  <aclAdministerApps> </aclAdministerApps>");
+    out.println("<aclAdministerApps>userallow</aclAdministerApps>");
-    out.println("  <queue name=\"queue1\">");
-    out.println("    <aclSubmitApps>userallow</aclSubmitApps>");
-    out.println("    <aclAdministerApps>userallow</aclAdministerApps>");
-    out.println("  </queue>");
     out.println("</queue>");
     out.println("</allocations>");
     out.close();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm

@@ -221,14 +221,10 @@ Allocation file format
      for containers, but apps submitted later may run concurrently if there is
      leftover space on the cluster after satisfying the earlier app's requests.
 
-   * aclSubmitApps: a list of users and/or groups that can submit apps to the
+   * aclSubmitApps: a list of users that can submit apps to the queue. A (default)
-     queue. Refer to the ACLs section below for more info on the format of this
+     value of "*" means that any users can submit apps. A queue inherits the ACL of
-     list and how queue ACLs work.
+     its parent, so if a queue2 descends from queue1, and user1 is in queue1's ACL,
-
+     and user2 is in queue2's ACL, then both users may submit to queue2.
-   * aclAdministerApps: a list of users and/or groups that can administer a
-     queue.  Currently the only administrative action is killing an application.
-     Refer to the ACLs section below for more info on the format of this list
-     and how queue ACLs work.
 
    * minSharePreemptionTimeout: number of seconds the queue is under its minimum share
      before it will try to preempt containers to take resources from other queues.
@@ -250,24 +246,6 @@ Allocation file format
 
   An example allocation file is given here:
 
-Queue Access Control Lists (ACLs)
-
-  Queue Access Control Lists (ACLs) allow administrators to control who may
-  take actions on particular queues. They are configured with the aclSubmitApps
-  and aclAdministerApps properties, which can be set per queue. Currently the
-  only supported administrative action is killing an application. Anybody who
-  may administer a queue may also submit applications to it. These properties
-  take values in a format like "user1,user2 group1,group2" or " group1,group2".
-  An action on a queue will be permitted if its user or group is in the ACL of
-  that queue or in the ACL of any of that queue's ancestors. So if queue2
-  is inside queue1, and user1 is in queue1's ACL, and user2 is in queue2's
-  ACL, then both users may submit to queue2.
-  
-  The root queue's ACLs are "*" by default which, because ACLs are passed down,
-  means that everybody may submit to and kill applications from every queue.
-  To start restricting access, change the root queue's ACLs to something other
-  than "*". 
-
 ---
 <?xml version="1.0"?>
 <allocations>
@@ -278,7 +256,6 @@ Queue Access Control Lists (ACLs)
     <weight>2.0</weight>
     <schedulingPolicy>fair</schedulingPolicy>
     <queue name="sample_sub_queue">
-      <aclSubmitApps>charlie</aclSubmitApps>
       <minResources>5000 mb,0vcores</minResources>
     </queue>
   </queue>