From f340be0927b0e676ba9e53aa168cb264853c0efd Mon Sep 17 00:00:00 2001 From: Mahadev Konar Date: Tue, 13 Dec 2011 00:24:48 +0000 Subject: [PATCH] MAPREDUCE-3510. Capacity Scheduler inherited ACLs not displayed by mapred queue -showacls (Jonathan Eagles via mahadev) - Merging r1213511 from trunk. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1213516 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../CapacitySchedulerConfiguration.java | 28 ++++++--- .../scheduler/capacity/LeafQueue.java | 7 +-- .../scheduler/capacity/TestLeafQueue.java | 58 +++++++++++++++++-- 4 files changed, 80 insertions(+), 16 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 77d82e0fd80..3e5899f92b3 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -221,6 +221,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3328. mapred queue -list output inconsistent and missing child queues. (Ravi Prakash via mahadev) + MAPREDUCE-3510. Capacity Scheduler inherited ACLs not displayed by mapred queue + -showacls (Jonathan Eagles via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index cf52a72b67c..82f6f85418a 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -198,19 +198,33 @@ public class CapacitySchedulerConfiguration extends Configuration { private static String getAclKey(QueueACL acl) { return "acl_" + acl.toString().toLowerCase(); } - - public Map getAcls(String queue) { - Map acls = - new HashMap(); + + public AccessControlList getAcl(String queue, QueueACL acl) { String queuePrefix = getQueuePrefix(queue); + String aclString = get(queuePrefix + getAclKey(acl), DEFAULT_ACL); + return new AccessControlList(aclString); + } + + public void setAcl(String queue, QueueACL acl, String aclString) { + String queuePrefix = getQueuePrefix(queue); + set(queuePrefix + getAclKey(acl), aclString); + } + + public Map getAcls(String queue) { + Map acls = + new HashMap(); for (QueueACL acl : QueueACL.values()) { - acls.put(acl, - new AccessControlList(get(queuePrefix + getAclKey(acl), - DEFAULT_ACL))); + acls.put(acl, getAcl(queue, acl)); } return acls; } + public void setAcls(String queue, Map acls) { + for (Map.Entry e : acls.entrySet()) { + setAcl(queue, e.getKey(), e.getValue().getAclString()); + } + } + public String[] getQueues(String queue) { LOG.info("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue)); String[] queues = getStrings(getQueuePrefix(queue) + QUEUES); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 37bfd8cc0f1..8716294ef83 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -492,11 +492,8 @@ public class LeafQueue implements CSQueue { QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(QueueUserACLInfo.class); List operations = new ArrayList(); - for (Map.Entry e : acls.entrySet()) { - QueueACL operation = e.getKey(); - AccessControlList acl = e.getValue(); - - if (acl.isUserAllowed(user)) { + for (QueueACL operation : QueueACL.values()) { + if (hasAccess(operation, user)) { operations.add(operation); } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 949d9521925..69fea5d998f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import static org.junit.Assert.*; import static org.mockito.Mockito.*; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -30,11 +31,14 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -102,20 +106,29 @@ public class TestLeafQueue { private static final String A = "a"; private static final String B = "b"; + private static final String C = "c"; private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B}); + conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B, C}); conf.setCapacity(CapacityScheduler.ROOT, 100); conf.setMaximumCapacity(CapacityScheduler.ROOT, 100); + conf.setAcl(CapacityScheduler.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); final String Q_A = CapacityScheduler.ROOT + "." + A; - conf.setCapacity(Q_A, 10); + conf.setCapacity(Q_A, 9); conf.setMaximumCapacity(Q_A, 20); + conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*"); final String Q_B = CapacityScheduler.ROOT + "." + B; conf.setCapacity(Q_B, 90); conf.setMaximumCapacity(Q_B, 99); + conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*"); + + final String Q_C = CapacityScheduler.ROOT + "." + C; + conf.setCapacity(Q_C, 1); + conf.setMaximumCapacity(Q_C, 10); + conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " "); LOG.info("Setup top-level queues a and b"); } @@ -167,8 +180,8 @@ public class TestLeafQueue { //can add more sturdy test with 3-layer queues //once MAPREDUCE:3410 is resolved LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); - assertEquals(0.1, a.getCapacity(), epsilon); - assertEquals(0.1, a.getAbsoluteCapacity(), epsilon); + assertEquals(0.09, a.getCapacity(), epsilon); + assertEquals(0.09, a.getAbsoluteCapacity(), epsilon); assertEquals(0.2, a.getMaximumCapacity(), epsilon); assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon); @@ -177,6 +190,12 @@ public class TestLeafQueue { assertEquals(0.9, b.getAbsoluteCapacity(), epsilon); assertEquals(0.99, b.getMaximumCapacity(), epsilon); assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon); + + LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C)); + assertEquals(0.01, c.getCapacity(), epsilon); + assertEquals(0.01, c.getAbsoluteCapacity(), epsilon); + assertEquals(0.1, c.getMaximumCapacity(), epsilon); + assertEquals(0.1, c.getAbsoluteMaximumCapacity(), epsilon); } @Test @@ -1082,6 +1101,37 @@ public class TestLeafQueue { assertEquals(0, app_0.getTotalRequiredResources(priority)); } + + public boolean hasQueueACL(List aclInfos, QueueACL acl) { + for (QueueUserACLInfo aclInfo : aclInfos) { + if (aclInfo.getUserAcls().contains(acl)) { + return true; + } + } + return false; + } + + @Test + public void testInheritedQueueAcls() throws IOException { + UserGroupInformation user = UserGroupInformation.getCurrentUser(); + + LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); + LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B)); + LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C)); + + assertFalse(root.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertTrue(a.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertTrue(b.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertFalse(c.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + + assertTrue(hasQueueACL( + a.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS)); + assertTrue(hasQueueACL( + b.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS)); + assertFalse(hasQueueACL( + c.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS)); + + } @After public void tearDown() throws Exception {