Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-3522. Ensure queues inherit ACLs from parent if they aren't explicitly specified. Contributed by Jonathan Eagles. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1225618 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2011-12-29 19:06:34 +00:00
parent 6c349f9420
commit 743a5b78e3
10 changed files with 91 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -378,6 +378,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3521. Fixed streaming to ensure it doesn't silently ignore
unknown arguments. (Robert Evans via acmurthy)
MAPREDUCE-3522. Ensure queues inherit ACLs from parent if they aren't
explicitly specified. (Jonathan Eagles via acmurthy)
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES

9
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
View File

@ -200,12 +200,9 @@ implements ResourceScheduler, CapacitySchedulerContext {
}
}
@Private
public static final String ROOT = "root";
@Private
public static final String ROOT_QUEUE =
CapacitySchedulerConfiguration.PREFIX + ROOT;
CapacitySchedulerConfiguration.PREFIX + CapacitySchedulerConfiguration.ROOT;
static class QueueHook {
public CSQueue hook(CSQueue queue) {
@ -218,7 +215,7 @@ implements ResourceScheduler, CapacitySchedulerContext {
private void initializeQueues(CapacitySchedulerConfiguration conf)
throws IOException {
root =
parseQueue(this, conf, null, ROOT, queues, queues,
parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, queues, queues,
queueComparator, applicationComparator, noop);
LOG.info("Initialized root queue " + root);
}
@ -229,7 +226,7 @@ implements ResourceScheduler, CapacitySchedulerContext {
// Parse new queues
Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
CSQueue newRoot =
parseQueue(this, conf, null, ROOT, newQueues, queues,
parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
queueComparator, applicationComparator, noop);
// Ensure all existing queues are still present

27
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -106,12 +106,18 @@ public class CapacitySchedulerConfiguration extends Configuration {
public static final float DEFAULT_USER_LIMIT_FACTOR = 1.0f;
@Private
public static final String DEFAULT_ACL = "*";
public static final String ALL_ACL = "*";
@Private
public static final String NONE_ACL = " ";
@Private public static final String ENABLE_USER_METRICS =
PREFIX +"user-metrics.enable";
@Private public static final boolean DEFAULT_ENABLE_USER_METRICS = false;
@Private
public static final String ROOT = "root";
public CapacitySchedulerConfiguration() {
this(new Configuration());
}
@ -143,14 +149,14 @@ public class CapacitySchedulerConfiguration extends Configuration {
throw new IllegalArgumentException("Illegal " +
"capacity of " + capacity + " for queue " + queue);
}
LOG.info("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) +
LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) +
", capacity=" + capacity);
return capacity;
}
public void setCapacity(String queue, int capacity) {
setInt(getQueuePrefix(queue) + CAPACITY, capacity);
LOG.info("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) +
LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) +
", capacity=" + capacity);
}
@ -162,7 +168,7 @@ public class CapacitySchedulerConfiguration extends Configuration {
public void setMaximumCapacity(String queue, int maxCapacity) {
setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity);
LOG.info("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) +
LOG.debug("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) +
", maxCapacity=" + maxCapacity);
}
@ -174,7 +180,7 @@ public class CapacitySchedulerConfiguration extends Configuration {
public void setUserLimit(String queue, int userLimit) {
setInt(getQueuePrefix(queue) + USER_LIMIT, userLimit);
LOG.info("here setUserLimit: queuePrefix=" + getQueuePrefix(queue) +
LOG.debug("here setUserLimit: queuePrefix=" + getQueuePrefix(queue) +
", userLimit=" + getUserLimit(queue));
}
@ -201,7 +207,10 @@ public class CapacitySchedulerConfiguration extends Configuration {
public AccessControlList getAcl(String queue, QueueACL acl) {
String queuePrefix = getQueuePrefix(queue);
String aclString = get(queuePrefix + getAclKey(acl), DEFAULT_ACL);
// The root queue defaults to all access if not defined
// Sub queues inherit access if not defined
String defaultAcl = queue.equals(ROOT) ? ALL_ACL : NONE_ACL;
String aclString = get(queuePrefix + getAclKey(acl), defaultAcl);
return new AccessControlList(aclString);
}
@ -226,16 +235,16 @@ public class CapacitySchedulerConfiguration extends Configuration {
}
public String[] getQueues(String queue) {
LOG.info("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue));
LOG.debug("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue));
String[] queues = getStrings(getQueuePrefix(queue) + QUEUES);
LOG.info("CSConf - getQueues: queuePrefix=" + getQueuePrefix(queue) +
LOG.debug("CSConf - getQueues: queuePrefix=" + getQueuePrefix(queue) +
", queues=" + ((queues == null) ? "" : StringUtils.arrayToString(queues)));
return queues;
}
public void setQueues(String queue, String[] subQueues) {
set(getQueuePrefix(queue) + QUEUES, StringUtils.arrayToString(subQueues));
LOG.info("CSConf - setQueues: qPrefix=" + getQueuePrefix(queue) +
LOG.debug("CSConf - setQueues: qPrefix=" + getQueuePrefix(queue) +
", queues=" + StringUtils.arrayToString(subQueues));
}

10
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
View File

@ -88,13 +88,13 @@ public class TestApplicationLimits {
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B});
conf.setCapacity(CapacityScheduler.ROOT, 100);
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String Q_A = CapacityScheduler.ROOT + "." + A;
final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 10);
final String Q_B = CapacityScheduler.ROOT + "." + B;
final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 90);
LOG.info("Setup top-level queues a and b");
@ -252,7 +252,7 @@ public class TestApplicationLimits {
public void testHeadroom() throws Exception {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
csConf.setUserLimit(CapacityScheduler.ROOT + "." + A, 25);
csConf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 25);
setupQueueConfiguration(csConf);
CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);

14
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
View File

@ -197,13 +197,13 @@ public class TestCapacityScheduler {
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b"});
conf.setCapacity(CapacityScheduler.ROOT, 100);
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String A = CapacityScheduler.ROOT + ".a";
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
final String B = CapacityScheduler.ROOT + ".b";
final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 90);
// Define 2nd-level queues
@ -250,9 +250,9 @@ public class TestCapacityScheduler {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
conf.setQueues(CapacityScheduler.ROOT + ".a.a1", new String[] {"b1"} );
conf.setCapacity(CapacityScheduler.ROOT + ".a.a1.b1", 100);
conf.setUserLimitFactor(CapacityScheduler.ROOT + ".a.a1.b1", 100.0f);
conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100);
conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
cs.reinitialize(conf, null, null);
}

28
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
View File

@ -107,29 +107,35 @@ public class TestLeafQueue {
private static final String A = "a";
private static final String B = "b";
private static final String C = "c";
private static final String C1 = "c1";
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B, C});
conf.setCapacity(CapacityScheduler.ROOT, 100);
conf.setMaximumCapacity(CapacityScheduler.ROOT, 100);
conf.setAcl(CapacityScheduler.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100);
conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
final String Q_A = CapacityScheduler.ROOT + "." + A;
final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 9);
conf.setMaximumCapacity(Q_A, 20);
conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*");
final String Q_B = CapacityScheduler.ROOT + "." + B;
final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 90);
conf.setMaximumCapacity(Q_B, 99);
conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*");
final String Q_C = CapacityScheduler.ROOT + "." + C;
final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C;
conf.setCapacity(Q_C, 1);
conf.setMaximumCapacity(Q_C, 10);
conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " ");
conf.setQueues(Q_C, new String[] {C1});
final String Q_C1 = Q_C + "." + C1;
conf.setCapacity(Q_C1, 100);
LOG.info("Setup top-level queues a and b");
}
@ -191,7 +197,7 @@ public class TestLeafQueue {
assertEquals(0.99, b.getMaximumCapacity(), epsilon);
assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C));
ParentQueue c = (ParentQueue)queues.get(C);
assertEquals(0.01, c.getCapacity(), epsilon);
assertEquals(0.01, c.getAbsoluteCapacity(), epsilon);
assertEquals(0.1, c.getMaximumCapacity(), epsilon);
@ -1117,12 +1123,14 @@ public class TestLeafQueue {
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C));
ParentQueue c = (ParentQueue)queues.get(C);
LeafQueue c1 = stubLeafQueue((LeafQueue)queues.get(C1));
assertFalse(root.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
assertTrue(a.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
assertTrue(b.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
assertFalse(c.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
assertFalse(c1.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
assertTrue(hasQueueACL(
a.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
@ -1130,6 +1138,8 @@ public class TestLeafQueue {
b.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
assertFalse(hasQueueACL(
c.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
assertFalse(hasQueueACL(
c1.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
}

24
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
View File

@ -70,13 +70,13 @@ public class TestParentQueue {
private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B});
conf.setCapacity(CapacityScheduler.ROOT, 100);
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String Q_A = CapacityScheduler.ROOT + "." + A;
final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 30);
final String Q_B = CapacityScheduler.ROOT + "." + B;
final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 70);
LOG.info("Setup top-level queues a and b");
@ -144,7 +144,7 @@ public class TestParentQueue {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue root =
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacityScheduler.ROOT, queues, queues,
CapacitySchedulerConfiguration.ROOT, queues, queues,
CapacityScheduler.queueComparator,
CapacityScheduler.applicationComparator,
TestUtils.spyHook);
@ -246,19 +246,19 @@ public class TestParentQueue {
private void setupMultiLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B, C, D});
conf.setCapacity(CapacityScheduler.ROOT, 100);
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String Q_A = CapacityScheduler.ROOT + "." + A;
final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 10);
final String Q_B = CapacityScheduler.ROOT + "." + B;
final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 50);
final String Q_C = CapacityScheduler.ROOT + "." + C;
final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C;
conf.setCapacity(Q_C, 20);
final String Q_D = CapacityScheduler.ROOT + "." + D;
final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D;
conf.setCapacity(Q_D, 20);
// Define 2-nd level queues
@ -282,7 +282,7 @@ public class TestParentQueue {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue root =
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacityScheduler.ROOT, queues, queues,
CapacitySchedulerConfiguration.ROOT, queues, queues,
CapacityScheduler.queueComparator,
CapacityScheduler.applicationComparator,
TestUtils.spyHook);

20
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
View File

@ -42,16 +42,16 @@ public class TestQueueParsing {
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b", "c"});
conf.setCapacity(CapacityScheduler.ROOT, 100);
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String A = CapacityScheduler.ROOT + ".a";
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
final String B = CapacityScheduler.ROOT + ".b";
final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 20);
final String C = CapacityScheduler.ROOT + ".c";
final String C = CapacitySchedulerConfiguration.ROOT + ".c";
conf.setCapacity(C, 70);
LOG.info("Setup top-level queues");
@ -100,7 +100,7 @@ public class TestQueueParsing {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
// non-100 percent value will throw IllegalArgumentException
conf.setCapacity(CapacityScheduler.ROOT, 90);
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 90);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.reinitialize(conf, null, null);
@ -109,14 +109,14 @@ public class TestQueueParsing {
public void testMaxCapacity() throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b", "c"});
conf.setCapacity(CapacityScheduler.ROOT, 100);
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String A = CapacityScheduler.ROOT + ".a";
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 50);
conf.setMaximumCapacity(A, 60);
final String B = CapacityScheduler.ROOT + ".b";
final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 50);
conf.setMaximumCapacity(B, 45); // Should throw an exception

10
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
View File

@ -155,16 +155,16 @@ public class TestRMWebApp {
static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b", "c"});
conf.setCapacity(CapacityScheduler.ROOT, 100);
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String A = CapacityScheduler.ROOT + ".a";
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
final String B = CapacityScheduler.ROOT + ".b";
final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 20);
final String C = CapacityScheduler.ROOT + ".c";
final String C = CapacitySchedulerConfiguration.ROOT + ".c";
conf.setCapacity(C, 70);
// Define 2nd-level queues

8
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
View File

@ -161,7 +161,7 @@ Hadoop MapReduce Next Generation - Capacity Scheduler
A given queue's children can be defined with the configuration knob:
<<<yarn.scheduler.capacity.<queue-path>.queues>>>. Children do not
inherit properties directly from the parent.
inherit properties directly from the parent unless otherwise noted.
Here is an example with three top-level child-queues <<<a>>>, <<<b>>> and
<<<c>>> and some sub-queues for <<<a>>> and <<<b>>>:
@ -275,16 +275,20 @@ Hadoop MapReduce Next Generation - Capacity Scheduler
| | The <ACL> which controls who can <submit> jobs to the given queue. |
| | If the given user/group has necessary ACLs on the given queue or |
| | <one of the parent queues in the hierarchy> they can submit jobs. |
| | <ACLs> for this property <are> inherited from the parent queue |
| | if not specified. |
*--------------------------------------+--------------------------------------+
| <<<yarn.scheduler.capacity.root.<queue-path>.acl_administer_jobs>>> | |
| | The <ACL> which controls who can <administer> jobs on the given queue. |
| | If the given user/group has necessary ACLs on the given queue or |
| | <one of the parent queues in the hierarchy> they can administer jobs. |
| | <ACLs> for this property <are> inherited from the parent queue |
| | if not specified. |
*--------------------------------------+--------------------------------------+
<Note:> An <ACL> is of the form <user1>, <user2><space><group1>, <group2>.
The special value of <<*>> implies <anyone>. The special value of <space>
implies <no one>. The default is <<*>> if not specified.
implies <no one>. The default is <<*>> for the root queue if not specified.
* Reviewing the configuration of the CapacityScheduler