Reverting "YARN-1258: Move to 2.2.1 in CHANGES.txt" because it contained unintended changes

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1534308 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sanford Ryza 2013-10-21 18:37:32 +00:00
parent 65bda6d168
commit 881d1d5c7e
10 changed files with 93 additions and 83 deletions

View File

@ -61,6 +61,8 @@ Release 2.3.0 - UNRELEASED
YARN-976. Document the meaning of a virtual core. (Sandy Ryza) YARN-976. Document the meaning of a virtual core. (Sandy Ryza)
YARN-1258. Allow configuring the Fair Scheduler root queue (Sandy Ryza)
YARN-1182. MiniYARNCluster creates and inits the RM/NM only on start() YARN-1182. MiniYARNCluster creates and inits the RM/NM only on start()
(Karthik Kambatla via Sandy Ryza) (Karthik Kambatla via Sandy Ryza)
@ -93,8 +95,6 @@ Release 2.2.1 - UNRELEASED
YARN-305. Fair scheduler logs too many "Node offered to app" messages. YARN-305. Fair scheduler logs too many "Node offered to app" messages.
(Lohit Vijayarenu via Sandy Ryza) (Lohit Vijayarenu via Sandy Ryza)
YARN-1258. Allow configuring the Fair Scheduler root queue (Sandy Ryza)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

View File

@ -19,10 +19,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler; package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.util.List; import java.util.List;
import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@ -42,6 +44,12 @@ public interface Queue {
*/ */
QueueMetrics getMetrics(); QueueMetrics getMetrics();
/**
* Get ACLs for the queue.
* @return ACLs for the queue
*/
public Map<QueueACL, AccessControlList> getQueueAcls();
/** /**
* Get queue information * Get queue information
* @param includeChildQueues include child queues? * @param includeChildQueues include child queues?

View File

@ -526,6 +526,11 @@ public class LeafQueue implements CSQueue {
return userLimitFactor; return userLimitFactor;
} }
@Override
public synchronized Map<QueueACL, AccessControlList> getQueueAcls() {
return new HashMap<QueueACL, AccessControlList>(acls);
}
@Override @Override
public synchronized QueueInfo getQueueInfo( public synchronized QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) { boolean includeChildQueues, boolean recursive) {

View File

@ -299,6 +299,11 @@ public class ParentQueue implements CSQueue {
return state; return state;
} }
@Override
public synchronized Map<QueueACL, AccessControlList> getQueueAcls() {
return new HashMap<QueueACL, AccessControlList>(acls);
}
@Override @Override
public synchronized QueueInfo getQueueInfo( public synchronized QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) { boolean includeChildQueues, boolean recursive) {

View File

@ -24,12 +24,14 @@ import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
@ -175,7 +177,8 @@ public class FSLeafQueue extends FSQueue {
recordFactory.newRecordInstance(QueueUserACLInfo.class); recordFactory.newRecordInstance(QueueUserACLInfo.class);
List<QueueACL> operations = new ArrayList<QueueACL>(); List<QueueACL> operations = new ArrayList<QueueACL>();
for (QueueACL operation : QueueACL.values()) { for (QueueACL operation : QueueACL.values()) {
if (hasAccess(operation, user)) { Map<QueueACL, AccessControlList> acls = queueMgr.getQueueAcls(getName());
if (acls.get(operation).isUserAllowed(user)) {
operations.add(operation); operations.add(operation);
} }
} }

View File

@ -20,10 +20,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueInfo;
@ -132,6 +135,12 @@ public abstract class FSQueue extends Schedulable implements Queue {
return queueInfo; return queueInfo;
} }
@Override
public Map<QueueACL, AccessControlList> getQueueAcls() {
Map<QueueACL, AccessControlList> acls = queueMgr.getQueueAcls(getName());
return new HashMap<QueueACL, AccessControlList>(acls);
}
@Override @Override
public FSQueueMetrics getMetrics() { public FSQueueMetrics getMetrics() {
return metrics; return metrics;
@ -145,7 +154,7 @@ public abstract class FSQueue extends Schedulable implements Queue {
public boolean hasAccess(QueueACL acl, UserGroupInformation user) { public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
// Check if the leaf-queue allows access // Check if the leaf-queue allows access
if (queueMgr.getQueueAcl(getName(), acl).isUserAllowed(user)) { if (queueMgr.getQueueAcls(getName()).get(acl).isUserAllowed(user)) {
return true; return true;
} }

View File

@ -73,9 +73,6 @@ public class QueueManager {
*/ */
public static final long ALLOC_RELOAD_WAIT = 5 * 1000; public static final long ALLOC_RELOAD_WAIT = 5 * 1000;
private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*");
private static final AccessControlList NOBODY_ACL = new AccessControlList(" ");
private final FairScheduler scheduler; private final FairScheduler scheduler;
// Path to XML file containing allocations. // Path to XML file containing allocations.
@ -385,6 +382,15 @@ public class QueueManager {
queueMetrics.setMaxShare(queue.getMaxShare()); queueMetrics.setMaxShare(queue.getMaxShare());
} }
// Root queue should have empty ACLs. As a queue's ACL is the union of
// its ACL and all its parents' ACLs, setting the roots' to empty will
// neither allow nor prohibit more access to its children.
Map<QueueACL, AccessControlList> rootAcls =
new HashMap<QueueACL, AccessControlList>();
rootAcls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(" "));
rootAcls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(" "));
queueAcls.put(ROOT_QUEUE, rootAcls);
// Create all queus // Create all queus
for (String name: queueNamesInAllocFile) { for (String name: queueNamesInAllocFile) {
getLeafQueue(name, true); getLeafQueue(name, true);
@ -448,10 +454,10 @@ public class QueueManager {
policy.initialize(scheduler.getClusterCapacity()); policy.initialize(scheduler.getClusterCapacity());
queuePolicies.put(queueName, policy); queuePolicies.put(queueName, policy);
} else if ("aclSubmitApps".equals(field.getTagName())) { } else if ("aclSubmitApps".equals(field.getTagName())) {
String text = ((Text)field.getFirstChild()).getData(); String text = ((Text)field.getFirstChild()).getData().trim();
acls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(text)); acls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(text));
} else if ("aclAdministerApps".equals(field.getTagName())) { } else if ("aclAdministerApps".equals(field.getTagName())) {
String text = ((Text)field.getFirstChild()).getData(); String text = ((Text)field.getFirstChild()).getData().trim();
acls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(text)); acls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(text));
} else if ("queue".endsWith(field.getTagName()) || } else if ("queue".endsWith(field.getTagName()) ||
"pool".equals(field.getTagName())) { "pool".equals(field.getTagName())) {
@ -571,16 +577,21 @@ public class QueueManager {
/** /**
* Get the ACLs associated with this queue. If a given ACL is not explicitly * Get the ACLs associated with this queue. If a given ACL is not explicitly
* configured, include the default value for that ACL. The default for the * configured, include the default value for that ACL.
* root queue is everybody ("*") and the default for all other queues is
* nobody ("")
*/ */
public AccessControlList getQueueAcl(String queue, QueueACL operation) { public Map<QueueACL, AccessControlList> getQueueAcls(String queue) {
Map<QueueACL, AccessControlList> queueAcls = info.queueAcls.get(queue); HashMap<QueueACL, AccessControlList> out = new HashMap<QueueACL, AccessControlList>();
if (queueAcls == null || !queueAcls.containsKey(operation)) { Map<QueueACL, AccessControlList> queueAcl = info.queueAcls.get(queue);
return (queue.equals(ROOT_QUEUE)) ? EVERYBODY_ACL : NOBODY_ACL; if (queueAcl != null) {
out.putAll(queueAcl);
} }
return queueAcls.get(operation); if (!out.containsKey(QueueACL.ADMINISTER_QUEUE)) {
out.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList("*"));
}
if (!out.containsKey(QueueACL.SUBMIT_APPLICATIONS)) {
out.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList("*"));
}
return out;
} }
static class QueueManagerInfo { static class QueueManagerInfo {

View File

@ -156,6 +156,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable {
return queueInfo; return queueInfo;
} }
@Override
public Map<QueueACL, AccessControlList> getQueueAcls() { public Map<QueueACL, AccessControlList> getQueueAcls() {
Map<QueueACL, AccessControlList> acls = Map<QueueACL, AccessControlList> acls =
new HashMap<QueueACL, AccessControlList>(); new HashMap<QueueACL, AccessControlList>();

View File

@ -865,25 +865,22 @@ public class TestFairScheduler {
assertEquals(10, queueManager.getUserMaxApps("user1")); assertEquals(10, queueManager.getUserMaxApps("user1"));
assertEquals(5, queueManager.getUserMaxApps("user2")); assertEquals(5, queueManager.getUserMaxApps("user2"));
// Root should get * ACL
assertEquals("*",queueManager.getQueueAcl("root",
QueueACL.ADMINISTER_QUEUE).getAclString());
assertEquals("*", queueManager.getQueueAcl("root",
QueueACL.SUBMIT_APPLICATIONS).getAclString());
// Unspecified queues should get default ACL // Unspecified queues should get default ACL
assertEquals(" ",queueManager.getQueueAcl("root.queueA", Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("root.queueA");
QueueACL.ADMINISTER_QUEUE).getAclString()); assertTrue(aclsA.containsKey(QueueACL.ADMINISTER_QUEUE));
assertEquals(" ", queueManager.getQueueAcl("root.queueA", assertEquals("*", aclsA.get(QueueACL.ADMINISTER_QUEUE).getAclString());
QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertTrue(aclsA.containsKey(QueueACL.SUBMIT_APPLICATIONS));
assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
// Queue B ACL // Queue B ACL
assertEquals("alice,bob admins",queueManager.getQueueAcl("root.queueB", Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
QueueACL.ADMINISTER_QUEUE).getAclString()); assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
// Queue C ACL // Queue c ACL
assertEquals("alice,bob admins",queueManager.getQueueAcl("root.queueC", Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." +
YarnConfiguration.DEFAULT_QUEUE_NAME)); YarnConfiguration.DEFAULT_QUEUE_NAME));
@ -1066,19 +1063,21 @@ public class TestFairScheduler {
assertEquals(5, queueManager.getUserMaxApps("user2")); assertEquals(5, queueManager.getUserMaxApps("user2"));
// Unspecified queues should get default ACL // Unspecified queues should get default ACL
assertEquals(" ", queueManager.getQueueAcl("root.queueA", Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("queueA");
QueueACL.ADMINISTER_QUEUE).getAclString()); assertTrue(aclsA.containsKey(QueueACL.ADMINISTER_QUEUE));
assertEquals(" ", queueManager.getQueueAcl("root.queueA", assertEquals("*", aclsA.get(QueueACL.ADMINISTER_QUEUE).getAclString());
QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertTrue(aclsA.containsKey(QueueACL.SUBMIT_APPLICATIONS));
assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
// Queue B ACL // Queue B ACL
assertEquals("alice,bob admins", queueManager.getQueueAcl("root.queueB", Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
QueueACL.ADMINISTER_QUEUE).getAclString()); assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
// Queue C ACL
assertEquals("alice,bob admins", queueManager.getQueueAcl("root.queueC",
QueueACL.SUBMIT_APPLICATIONS).getAclString());
// Queue c ACL
Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." +
YarnConfiguration.DEFAULT_QUEUE_NAME)); YarnConfiguration.DEFAULT_QUEUE_NAME));
@ -1665,14 +1664,10 @@ public class TestFairScheduler {
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>"); out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>"); out.println("<allocations>");
out.println("<queue name=\"root\">");
out.println(" <aclSubmitApps> </aclSubmitApps>");
out.println(" <aclAdministerApps> </aclAdministerApps>");
out.println("<queue name=\"queue1\">"); out.println("<queue name=\"queue1\">");
out.println("<aclSubmitApps>norealuserhasthisname</aclSubmitApps>"); out.println("<aclSubmitApps>norealuserhasthisname</aclSubmitApps>");
out.println("<aclAdministerApps>norealuserhasthisname</aclAdministerApps>"); out.println("<aclAdministerApps>norealuserhasthisname</aclAdministerApps>");
out.println("</queue>"); out.println("</queue>");
out.println("</queue>");
out.println("</allocations>"); out.println("</allocations>");
out.close(); out.close();
@ -1898,14 +1893,10 @@ public class TestFairScheduler {
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>"); out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>"); out.println("<allocations>");
out.println("<queue name=\"root\">");
out.println(" <aclSubmitApps> </aclSubmitApps>");
out.println(" <aclAdministerApps> </aclAdministerApps>");
out.println("<queue name=\"queue1\">"); out.println("<queue name=\"queue1\">");
out.println("<aclSubmitApps>userallow</aclSubmitApps>"); out.println("<aclSubmitApps>userallow</aclSubmitApps>");
out.println("<aclAdministerApps>userallow</aclAdministerApps>"); out.println("<aclAdministerApps>userallow</aclAdministerApps>");
out.println("</queue>"); out.println("</queue>");
out.println("</queue>");
out.println("</allocations>"); out.println("</allocations>");
out.close(); out.close();
QueueManager queueManager = scheduler.getQueueManager(); QueueManager queueManager = scheduler.getQueueManager();

View File

@ -221,14 +221,10 @@ Allocation file format
for containers, but apps submitted later may run concurrently if there is for containers, but apps submitted later may run concurrently if there is
leftover space on the cluster after satisfying the earlier app's requests. leftover space on the cluster after satisfying the earlier app's requests.
* aclSubmitApps: a list of users and/or groups that can submit apps to the * aclSubmitApps: a list of users that can submit apps to the queue. A (default)
queue. Refer to the ACLs section below for more info on the format of this value of "*" means that any users can submit apps. A queue inherits the ACL of
list and how queue ACLs work. its parent, so if a queue2 descends from queue1, and user1 is in queue1's ACL,
and user2 is in queue2's ACL, then both users may submit to queue2.
* aclAdministerApps: a list of users and/or groups that can administer a
queue. Currently the only administrative action is killing an application.
Refer to the ACLs section below for more info on the format of this list
and how queue ACLs work.
* minSharePreemptionTimeout: number of seconds the queue is under its minimum share * minSharePreemptionTimeout: number of seconds the queue is under its minimum share
before it will try to preempt containers to take resources from other queues. before it will try to preempt containers to take resources from other queues.
@ -250,24 +246,6 @@ Allocation file format
An example allocation file is given here: An example allocation file is given here:
Queue Access Control Lists (ACLs)
Queue Access Control Lists (ACLs) allow administrators to control who may
take actions on particular queues. They are configured with the aclSubmitApps
and aclAdministerApps properties, which can be set per queue. Currently the
only supported administrative action is killing an application. Anybody who
may administer a queue may also submit applications to it. These properties
take values in a format like "user1,user2 group1,group2" or " group1,group2".
An action on a queue will be permitted if its user or group is in the ACL of
that queue or in the ACL of any of that queue's ancestors. So if queue2
is inside queue1, and user1 is in queue1's ACL, and user2 is in queue2's
ACL, then both users may submit to queue2.
The root queue's ACLs are "*" by default which, because ACLs are passed down,
means that everybody may submit to and kill applications from every queue.
To start restricting access, change the root queue's ACLs to something other
than "*".
--- ---
<?xml version="1.0"?> <?xml version="1.0"?>
<allocations> <allocations>
@ -278,7 +256,6 @@ Queue Access Control Lists (ACLs)
<weight>2.0</weight> <weight>2.0</weight>
<schedulingPolicy>fair</schedulingPolicy> <schedulingPolicy>fair</schedulingPolicy>
<queue name="sample_sub_queue"> <queue name="sample_sub_queue">
<aclSubmitApps>charlie</aclSubmitApps>
<minResources>5000 mb,0vcores</minResources> <minResources>5000 mb,0vcores</minResources>
</queue> </queue>
</queue> </queue>