diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index c0c280e9f1e..1af325023d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -86,6 +86,10 @@ public interface CSQueue extends SchedulerQueue { public PrivilegedEntity getPrivilegedEntity(); + Resource getMaximumAllocation(); + + Resource getMinimumAllocation(); + /** * Get the configured capacity of the queue. * @return configured queue capacity diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 760636e1497..ada554cac13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -395,7 +395,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur } } - static String getQueuePrefix(String queue) { + public static String getQueuePrefix(String queue) { String queueName = PREFIX + queue + DOT; return queueName; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index 2e4204e08af..8c9c0568fa2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -55,7 +55,8 @@ public class JAXBContextResolver implements ContextResolver { UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, - AppTimeoutInfo.class, ResourceInformationsInfo.class }; + AppTimeoutInfo.class, ResourceInformationsInfo.class, + QueueAclsInfo.class, QueueAclInfo.class}; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index ca420419c89..b8a5426c77e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -61,7 +61,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), parent.getQueueResourceQuotas(), false); - queues = getQueues(parent); + queues = getQueues(cs, parent); health = new CapacitySchedulerHealthInfo(cs); } @@ -89,7 +89,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo { return this.queues; } - protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) { + protected CapacitySchedulerQueueInfoList getQueues( + CapacityScheduler cs, CSQueue parent) { CapacitySchedulerQueueInfoList queuesInfo = new CapacitySchedulerQueueInfoList(); // JAXB marashalling leads to situation where the "type" field injected @@ -112,10 +113,10 @@ public class CapacitySchedulerInfo extends SchedulerInfo { for (CSQueue queue : childQueues) { CapacitySchedulerQueueInfo info; if (queue instanceof LeafQueue) { - info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue); + info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue); } else { - info = new CapacitySchedulerQueueInfo(queue); - info.queues = getQueues(queue); + info = new CapacitySchedulerQueueInfo(cs, queue); + info.queues = getQueues(cs, queue); } queuesInfo.addToQueueInfoList(info); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index a53e9212eeb..64cc04ae1fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuot import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .AutoCreatedLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; @@ -45,6 +46,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { protected int userLimit; protected UsersInfo users; // To add another level in the XML protected float userLimitFactor; + protected float configuredMaxAMResourceLimit; protected ResourceInfo AMResourceLimit; protected ResourceInfo usedAMResource; protected ResourceInfo userAMResourceLimit; @@ -53,15 +55,14 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { protected String defaultNodeLabelExpression; protected int defaultPriority; protected boolean isAutoCreatedLeafQueue; - - @XmlTransient - protected String orderingPolicyInfo; + protected long maxApplicationLifetime; + protected long defaultApplicationLifetime; CapacitySchedulerLeafQueueInfo() { }; - CapacitySchedulerLeafQueueInfo(LeafQueue q) { - super(q); + CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, LeafQueue q) { + super(cs, q); numActiveApplications = q.getNumActiveApplications(); numPendingApplications = q.getNumPendingApplications(); numContainers = q.getNumContainers(); @@ -70,6 +71,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { userLimit = q.getUserLimit(); users = new UsersInfo(q.getUsersManager().getUsersInfo()); userLimitFactor = q.getUserLimitFactor(); + configuredMaxAMResourceLimit = q.getMaxAMResourcePerQueuePercent(); AMResourceLimit = new ResourceInfo(q.getAMResourceLimit()); usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed()); preemptionDisabled = q.getPreemptionDisabled(); @@ -91,6 +93,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { if ( q instanceof AutoCreatedLeafQueue) { isAutoCreatedLeafQueue = true; } + defaultApplicationLifetime = q.getDefaultApplicationLifetime(); + maxApplicationLifetime = q.getMaximumApplicationLifetime(); } @Override @@ -133,6 +137,10 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { return users; } + public float getConfiguredMaxAMResourceLimit() { + return configuredMaxAMResourceLimit; + } + public float getUserLimitFactor() { return userLimitFactor; } @@ -156,10 +164,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { public boolean getIntraQueuePreemptionDisabled() { return intraQueuePreemptionDisabled; } - - public String getOrderingPolicyInfo() { - return orderingPolicyInfo; - } public String getDefaultNodeLabelExpression() { return defaultNodeLabelExpression; @@ -172,4 +176,12 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { public boolean isAutoCreatedLeafQueue() { return isAutoCreatedLeafQueue; } + + public long getDefaultApplicationLifetime() { + return defaultApplicationLifetime; + } + + public long getMaxApplicationLifetime() { + return maxApplicationLifetime; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index fd256b143bf..c066e0d4b73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import java.util.ArrayList; import java.util.Collections; +import java.util.Map; import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; @@ -27,10 +28,16 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @@ -65,11 +72,15 @@ public class CapacitySchedulerQueueInfo { protected ResourcesInfo resources; protected ResourceInfo minEffectiveCapacity; protected ResourceInfo maxEffectiveCapacity; + protected ResourceInfo maximumAllocation; + protected QueueAclsInfo queueAcls; + protected int queuePriority; + protected String orderingPolicyInfo; CapacitySchedulerQueueInfo() { }; - CapacitySchedulerQueueInfo(CSQueue q) { + CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) { queuePath = q.getQueuePath(); capacity = q.getCapacity() * 100; @@ -114,6 +125,31 @@ public class CapacitySchedulerQueueInfo { q.getQueueResourceQuotas().getEffectiveMinResource()); maxEffectiveCapacity = new ResourceInfo( q.getQueueResourceQuotas().getEffectiveMaxResource()); + maximumAllocation = new ResourceInfo(q.getMaximumAllocation()); + + CapacitySchedulerConfiguration conf = cs.getConfiguration(); + queueAcls = new QueueAclsInfo(); + for (Map.Entry e : conf + .getAcls(queuePath).entrySet()) { + QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(), + e.getValue().getAclString()); + queueAcls.add(queueAcl); + } + + String aclApplicationMaxPriority = "acl_" + + StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString()); + String priorityAcls = conf.get(CapacitySchedulerConfiguration + .getQueuePrefix(queuePath) + aclApplicationMaxPriority, conf.ALL_ACL); + + QueueAclInfo queueAcl = new QueueAclInfo( + AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls); + queueAcls.add(queueAcl); + + queuePriority = q.getPriority().getPriority(); + if (q instanceof ParentQueue) { + orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() + .getConfigName(); + } } protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { @@ -220,6 +256,22 @@ public class CapacitySchedulerQueueInfo { return maxEffectiveCapacity; } + public ResourceInfo getMaximumAllocation() { + return maximumAllocation; + } + + public QueueAclsInfo getQueueAcls() { + return queueAcls; + } + + public int getPriority() { + return queuePriority; + } + + public String getOrderingPolicyInfo() { + return orderingPolicyInfo; + } + public boolean isLeafQueue() { return getQueues() == null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java new file mode 100644 index 00000000000..8d78ed53a9f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "queueAcl") +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueAclInfo { + + protected String accessType; + protected String accessControlList; + + public QueueAclInfo() { + // JAXB needs this + } + + public QueueAclInfo(String accessType, String accessControlList) { + this.accessType = accessType; + this.accessControlList = accessControlList; + } + + public String getAccessType() { + return accessType; + } + + public String getAccessControlList() { + return accessControlList; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java new file mode 100644 index 00000000000..38e836e730f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "queueAcls") +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueAclsInfo { + + protected ArrayList queueAcl = new ArrayList(); + + public QueueAclsInfo() { + } // JAXB needs this + + public void add(QueueAclInfo queueAclInfo) { + queueAcl.add(queueAclInfo); + } + + public ArrayList getQueueAcls() { + return queueAcl; + } + + public void addAll(ArrayList queueAclsInfo) { + queueAcl.addAll(queueAclsInfo); + } + +}