Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9937. Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo. 

Contributed by Prabhu Joseph.
This commit is contained in:
Szilard Nemeth 2019-10-29 12:24:37 +05:30 committed by prabhujoseph
parent 66e58b5525
commit 8eda9fcab8
8 changed files with 182 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
View File

@ -86,6 +86,10 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
public PrivilegedEntity getPrivilegedEntity();
Resource getMaximumAllocation();
Resource getMinimumAllocation();
/**
* Get the configured <em>capacity</em> of the queue.
* @return configured queue capacity

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -395,7 +395,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
}
static String getQueuePrefix(String queue) {
public static String getQueuePrefix(String queue) {
String queueName = PREFIX + queue + DOT;
return queueName;
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
View File

@ -55,7 +55,8 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
AppTimeoutInfo.class, ResourceInformationsInfo.class };
AppTimeoutInfo.class, ResourceInformationsInfo.class,
QueueAclsInfo.class, QueueAclInfo.class};
// these dao classes need root unwrapping
final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class,

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
View File

@ -61,7 +61,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
parent.getQueueResourceQuotas(), false);
queues = getQueues(parent);
queues = getQueues(cs, parent);
health = new CapacitySchedulerHealthInfo(cs);
}
@ -89,7 +89,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
return this.queues;
}
protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) {
protected CapacitySchedulerQueueInfoList getQueues(
CapacityScheduler cs, CSQueue parent) {
CapacitySchedulerQueueInfoList queuesInfo =
new CapacitySchedulerQueueInfoList();
// JAXB marashalling leads to situation where the "type" field injected
@ -112,10 +113,10 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
for (CSQueue queue : childQueues) {
CapacitySchedulerQueueInfo info;
if (queue instanceof LeafQueue) {
info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue);
info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue);
} else {
info = new CapacitySchedulerQueueInfo(queue);
info.queues = getQueues(queue);
info = new CapacitySchedulerQueueInfo(cs, queue);
info.queues = getQueues(cs, queue);
}
queuesInfo.addToQueueInfoList(info);
}

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
View File

@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuot
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.AutoCreatedLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
@ -45,6 +46,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected int userLimit;
protected UsersInfo users; // To add another level in the XML
protected float userLimitFactor;
protected float configuredMaxAMResourceLimit;
protected ResourceInfo AMResourceLimit;
protected ResourceInfo usedAMResource;
protected ResourceInfo userAMResourceLimit;
@ -53,15 +55,14 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected String defaultNodeLabelExpression;
protected int defaultPriority;
protected boolean isAutoCreatedLeafQueue;
@XmlTransient
protected String orderingPolicyInfo;
protected long maxApplicationLifetime;
protected long defaultApplicationLifetime;
CapacitySchedulerLeafQueueInfo() {
};
CapacitySchedulerLeafQueueInfo(LeafQueue q) {
super(q);
CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, LeafQueue q) {
super(cs, q);
numActiveApplications = q.getNumActiveApplications();
numPendingApplications = q.getNumPendingApplications();
numContainers = q.getNumContainers();
@ -70,6 +71,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
userLimit = q.getUserLimit();
users = new UsersInfo(q.getUsersManager().getUsersInfo());
userLimitFactor = q.getUserLimitFactor();
configuredMaxAMResourceLimit = q.getMaxAMResourcePerQueuePercent();
AMResourceLimit = new ResourceInfo(q.getAMResourceLimit());
usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed());
preemptionDisabled = q.getPreemptionDisabled();
@ -91,6 +93,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
if ( q instanceof AutoCreatedLeafQueue) {
isAutoCreatedLeafQueue = true;
}
defaultApplicationLifetime = q.getDefaultApplicationLifetime();
maxApplicationLifetime = q.getMaximumApplicationLifetime();
}
@Override
@ -133,6 +137,10 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
return users;
}
public float getConfiguredMaxAMResourceLimit() {
return configuredMaxAMResourceLimit;
}
public float getUserLimitFactor() {
return userLimitFactor;
}
@ -156,10 +164,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public boolean getIntraQueuePreemptionDisabled() {
return intraQueuePreemptionDisabled;
}
public String getOrderingPolicyInfo() {
return orderingPolicyInfo;
}
public String getDefaultNodeLabelExpression() {
return defaultNodeLabelExpression;
@ -172,4 +176,12 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public boolean isAutoCreatedLeafQueue() {
return isAutoCreatedLeafQueue;
}
public long getDefaultApplicationLifetime() {
return defaultApplicationLifetime;
}
public long getMaxApplicationLifetime() {
return maxApplicationLifetime;
}
}

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
import javax.xml.bind.annotation.XmlAccessType;
@ -27,10 +28,16 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
@ -65,11 +72,15 @@ public class CapacitySchedulerQueueInfo {
protected ResourcesInfo resources;
protected ResourceInfo minEffectiveCapacity;
protected ResourceInfo maxEffectiveCapacity;
protected ResourceInfo maximumAllocation;
protected QueueAclsInfo queueAcls;
protected int queuePriority;
protected String orderingPolicyInfo;
CapacitySchedulerQueueInfo() {
};
CapacitySchedulerQueueInfo(CSQueue q) {
CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) {
queuePath = q.getQueuePath();
capacity = q.getCapacity() * 100;
@ -114,6 +125,31 @@ public class CapacitySchedulerQueueInfo {
q.getQueueResourceQuotas().getEffectiveMinResource());
maxEffectiveCapacity = new ResourceInfo(
q.getQueueResourceQuotas().getEffectiveMaxResource());
maximumAllocation = new ResourceInfo(q.getMaximumAllocation());
CapacitySchedulerConfiguration conf = cs.getConfiguration();
queueAcls = new QueueAclsInfo();
for (Map.Entry<AccessType, AccessControlList> e : conf
.getAcls(queuePath).entrySet()) {
QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(),
e.getValue().getAclString());
queueAcls.add(queueAcl);
}
String aclApplicationMaxPriority = "acl_" +
StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString());
String priorityAcls = conf.get(CapacitySchedulerConfiguration
.getQueuePrefix(queuePath) + aclApplicationMaxPriority, conf.ALL_ACL);
QueueAclInfo queueAcl = new QueueAclInfo(
AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls);
queueAcls.add(queueAcl);
queuePriority = q.getPriority().getPriority();
if (q instanceof ParentQueue) {
orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy()
.getConfigName();
}
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@ -220,6 +256,22 @@ public class CapacitySchedulerQueueInfo {
return maxEffectiveCapacity;
}
public ResourceInfo getMaximumAllocation() {
return maximumAllocation;
}
public QueueAclsInfo getQueueAcls() {
return queueAcls;
}
public int getPriority() {
return queuePriority;
}
public String getOrderingPolicyInfo() {
return orderingPolicyInfo;
}
public boolean isLeafQueue() {
return getQueues() == null;
}

48
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java Normal file
View File

@ -0,0 +1,48 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement(name = "queueAcl")
@XmlAccessorType(XmlAccessType.FIELD)
public class QueueAclInfo {
protected String accessType;
protected String accessControlList;
public QueueAclInfo() {
// JAXB needs this
}
public QueueAclInfo(String accessType, String accessControlList) {
this.accessType = accessType;
this.accessControlList = accessControlList;
}
public String getAccessType() {
return accessType;
}
public String getAccessControlList() {
return accessControlList;
}
}

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement(name = "queueAcls")
@XmlAccessorType(XmlAccessType.FIELD)
public class QueueAclsInfo {
protected ArrayList<QueueAclInfo> queueAcl = new ArrayList<QueueAclInfo>();
public QueueAclsInfo() {
} // JAXB needs this
public void add(QueueAclInfo queueAclInfo) {
queueAcl.add(queueAclInfo);
}
public ArrayList<QueueAclInfo> getQueueAcls() {
return queueAcl;
}
public void addAll(ArrayList<QueueAclInfo> queueAclsInfo) {
queueAcl.addAll(queueAclsInfo);
}
}