Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4751. In 2.7, Labeled queue usage not shown properly in capacity scheduler UI. Contributed by Eric Payne

This commit is contained in:
Naganarasimha 2016-05-25 04:48:14 +08:00
parent 27aeab63c7
commit 94a9a5d014
6 changed files with 90 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
View File

@ -87,6 +87,8 @@ public abstract class AbstractCSQueue implements CSQueue {
protected CapacitySchedulerContext csContext;
protected YarnAuthorizationProvider authorizer = null;
private Resource clusterResource;
public AbstractCSQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
this.labelManager = cs.getRMContext().getNodeLabelManager();
@ -291,6 +293,7 @@ public abstract class AbstractCSQueue implements CSQueue {
.getReservationContinueLook();
this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this);
this.clusterResource = clusterResource;
}
protected QueueInfo getQueueInfo() {
@ -322,7 +325,9 @@ public abstract class AbstractCSQueue implements CSQueue {
if (nodeLabels == null || nodeLabels.isEmpty()) {
queueUsage.incUsed(resource);
} else {
for (String label : Sets.intersection(accessibleLabels, nodeLabels)) {
Set<String> anls = (accessibleLabels.contains(RMNodeLabelsManager.ANY))
? labelManager.getClusterNodeLabels() : accessibleLabels;
for (String label : Sets.intersection(anls, nodeLabels)) {
queueUsage.incUsed(label, resource);
}
}
@ -338,7 +343,9 @@ public abstract class AbstractCSQueue implements CSQueue {
if (null == nodeLabels || nodeLabels.isEmpty()) {
queueUsage.decUsed(resource);
} else {
for (String label : Sets.intersection(accessibleLabels, nodeLabels)) {
Set<String> anls = (accessibleLabels.contains(RMNodeLabelsManager.ANY))
? labelManager.getClusterNodeLabels() : accessibleLabels;
for (String label : Sets.intersection(anls, nodeLabels)) {
queueUsage.decUsed(label, resource);
}
}
@ -527,4 +534,43 @@ public abstract class AbstractCSQueue implements CSQueue {
// sorry, you cannot access
return false;
}
/**
* Retrieve used resources by this queue for the specified node label.
* Used capacity of label =
* (queue's used resources labeled by nodeLabel)
* / ( (all resources labeled by nodLabel)
* X (percent of labeled resources allocated to this queue) )
* @param nodeLabel label for which to get used resources
* @return used resources by this queue for specified label
*/
public final synchronized float getUsedCapacity(final String nodeLabel) {
Resource availableToQueue =
Resources.multiply(
labelManager.getResourceByLabel(nodeLabel, this.clusterResource),
queueCapacities.getAbsoluteCapacity(nodeLabel));
if (!Resources.greaterThan(resourceCalculator, this.clusterResource,
availableToQueue, Resources.none())) {
return 0.0f;
}
return
Resources.divide(resourceCalculator, this.clusterResource,
queueUsage.getUsed(nodeLabel), availableToQueue);
}
/**
* Retrieve absolute used resources by this queue for the specified node label.
* @param nodeLabel label for which to get absolute used resources
* @return absolute used resources by this queue for specified label
*/
public final synchronized float getAbsoluteUsedCapacity(final String nodeLabel) {
Resource labeledResources =
labelManager.getResourceByLabel(nodeLabel, this.clusterResource);
if (!Resources.greaterThan(resourceCalculator, this.clusterResource,
labeledResources, Resources.none())) {
return 0.0f;
}
return Resources.divide(resourceCalculator, this.clusterResource,
queueUsage.getUsed(nodeLabel), labeledResources);
}
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
View File

@ -284,4 +284,18 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
* @return resourceUsage
*/
public ResourceUsage getQueueResourceUsage();
/**
* Get used capacity for the specified label.
* @param nodeLabel
* @return used capacity
*/
public float getUsedCapacity(String nodeLabel);
/**
* Get absolute used capacity for the specified label.
* @param nodeLabel
* @return absolute used capacity
*/
public float getAbsoluteUsedCapacity(String nodeLabel);
}

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -424,8 +424,12 @@ public class LeafQueue extends AbstractCSQueue {
ArrayList<UserInfo> usersToReturn = new ArrayList<UserInfo>();
for (Map.Entry<String, User> entry : users.entrySet()) {
User user = entry.getValue();
usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(user
.getUsed()), user.getActiveApplications(), user
Resource usedRes = Resource.newInstance(0, 0);
for (String nl : getAccessibleLabelSet()) {
Resources.addTo(usedRes, user.getUsed(nl));
}
usersToReturn.add(new UserInfo(entry.getKey(), usedRes,
user.getActiveApplications(), user
.getPendingApplications(), Resources.clone(user
.getConsumedAMResources()), Resources.clone(user
.getUserResourceLimit())));
@ -433,6 +437,22 @@ public class LeafQueue extends AbstractCSQueue {
return usersToReturn;
}
/**
* Gets the labels which are accessible by this queue. If ANY label can be
* accessed, put all labels in the set.
* @return accessiglbe node labels
*/
protected final Set<String> getAccessibleLabelSet() {
Set<String> nodeLabels = new HashSet<String>();
if (this.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
nodeLabels.addAll(labelManager.getClusterNodeLabels());
} else {
nodeLabels.addAll(this.getAccessibleNodeLabels());
}
nodeLabels.add(RMNodeLabelsManager.NO_LABEL);
return nodeLabels;
}
@Override
public synchronized void reinitialize(
CSQueue newlyParsedQueue, Resource clusterResource)

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
View File

@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo;
@ -319,8 +318,7 @@ class CapacitySchedulerPage extends RmView {
String nodeLabel =
csqinfo.label.length() == 0 ? "<DEFAULT_PARTITION>"
: csqinfo.label;
QueueCapacities queueCapacities = root.getQueueCapacities();
used = queueCapacities.getUsedCapacity(label.getLabelName());
used = root.getUsedCapacity(label.getLabelName());
String partitionUiTag =
"Partition: " + nodeLabel + " " + label.getResource();
ul.li().

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
View File

@ -58,7 +58,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
String label = nodeLabel.getLabelName();
QueueCapacities parentQueueCapacities = parent.getQueueCapacities();
this.queueName = parent.getQueueName();
this.usedCapacity = parentQueueCapacities.getUsedCapacity(label) * 100;
this.usedCapacity = parent.getUsedCapacity(label) * 100;
this.capacity = parentQueueCapacities.getCapacity(label) * 100;
float max = parentQueueCapacities.getMaximumCapacity(label);
if (max < EPSILON || max > 1f)
@ -106,8 +106,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
List<CSQueue> childNonLeafQueues = new ArrayList<>();
for (CSQueue queue : parent.getChildQueues()) {
if (!((AbstractCSQueue) queue).accessibleToPartition(nodeLabel
.getLabelName())) {
// Skip displaying the hierarchy for the queues for which the exclusive
.getLabelName())) {
// Skip displaying the hierarchy for the queues for which the
// labels are not accessible
continue;
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -71,7 +71,7 @@ public class CapacitySchedulerQueueInfo {
queuePath = q.getQueuePath();
capacity = qCapacities.getCapacity(nodeLabel) * 100;
usedCapacity = qCapacities.getUsedCapacity(nodeLabel) * 100;
usedCapacity = q.getUsedCapacity(nodeLabel) * 100;
maxCapacity = qCapacities.getMaximumCapacity(nodeLabel);
if (maxCapacity < EPSILON || maxCapacity > 1f)
@ -82,8 +82,7 @@ public class CapacitySchedulerQueueInfo {
cap(qCapacities.getAbsoluteCapacity(nodeLabel), 0f, 1f) * 100;
absoluteMaxCapacity =
cap(qCapacities.getAbsoluteMaximumCapacity(nodeLabel), 0f, 1f) * 100;
absoluteUsedCapacity =
cap(qCapacities.getAbsoluteUsedCapacity(nodeLabel), 0f, 1f) * 100;
absoluteUsedCapacity = q.getAbsoluteUsedCapacity(nodeLabel) * 100;
numApplications = q.getNumApplications();
queueName = q.getQueueName();
state = q.getState();