Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza). 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1512111 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sanford Ryza 2013-08-09 00:26:45 +00:00
parent 7705d57a78
commit 5b001e1a8f
7 changed files with 92 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -38,6 +38,8 @@ Release 2.1.1-beta - UNRELEASED
IMPROVEMENTS
YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza).
OPTIMIZATIONS
BUG FIXES

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
@ -60,6 +61,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
@ -145,6 +147,9 @@ public class RMWebServices {
CapacityScheduler cs = (CapacityScheduler) rs;
CSQueue root = cs.getRootQueue();
sinfo = new CapacitySchedulerInfo(root);
} else if (rs instanceof FairScheduler) {
FairScheduler fs = (FairScheduler) rs;
sinfo = new FairSchedulerInfo(fs);
} else if (rs instanceof FifoScheduler) {
sinfo = new FifoSchedulerInfo(this.rm);
} else {

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
View File

@ -18,14 +18,31 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
public class FairSchedulerInfo {
@XmlRootElement(name = "fairScheduler")
@XmlType(name = "fairScheduler")
@XmlAccessorType(XmlAccessType.FIELD)
public class FairSchedulerInfo extends SchedulerInfo {
private FairSchedulerQueueInfo rootQueue;
@XmlTransient
private FairScheduler scheduler;
public FairSchedulerInfo() {
} // JAXB needs this
public FairSchedulerInfo(FairScheduler fs) {
scheduler = fs;
rootQueue = new FairSchedulerQueueInfo(scheduler.getQueueManager().
getRootQueue(), scheduler);
}
public int getAppFairShare(ApplicationAttemptId appAttemptId) {
@ -34,7 +51,6 @@ public class FairSchedulerInfo {
}
public FairSchedulerQueueInfo getRootQueueInfo() {
return new FairSchedulerQueueInfo(scheduler.getQueueManager().
getRootQueue(), scheduler);
return rootQueue;
}
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
View File

@ -20,14 +20,23 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.Collection;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class FairSchedulerLeafQueueInfo extends FairSchedulerQueueInfo {
private int numPendingApps;
private int numActiveApps;
public FairSchedulerLeafQueueInfo() {
}
public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
super(queue, scheduler);
Collection<AppSchedulable> apps = queue.getAppSchedulables();

73
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
View File

@ -22,6 +22,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.Collection;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@ -29,39 +35,53 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedule
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
import org.apache.hadoop.yarn.util.resource.Resources;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@XmlSeeAlso({FairSchedulerLeafQueueInfo.class})
public class FairSchedulerQueueInfo {
private int maxApps;
@XmlTransient
private float fractionMemUsed;
@XmlTransient
private float fractionMemFairShare;
@XmlTransient
private float fractionMemMinShare;
@XmlTransient
private float fractionMemMaxShare;
private Resource minResources;
private Resource maxResources;
private Resource usedResources;
private Resource fairResources;
private Resource clusterResources;
private ResourceInfo minResources;
private ResourceInfo maxResources;
private ResourceInfo usedResources;
private ResourceInfo fairResources;
private ResourceInfo clusterResources;
private String queueName;
private String schedulingPolicy;
private Collection<FairSchedulerQueueInfo> childInfos;
private Collection<FairSchedulerQueueInfo> childQueues;
public FairSchedulerQueueInfo() {
}
public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) {
QueueManager manager = scheduler.getQueueManager();
queueName = queue.getName();
clusterResources = scheduler.getClusterCapacity();
schedulingPolicy = queue.getPolicy().getName();
usedResources = queue.getResourceUsage();
clusterResources = new ResourceInfo(scheduler.getClusterCapacity());
usedResources = new ResourceInfo(queue.getResourceUsage());
fractionMemUsed = (float)usedResources.getMemory() /
clusterResources.getMemory();
fairResources = queue.getFairShare();
minResources = queue.getMinShare();
maxResources = queue.getMaxShare();
maxResources = Resources.componentwiseMin(maxResources, clusterResources);
fairResources = new ResourceInfo(queue.getFairShare());
minResources = new ResourceInfo(queue.getMinShare());
maxResources = new ResourceInfo(queue.getMaxShare());
maxResources = new ResourceInfo(
Resources.componentwiseMin(queue.getMaxShare(),
scheduler.getClusterCapacity()));
fractionMemFairShare = (float)fairResources.getMemory() / clusterResources.getMemory();
fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory();
@ -69,13 +89,13 @@ public class FairSchedulerQueueInfo {
maxApps = manager.getQueueMaxApps(queueName);
Collection<FSQueue> childQueues = queue.getChildQueues();
childInfos = new ArrayList<FairSchedulerQueueInfo>();
for (FSQueue child : childQueues) {
Collection<FSQueue> children = queue.getChildQueues();
childQueues = new ArrayList<FairSchedulerQueueInfo>();
for (FSQueue child : children) {
if (child instanceof FSLeafQueue) {
childInfos.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
childQueues.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
} else {
childInfos.add(new FairSchedulerQueueInfo(child, scheduler));
childQueues.add(new FairSchedulerQueueInfo(child, scheduler));
}
}
}
@ -90,15 +110,15 @@ public class FairSchedulerQueueInfo {
/**
* Returns the fair share of this queue in megabytes.
*/
public Resource getFairShare() {
public ResourceInfo getFairShare() {
return fairResources;
}
public Resource getMinResources() {
public ResourceInfo getMinResources() {
return minResources;
}
public Resource getMaxResources() {
public ResourceInfo getMaxResources() {
return maxResources;
}
@ -110,7 +130,7 @@ public class FairSchedulerQueueInfo {
return queueName;
}
public Resource getUsedResources() {
public ResourceInfo getUsedResources() {
return usedResources;
}
@ -138,7 +158,14 @@ public class FairSchedulerQueueInfo {
return fractionMemMaxShare;
}
/**
* Returns the name of the scheduling policy used by this queue.
*/
public String getSchedulingPolicy() {
return schedulingPolicy;
}
public Collection<FairSchedulerQueueInfo> getChildQueues() {
return childInfos;
return childQueues;
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
View File

@ -45,4 +45,9 @@ public class ResourceInfo {
public int getvCores() {
return vCores;
}
@Override
public String toString() {
return "<memory:" + memory + ", vCores:" + vCores + ">";
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
View File

@ -22,7 +22,8 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
@XmlRootElement
@XmlSeeAlso({ CapacitySchedulerInfo.class, FifoSchedulerInfo.class })
@XmlSeeAlso({ CapacitySchedulerInfo.class, FairSchedulerInfo.class,
FifoSchedulerInfo.class })
public class SchedulerInfo {
public SchedulerInfo() {