diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d2fe39225b3..51f76fe92a3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -38,6 +38,8 @@ Release 2.1.1-beta - UNRELEASED IMPROVEMENTS + YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza). + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 12213a3e721..a9b1523eeae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; @@ -60,6 +61,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; @@ -145,6 +147,9 @@ public class RMWebServices { CapacityScheduler cs = (CapacityScheduler) rs; CSQueue root = cs.getRootQueue(); sinfo = new CapacitySchedulerInfo(root); + } else if (rs instanceof FairScheduler) { + FairScheduler fs = (FairScheduler) rs; + sinfo = new FairSchedulerInfo(fs); } else if (rs instanceof FifoScheduler) { sinfo = new FifoSchedulerInfo(this.rm); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java index e1fac4a2cf4..f136f940107 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java @@ -18,14 +18,31 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.XmlType; + import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; -public class FairSchedulerInfo { +@XmlRootElement(name = "fairScheduler") +@XmlType(name = "fairScheduler") +@XmlAccessorType(XmlAccessType.FIELD) +public class FairSchedulerInfo extends SchedulerInfo { + private FairSchedulerQueueInfo rootQueue; + + @XmlTransient private FairScheduler scheduler; + public FairSchedulerInfo() { + } // JAXB needs this + public FairSchedulerInfo(FairScheduler fs) { scheduler = fs; + rootQueue = new FairSchedulerQueueInfo(scheduler.getQueueManager(). + getRootQueue(), scheduler); } public int getAppFairShare(ApplicationAttemptId appAttemptId) { @@ -34,7 +51,6 @@ public class FairSchedulerInfo { } public FairSchedulerQueueInfo getRootQueueInfo() { - return new FairSchedulerQueueInfo(scheduler.getQueueManager(). - getRootQueue(), scheduler); + return rootQueue; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java index 9a74ef4fe06..e3c8e6b1e61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java @@ -20,14 +20,23 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import java.util.Collection; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) public class FairSchedulerLeafQueueInfo extends FairSchedulerQueueInfo { private int numPendingApps; private int numActiveApps; + public FairSchedulerLeafQueueInfo() { + } + public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) { super(queue, scheduler); Collection apps = queue.getAppSchedulables(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java index 087344ff808..fc4732d6bfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java @@ -22,6 +22,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import java.util.ArrayList; import java.util.Collection; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlSeeAlso; +import javax.xml.bind.annotation.XmlTransient; + import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; @@ -29,39 +35,53 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedule import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; import org.apache.hadoop.yarn.util.resource.Resources; +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +@XmlSeeAlso({FairSchedulerLeafQueueInfo.class}) public class FairSchedulerQueueInfo { private int maxApps; + @XmlTransient private float fractionMemUsed; + @XmlTransient private float fractionMemFairShare; + @XmlTransient private float fractionMemMinShare; + @XmlTransient private float fractionMemMaxShare; - private Resource minResources; - private Resource maxResources; - private Resource usedResources; - private Resource fairResources; - private Resource clusterResources; + private ResourceInfo minResources; + private ResourceInfo maxResources; + private ResourceInfo usedResources; + private ResourceInfo fairResources; + private ResourceInfo clusterResources; private String queueName; + private String schedulingPolicy; - private Collection childInfos; + private Collection childQueues; + + public FairSchedulerQueueInfo() { + } public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { QueueManager manager = scheduler.getQueueManager(); queueName = queue.getName(); - - clusterResources = scheduler.getClusterCapacity(); + schedulingPolicy = queue.getPolicy().getName(); - usedResources = queue.getResourceUsage(); + clusterResources = new ResourceInfo(scheduler.getClusterCapacity()); + + usedResources = new ResourceInfo(queue.getResourceUsage()); fractionMemUsed = (float)usedResources.getMemory() / clusterResources.getMemory(); - fairResources = queue.getFairShare(); - minResources = queue.getMinShare(); - maxResources = queue.getMaxShare(); - maxResources = Resources.componentwiseMin(maxResources, clusterResources); + fairResources = new ResourceInfo(queue.getFairShare()); + minResources = new ResourceInfo(queue.getMinShare()); + maxResources = new ResourceInfo(queue.getMaxShare()); + maxResources = new ResourceInfo( + Resources.componentwiseMin(queue.getMaxShare(), + scheduler.getClusterCapacity())); fractionMemFairShare = (float)fairResources.getMemory() / clusterResources.getMemory(); fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory(); @@ -69,13 +89,13 @@ public class FairSchedulerQueueInfo { maxApps = manager.getQueueMaxApps(queueName); - Collection childQueues = queue.getChildQueues(); - childInfos = new ArrayList(); - for (FSQueue child : childQueues) { + Collection children = queue.getChildQueues(); + childQueues = new ArrayList(); + for (FSQueue child : children) { if (child instanceof FSLeafQueue) { - childInfos.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler)); + childQueues.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler)); } else { - childInfos.add(new FairSchedulerQueueInfo(child, scheduler)); + childQueues.add(new FairSchedulerQueueInfo(child, scheduler)); } } } @@ -90,15 +110,15 @@ public class FairSchedulerQueueInfo { /** * Returns the fair share of this queue in megabytes. */ - public Resource getFairShare() { + public ResourceInfo getFairShare() { return fairResources; } - public Resource getMinResources() { + public ResourceInfo getMinResources() { return minResources; } - public Resource getMaxResources() { + public ResourceInfo getMaxResources() { return maxResources; } @@ -110,7 +130,7 @@ public class FairSchedulerQueueInfo { return queueName; } - public Resource getUsedResources() { + public ResourceInfo getUsedResources() { return usedResources; } @@ -138,7 +158,14 @@ public class FairSchedulerQueueInfo { return fractionMemMaxShare; } + /** + * Returns the name of the scheduling policy used by this queue. + */ + public String getSchedulingPolicy() { + return schedulingPolicy; + } + public Collection getChildQueues() { - return childInfos; + return childQueues; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 84276bf105a..6b4422ce61a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -45,4 +45,9 @@ public class ResourceInfo { public int getvCores() { return vCores; } + + @Override + public String toString() { + return ""; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java index 1e042a37d81..898aef76823 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java @@ -22,7 +22,8 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlSeeAlso; @XmlRootElement -@XmlSeeAlso({ CapacitySchedulerInfo.class, FifoSchedulerInfo.class }) +@XmlSeeAlso({ CapacitySchedulerInfo.class, FairSchedulerInfo.class, + FifoSchedulerInfo.class }) public class SchedulerInfo { public SchedulerInfo() {