From edcecedc1c39d54db0f86a1325b4db26c38d2d4d Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Fri, 27 Feb 2015 16:13:32 -0800 Subject: [PATCH] YARN-3262. Surface application outstanding resource requests table in RM web UI. (Jian He via wangda) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../impl/pb/ResourceRequestPBImpl.java | 4 +- .../scheduler/AbstractYarnScheduler.java | 9 ++++ .../scheduler/AppSchedulingInfo.java | 33 ++++++------- .../SchedulerApplicationAttempt.java | 6 ++- .../resourcemanager/webapp/AppBlock.java | 46 ++++++++++++++++++- .../resourcemanager/webapp/AppPage.java | 4 ++ .../resourcemanager/webapp/AppsBlock.java | 5 +- .../webapp/FairSchedulerAppsBlock.java | 5 +- .../resourcemanager/webapp/RMWebServices.java | 6 +-- .../resourcemanager/webapp/dao/AppInfo.java | 17 ++++++- .../webapp/TestRMWebAppFairScheduler.java | 10 +++- .../webapp/TestRMWebServicesApps.java | 3 +- 13 files changed, 118 insertions(+), 33 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 38dd9fa2363..e7af84bd71f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -336,6 +336,9 @@ Release 2.7.0 - UNRELEASED YARN-2820. Retry in FileSystemRMStateStore when FS's operations fail due to IOException. (Zhihai Xu via ozawa) + YARN-3262. Surface application outstanding resource requests table + in RM web UI. (Jian He via wangda) + OPTIMIZATIONS YARN-2990. FairScheduler's delay-scheduling always waits for node-local and diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java index 0c8491fc29a..27fb5ae40c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java @@ -140,13 +140,13 @@ public class ResourceRequestPBImpl extends ResourceRequest { this.capability = capability; } @Override - public int getNumContainers() { + public synchronized int getNumContainers() { ResourceRequestProtoOrBuilder p = viaProto ? proto : builder; return (p.getNumContainers()); } @Override - public void setNumContainers(int numContainers) { + public synchronized void setNumContainers(int numContainers) { maybeInitBuilder(); builder.setNumContainers((numContainers)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 04b3452fe39..968a767f517 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -658,4 +658,13 @@ public abstract class AbstractYarnScheduler maxAllocWriteLock.unlock(); } } + + public List getPendingResourceRequestsForAttempt( + ApplicationAttemptId attemptId) { + SchedulerApplicationAttempt attempt = getApplicationAttempt(attemptId); + if (attempt != null) { + return attempt.getAppSchedulingInfo().getAllResourceRequests(); + } + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index a9a459f95e0..97dc231815d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -20,12 +20,14 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; @@ -64,7 +66,7 @@ public class AppSchedulingInfo { final Set priorities = new TreeSet( new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator()); final Map> requests = - new HashMap>(); + new ConcurrentHashMap>(); private Set blacklist = new HashSet(); //private final ApplicationStore store; @@ -159,7 +161,7 @@ public class AppSchedulingInfo { Map asks = this.requests.get(priority); if (asks == null) { - asks = new HashMap(); + asks = new ConcurrentHashMap(); this.requests.put(priority, asks); this.priorities.add(priority); } @@ -221,7 +223,7 @@ public class AppSchedulingInfo { return requests.get(priority); } - synchronized public List getAllResourceRequests() { + public List getAllResourceRequests() { List ret = new ArrayList(); for (Map r : requests.values()) { ret.addAll(r.values()); @@ -300,17 +302,11 @@ public class AppSchedulingInfo { Priority priority, ResourceRequest nodeLocalRequest, Container container, List resourceRequests) { // Update future requirements - nodeLocalRequest.setNumContainers(nodeLocalRequest.getNumContainers() - 1); - if (nodeLocalRequest.getNumContainers() == 0) { - this.requests.get(priority).remove(node.getNodeName()); - } + decResourceRequest(node.getNodeName(), priority, nodeLocalRequest); ResourceRequest rackLocalRequest = requests.get(priority).get( node.getRackName()); - rackLocalRequest.setNumContainers(rackLocalRequest.getNumContainers() - 1); - if (rackLocalRequest.getNumContainers() == 0) { - this.requests.get(priority).remove(node.getRackName()); - } + decResourceRequest(node.getRackName(), priority, rackLocalRequest); ResourceRequest offRackRequest = requests.get(priority).get( ResourceRequest.ANY); @@ -322,6 +318,14 @@ public class AppSchedulingInfo { resourceRequests.add(cloneResourceRequest(offRackRequest)); } + private void decResourceRequest(String resourceName, Priority priority, + ResourceRequest request) { + request.setNumContainers(request.getNumContainers() - 1); + if (request.getNumContainers() == 0) { + requests.get(priority).remove(resourceName); + } + } + /** * The {@link ResourceScheduler} is allocating data-local resources to the * application. @@ -333,11 +337,8 @@ public class AppSchedulingInfo { Priority priority, ResourceRequest rackLocalRequest, Container container, List resourceRequests) { // Update future requirements - rackLocalRequest.setNumContainers(rackLocalRequest.getNumContainers() - 1); - if (rackLocalRequest.getNumContainers() == 0) { - this.requests.get(priority).remove(node.getRackName()); - } - + decResourceRequest(node.getRackName(), priority, rackLocalRequest); + ResourceRequest offRackRequest = requests.get(priority).get( ResourceRequest.ANY); decrementOutstanding(offRackRequest); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index d5b6ce66987..532df05d592 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -153,7 +153,11 @@ public class SchedulerApplicationAttempt { public synchronized Collection getLiveContainers() { return new ArrayList(liveContainers.values()); } - + + public AppSchedulingInfo getAppSchedulingInfo() { + return this.appSchedulingInfo; + } + /** * Is this application pending? * @return true if it is else false. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index c2b376e8309..62ad8dfce44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; @@ -50,6 +51,7 @@ import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; @@ -90,7 +92,8 @@ public class AppBlock extends HtmlBlock { puts("Application not found: "+ aid); return; } - AppInfo app = new AppInfo(rmApp, true, WebAppUtils.getHttpSchemePrefix(conf)); + AppInfo app = + new AppInfo(rm, rmApp, true, WebAppUtils.getHttpSchemePrefix(conf)); // Check for the authorization. String remoteUser = request().getRemoteUser(); @@ -134,7 +137,7 @@ public class AppBlock extends HtmlBlock { ._("Application Type:", app.getApplicationType()) ._("Application Tags:", app.getApplicationTags()) ._("YarnApplicationState:", clarifyAppState(app.getState())) - ._("FinalStatus reported by AM:", + ._("FinalStatus Reported by AM:", clairfyAppFinalStatus(app.getFinalStatus())) ._("Started:", Times.format(app.getStartTime())) ._("Elapsed:", @@ -200,6 +203,45 @@ public class AppBlock extends HtmlBlock { table._(); div._(); + + createResourceRequestsTable(html, app); + } + + private void createResourceRequestsTable(Block html, AppInfo app) { + TBODY> tbody = + html.table("#ResourceRequests").thead().tr() + .th(".priority", "Priority") + .th(".resourceName", "ResourceName") + .th(".totalResource", "Capability") + .th(".numContainers", "NumContainers") + .th(".relaxLocality", "RelaxLocality") + .th(".nodeLabelExpression", "NodeLabelExpression")._()._().tbody(); + + Resource totalResource = Resource.newInstance(0, 0); + if (app.getResourceRequests() != null) { + for (ResourceRequest request : app.getResourceRequests()) { + if (request.getNumContainers() == 0) { + continue; + } + + tbody.tr() + .td(String.valueOf(request.getPriority())) + .td(request.getResourceName()) + .td(String.valueOf(request.getCapability())) + .td(String.valueOf(request.getNumContainers())) + .td(String.valueOf(request.getRelaxLocality())) + .td(request.getNodeLabelExpression() == null ? "N/A" : request + .getNodeLabelExpression())._(); + if (request.getResourceName().equals(ResourceRequest.ANY)) { + Resources.addTo(totalResource, + Resources.multiply(request.getCapability(), + request.getNumContainers())); + } + } + } + html.div().$class("totalResourceRequests") + .h3("Total Outstanding Resource Requests: " + totalResource)._(); + tbody._()._(); } private String clarifyAppState(YarnApplicationState state) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java index a55c62f4b59..899332477f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java @@ -18,12 +18,16 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; + import org.apache.hadoop.yarn.webapp.SubView; public class AppPage extends RmView { @Override protected void preHead(Page.HTML<_> html) { commonPreHead(html); + set(DATATABLES_ID, "ResourceRequests"); + setTableStyles(html, "ResourceRequests"); } @Override protected Class content() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java index 054a1a7020e..935be612c72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java @@ -46,12 +46,13 @@ import com.google.inject.Inject; class AppsBlock extends HtmlBlock { final ConcurrentMap apps; private final Configuration conf; - + final ResourceManager rm; @Inject AppsBlock(ResourceManager rm, ViewContext ctx, Configuration conf) { super(ctx); apps = rm.getRMContext().getRMApps(); this.conf = conf; + this.rm = rm; } @Override public void render(Block html) { @@ -85,7 +86,7 @@ class AppsBlock extends HtmlBlock { if (reqAppStates != null && !reqAppStates.contains(app.createApplicationState())) { continue; } - AppInfo appInfo = new AppInfo(app, true, WebAppUtils.getHttpSchemePrefix(conf)); + AppInfo appInfo = new AppInfo(rm, app, true, WebAppUtils.getHttpSchemePrefix(conf)); String percent = String.format("%.1f", appInfo.getProgress()); //AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js appsTableData.append("[\"