From 54d0bf8935e35aad0f4d67df358ceb970cfcd713 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Tue, 21 Aug 2018 19:00:31 +0800 Subject: [PATCH] YARN-8683. Support to display pending scheduling requests in RM app attempt page. Contributed by Tao Yang. --- .../scheduler/AbstractYarnScheduler.java | 10 ++++ .../scheduler/AppSchedulingInfo.java | 17 ++++++ .../scheduler/YarnScheduler.java | 10 ++++ .../placement/AppPlacementAllocator.java | 6 +++ .../LocalityAppPlacementAllocator.java | 5 ++ ...SingleConstraintAppPlacementAllocator.java | 4 +- .../webapp/RMAppAttemptBlock.java | 24 +++++++-- .../resourcemanager/webapp/dao/AppInfo.java | 10 ++++ .../webapp/dao/ResourceRequestInfo.java | 52 +++++++++++++++++-- 9 files changed, 129 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index d2e81a50d94..9d2b0586dc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -877,6 +877,16 @@ public abstract class AbstractYarnScheduler return null; } + @Override + public List getPendingSchedulingRequestsForAttempt( + ApplicationAttemptId attemptId) { + SchedulerApplicationAttempt attempt = getApplicationAttempt(attemptId); + if (attempt != null) { + return attempt.getAppSchedulingInfo().getAllSchedulingRequests(); + } + return null; + } + @Override public Priority checkAndGetApplicationPriority( Priority priorityRequestedByApp, UserGroupInformation user, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 8074f069698..d63d2b82fbd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -476,6 +476,23 @@ public class AppSchedulingInfo { return ret; } + /** + * Fetch SchedulingRequests. + * @return All pending SchedulingRequests. + */ + public List getAllSchedulingRequests() { + List ret = new ArrayList<>(); + try { + this.readLock.lock(); + schedulerKeyToAppPlacementAllocator.values().stream() + .filter(ap -> ap.getSchedulingRequest() != null) + .forEach(ap -> ret.add(ap.getSchedulingRequest())); + } finally { + this.readLock.unlock(); + } + return ret; + } + public PendingAsk getNextPendingAsk() { try { readLock.lock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 43d55c4f1dc..0f7a5b5b3ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -363,6 +363,16 @@ public interface YarnScheduler extends EventHandler { List getPendingResourceRequestsForAttempt( ApplicationAttemptId attemptId); + /** + * Get pending scheduling request for specified application attempt. + * + * @param attemptId the id of the application attempt + * + * @return pending scheduling requests + */ + List getPendingSchedulingRequestsForAttempt( + ApplicationAttemptId attemptId); + /** * Get cluster max priority. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java index df58157efea..088b3dd9c70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java @@ -192,4 +192,10 @@ public abstract class AppPlacementAllocator { this.rmContext = rmContext; this.schedulerRequestKey = schedulerRequestKey; } + + /** + * Get pending SchedulingRequest. + * @return SchedulingRequest + */ + public abstract SchedulingRequest getSchedulingRequest(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java index e1239a9db34..f1df3432a86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java @@ -425,4 +425,9 @@ public class LocalityAppPlacementAllocator writeLock.unlock(); } } + + @Override + public SchedulingRequest getSchedulingRequest() { + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java index 2b610f2fe76..914f35d7bf0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java @@ -505,8 +505,8 @@ public class SingleConstraintAppPlacementAllocator } } - @VisibleForTesting - SchedulingRequest getSchedulingRequest() { + @Override + public SchedulingRequest getSchedulingRequest() { return schedulingRequest; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java index 43a6ac94b05..99a569af890 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -87,10 +87,14 @@ public class RMAppAttemptBlock extends AppAttemptBlock{ .h3("Total Outstanding Resource Requests: " + getTotalResource(resourceRequests)) .table("#resourceRequests").thead().tr().th(".priority", "Priority") + .th(".allocationRequestId", "AllocationRequestId") .th(".resource", "ResourceName").th(".capacity", "Capability") .th(".containers", "NumContainers") .th(".relaxlocality", "RelaxLocality") - .th(".labelexpression", "NodeLabelExpression").__().__().tbody(); + .th(".labelexpression", "NodeLabelExpression") + .th(".executiontype", "ExecutionType") + .th(".allocationTags", "AllocationTags") + .th(".placementConstraint", "PlacementConstraint").__().__().tbody(); StringBuilder resourceRequestTableData = new StringBuilder("[\n"); for (ResourceRequestInfo resourceRequest : resourceRequests) { @@ -99,7 +103,11 @@ public class RMAppAttemptBlock extends AppAttemptBlock{ } resourceRequestTableData.append("[\"") .append(String.valueOf(resourceRequest.getPriority())).append("\",\"") - .append(resourceRequest.getResourceName()).append("\",\"") + .append(String.valueOf(resourceRequest.getAllocationRequestId())) + .append("\",\"") + .append(resourceRequest.getResourceName() == null ? "N/A" + : resourceRequest.getResourceName()) + .append("\",\"") .append(StringEscapeUtils.escapeEcmaScript(StringEscapeUtils .escapeHtml4(String.valueOf(resourceRequest.getCapability())))) .append("\",\"") @@ -109,6 +117,15 @@ public class RMAppAttemptBlock extends AppAttemptBlock{ .append("\",\"") .append(resourceRequest.getNodeLabelExpression() == null ? "N/A" : resourceRequest.getNodeLabelExpression()) + .append("\",\"") + .append(resourceRequest.getExecutionTypeRequest() == null ? "N/A" + : resourceRequest.getExecutionTypeRequest().getExecutionType()) + .append("\",\"") + .append(resourceRequest.getAllocationTags() == null ? "N/A" : + StringUtils.join(resourceRequest.getAllocationTags(), ",")) + .append("\",\"") + .append(resourceRequest.getPlacementConstraint() == null ? "N/A" + : resourceRequest.getPlacementConstraint()) .append("\"],\n"); } if (resourceRequestTableData @@ -132,7 +149,8 @@ public class RMAppAttemptBlock extends AppAttemptBlock{ if (request.getNumContainers() == 0) { continue; } - if (request.getResourceName().equals(ResourceRequest.ANY)) { + if (request.getResourceName() == null || request.getResourceName() + .equals(ResourceRequest.ANY)) { Resources.addTo( totalResource, Resources.multiply(request.getCapability().getResource(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 9d82bc78c76..63b6fe072d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -230,6 +231,15 @@ public class AppInfo { resourceRequests.add(new ResourceRequestInfo(req)); } } + + List schedulingRequestsRaw = rm.getRMContext() + .getScheduler().getPendingSchedulingRequestsForAttempt( + attempt.getAppAttemptId()); + if (schedulingRequestsRaw != null) { + for (SchedulingRequest req : schedulingRequestsRaw) { + resourceRequests.add(new ResourceRequestInfo(req)); + } + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java index 030af45cced..beab9d4f24a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java @@ -21,11 +21,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import java.util.Set; /** * Simple class representing a resource request. @@ -36,6 +38,8 @@ public class ResourceRequestInfo { @XmlElement(name = "priority") private int priority; + @XmlElement(name = "allocationRequestId") + private long allocationRequestId; @XmlElement(name = "resourceName") private String resourceName; @XmlElement(name = "capability") @@ -50,16 +54,17 @@ public class ResourceRequestInfo { @XmlElement(name = "executionTypeRequest") private ExecutionTypeRequestInfo executionTypeRequest; - @XmlElement(name = "executionType") - private String executionType; - @XmlElement(name = "enforceExecutionType") - private boolean enforceExecutionType; + @XmlElement(name = "placementConstraint") + private String placementConstraint; + @XmlElement(name = "allocationTags") + private Set allocationTags; public ResourceRequestInfo() { } public ResourceRequestInfo(ResourceRequest request) { priority = request.getPriority().getPriority(); + allocationRequestId = request.getAllocationRequestId(); resourceName = request.getResourceName(); capability = new ResourceInfo(request.getCapability()); numContainers = request.getNumContainers(); @@ -71,6 +76,21 @@ public class ResourceRequestInfo { } } + public ResourceRequestInfo(SchedulingRequest request) { + priority = request.getPriority().getPriority(); + allocationRequestId = request.getAllocationRequestId(); + capability = new ResourceInfo(request.getResourceSizing().getResources()); + numContainers = request.getResourceSizing().getNumAllocations(); + if (request.getExecutionType() != null) { + executionTypeRequest = + new ExecutionTypeRequestInfo(request.getExecutionType()); + } + allocationTags = request.getAllocationTags(); + if (request.getPlacementConstraint() != null) { + placementConstraint = request.getPlacementConstraint().toString(); + } + } + public Priority getPriority() { return Priority.newInstance(priority); } @@ -128,4 +148,28 @@ public class ResourceRequestInfo { public ExecutionTypeRequestInfo getExecutionTypeRequest() { return executionTypeRequest; } + + public String getPlacementConstraint() { + return placementConstraint; + } + + public void setPlacementConstraint(String placementConstraint) { + this.placementConstraint = placementConstraint; + } + + public Set getAllocationTags() { + return allocationTags; + } + + public void setAllocationTags(Set allocationTags) { + this.allocationTags = allocationTags; + } + + public long getAllocationRequestId() { + return allocationRequestId; + } + + public void setAllocationRequestId(long allocationRequestId) { + this.allocationRequestId = allocationRequestId; + } }