From 154c9d2e421390c808dea3b2826cdaae492ea337 Mon Sep 17 00:00:00 2001 From: Arun Suresh Date: Wed, 5 Aug 2015 23:14:14 -0700 Subject: [PATCH] YARN-3961. Expose pending, running and reserved containers of a queue in REST api and yarn top (adhoot via asuresh) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../yarn/api/records/QueueStatistics.java | 36 +++++++++++++++++++ .../src/main/proto/yarn_protos.proto | 3 ++ .../apache/hadoop/yarn/client/cli/TopCLI.java | 24 +++++++++---- .../impl/pb/QueueStatisticsPBImpl.java | 36 +++++++++++++++++++ .../scheduler/capacity/AbstractCSQueue.java | 3 ++ .../dao/CapacitySchedulerQueueInfo.java | 18 ++++++++++ .../webapp/dao/FairSchedulerQueueInfo.java | 24 +++++++++++-- .../TestRMWebServicesCapacitySched.java | 4 +-- 9 files changed, 141 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 29a7630448e..6b409ddd5c6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -371,6 +371,9 @@ Release 2.8.0 - UNRELEASED YARN-4004. container-executor should print output of docker logs if the docker container exits with non-0 exit status. (Varun Vasudev via xgong) + YARN-3961. Expose pending, running and reserved containers of a queue in REST + api and yarn top (adhoot via asuresh) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueStatistics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueStatistics.java index a93047e725d..808766364f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueStatistics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueStatistics.java @@ -262,6 +262,42 @@ public abstract class QueueStatistics { */ public abstract void setPendingVCores(long pendingVCores); + /** + * Get the number of pending containers. + * @return the number of pending containers. + */ + public abstract long getPendingContainers(); + + /** + * Set the number of pending containers. + * @param pendingContainers the pending containers. + */ + public abstract void setPendingContainers(long pendingContainers); + + /** + * Get the number of allocated containers. + * @return the number of allocated containers. + */ + public abstract long getAllocatedContainers(); + + /** + * Set the number of allocated containers. + * @param allocatedContainers the allocated containers. + */ + public abstract void setAllocatedContainers(long allocatedContainers); + + /** + * Get the number of reserved containers. + * @return the number of reserved containers. + */ + public abstract long getReservedContainers(); + + /** + * Set the number of reserved containers. + * @param reservedContainers the reserved containers. + */ + public abstract void setReservedContainers(long reservedContainers); + /** * Get the reserved vcores * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 7bd03975a1a..fdacab3edf0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -384,6 +384,9 @@ message QueueStatisticsProto { optional int64 allocatedVCores = 13; optional int64 pendingVCores = 14; optional int64 reservedVCores = 15; + optional int64 allocatedContainers = 16; + optional int64 pendingContainers = 17; + optional int64 reservedContainers = 18; } message QueueInfoProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java index 2b7d7f3ec67..423fb3c8733 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java @@ -71,7 +71,7 @@ public class TopCLI extends YarnCLI { private String SET_CURSOR_HOME = "\u001b[H"; private String CHANGE_BACKGROUND = "\u001b[7m"; private String RESET_BACKGROUND = "\u001b[0m"; - private String SET_CURSOR_LINE_6_COLUMN_0 = "\u001b[6;0f"; + private String SET_CURSOR_LINE_7_COLUMN_0 = "\u001b[7;0f"; // guava cache for getapplications call protected Cache> @@ -331,6 +331,9 @@ public class TopCLI extends YarnCLI { long allocatedVCores; long pendingVCores; long reservedVCores; + long allocatedContainers; + long reservedContainers; + long pendingContainers; } private class KeyboardMonitor extends Thread { @@ -596,14 +599,14 @@ public class TopCLI extends YarnCLI { String[] tput_cursor_home = { "tput", "cup", "0", "0" }; String[] tput_clear = { "tput", "clear" }; String[] tput_clear_line = { "tput", "el" }; - String[] tput_set_cursor_line_6_column_0 = { "tput", "cup", "5", "0" }; + String[] tput_set_cursor_line_7_column_0 = { "tput", "cup", "6", "0" }; String[] tput_change_background = { "tput", "smso" }; String[] tput_reset_background = { "tput", "rmso" }; SET_CURSOR_HOME = getCommandOutput(tput_cursor_home); CLEAR = getCommandOutput(tput_clear); CLEAR_LINE = getCommandOutput(tput_clear_line); - SET_CURSOR_LINE_6_COLUMN_0 = - getCommandOutput(tput_set_cursor_line_6_column_0); + SET_CURSOR_LINE_7_COLUMN_0 = + getCommandOutput(tput_set_cursor_line_7_column_0); CHANGE_BACKGROUND = getCommandOutput(tput_change_background); RESET_BACKGROUND = getCommandOutput(tput_reset_background); } @@ -712,6 +715,9 @@ public class TopCLI extends YarnCLI { queueMetrics.allocatedVCores += stats.getAllocatedVCores(); queueMetrics.pendingVCores += stats.getPendingVCores(); queueMetrics.reservedVCores += stats.getReservedVCores(); + queueMetrics.allocatedContainers += stats.getAllocatedContainers(); + queueMetrics.pendingContainers += stats.getPendingContainers(); + queueMetrics.reservedContainers += stats.getReservedContainers(); } } queueMetrics.availableMemoryGB = queueMetrics.availableMemoryGB / 1024; @@ -793,12 +799,18 @@ public class TopCLI extends YarnCLI { queueMetrics.availableVCores, queueMetrics.allocatedVCores, queueMetrics.pendingVCores, queueMetrics.reservedVCores), terminalWidth, true)); + + ret.append(CLEAR_LINE); + ret.append(limitLineLength(String.format( + "Queue(s) Containers: %d allocated, %d pending, %d reserved%n", + queueMetrics.allocatedContainers, queueMetrics.pendingContainers, + queueMetrics.reservedContainers), terminalWidth, true)); return ret.toString(); } String getPrintableAppInformation(List appsInfo) { StringBuilder ret = new StringBuilder(); - int limit = terminalHeight - 8; + int limit = terminalHeight - 9; List columns = new ArrayList<>(); for (int i = 0; i < limit; ++i) { ret.append(CLEAR_LINE); @@ -944,7 +956,7 @@ public class TopCLI extends YarnCLI { synchronized (lock) { printHeader(header); printApps(appsStr); - System.out.print(SET_CURSOR_LINE_6_COLUMN_0); + System.out.print(SET_CURSOR_LINE_7_COLUMN_0); System.out.print(CLEAR_LINE); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueStatisticsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueStatisticsPBImpl.java index 9506a5fa64e..ba394dc6adb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueStatisticsPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueStatisticsPBImpl.java @@ -254,4 +254,40 @@ public class QueueStatisticsPBImpl extends QueueStatistics { maybeInitBuilder(); builder.setReservedVCores(reservedVCores); } + + @Override + public long getPendingContainers() { + QueueStatisticsProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasPendingContainers()) ? p.getPendingContainers() : -1; + } + + @Override + public void setPendingContainers(long pendingContainers) { + maybeInitBuilder(); + builder.setPendingContainers(pendingContainers); + } + + @Override + public long getAllocatedContainers() { + QueueStatisticsProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasAllocatedContainers()) ? p.getAllocatedContainers() : -1; + } + + @Override + public void setAllocatedContainers(long allocatedContainers) { + maybeInitBuilder(); + builder.setAllocatedContainers(allocatedContainers); + } + + @Override + public long getReservedContainers() { + QueueStatisticsProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasReservedContainers()) ? p.getReservedContainers() : -1; + } + + @Override + public void setReservedContainers(long reservedContainers) { + maybeInitBuilder(); + builder.setReservedContainers(reservedContainers); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 134b9414648..792c25c332b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -323,6 +323,9 @@ public abstract class AbstractCSQueue implements CSQueue { stats.setAllocatedVCores(getMetrics().getAllocatedVirtualCores()); stats.setPendingVCores(getMetrics().getPendingVirtualCores()); stats.setReservedVCores(getMetrics().getReservedVirtualCores()); + stats.setPendingContainers(getMetrics().getPendingContainers()); + stats.setAllocatedContainers(getMetrics().getAllocatedContainers()); + stats.setReservedContainers(getMetrics().getReservedContainers()); return stats; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 81b28fd5ec9..d85687116f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -57,6 +57,9 @@ public class CapacitySchedulerQueueInfo { protected ResourceInfo resourcesUsed; private boolean hideReservationQueues = false; protected ArrayList nodeLabels = new ArrayList(); + protected long allocatedContainers; + protected long reservedContainers; + protected long pendingContainers; CapacitySchedulerQueueInfo() { }; @@ -81,6 +84,9 @@ public class CapacitySchedulerQueueInfo { absoluteUsedCapacity = cap(qCapacities.getAbsoluteUsedCapacity(nodeLabel), 0f, 1f) * 100; numApplications = q.getNumApplications(); + allocatedContainers = q.getMetrics().getAllocatedContainers(); + pendingContainers = q.getMetrics().getPendingContainers(); + reservedContainers = q.getMetrics().getReservedContainers(); queueName = q.getQueueName(); state = q.getState(); resourcesUsed = new ResourceInfo(queueResourceUsage.getUsed(nodeLabel)); @@ -124,6 +130,18 @@ public class CapacitySchedulerQueueInfo { return numApplications; } + public long getAllocatedContainers() { + return allocatedContainers; + } + + public long getReservedContainers() { + return reservedContainers; + } + + public long getPendingContainers() { + return pendingContainers; + } + public String getQueueName() { return this.queueName; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java index 7ba0988be80..ee37f184eda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java @@ -57,7 +57,11 @@ public class FairSchedulerQueueInfo { private ResourceInfo steadyFairResources; private ResourceInfo fairResources; private ResourceInfo clusterResources; - + + private long pendingContainers; + private long allocatedContainers; + private long reservedContainers; + private String queueName; private String schedulingPolicy; @@ -95,6 +99,10 @@ public class FairSchedulerQueueInfo { maxApps = allocConf.getQueueMaxApps(queueName); + pendingContainers = queue.getMetrics().getPendingContainers(); + allocatedContainers = queue.getMetrics().getAllocatedContainers(); + reservedContainers = queue.getMetrics().getReservedContainers(); + if (allocConf.isReservable(queueName) && !allocConf.getShowReservationAsQueues(queueName)) { return; @@ -103,6 +111,18 @@ public class FairSchedulerQueueInfo { childQueues = getChildQueues(queue, scheduler); } + public long getPendingContainers() { + return pendingContainers; + } + + public long getAllocatedContainers() { + return allocatedContainers; + } + + public long getReservedContainers() { + return reservedContainers; + } + protected FairSchedulerQueueInfoList getChildQueues(FSQueue queue, FairScheduler scheduler) { // Return null to omit 'childQueues' field from the return value of @@ -172,7 +192,7 @@ public class FairSchedulerQueueInfo { public ResourceInfo getUsedResources() { return usedResources; } - + /** * Returns the queue's min share in as a fraction of the entire * cluster capacity. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 456595de234..dad720e74c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -349,10 +349,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 13; + int numExpectedElements = 16; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 25; + numExpectedElements = 28; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length());