From 16e6030e25a6607b9886d01f9174984b1e71fb38 Mon Sep 17 00:00:00 2001 From: Tamas Domok Date: Fri, 27 Aug 2021 23:09:54 +0200 Subject: [PATCH] YARN-10891. Extend QueueInfo with max-parallel-apps in CS. (#3314) Co-authored-by: Tamas Domok --- .../hadoop/yarn/api/records/QueueInfo.java | 24 +++++++++++++++---- .../src/main/proto/yarn_protos.proto | 1 + .../hadoop/yarn/client/cli/QueueCLI.java | 2 ++ .../yarn/client/ProtocolHATestBase.java | 2 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 8 ++++--- .../api/records/impl/pb/QueueInfoPBImpl.java | 12 ++++++++++ .../hadoop/yarn/api/TestPBImplRecords.java | 2 +- .../scheduler/capacity/AbstractCSQueue.java | 2 ++ .../scheduler/capacity/CSQueue.java | 6 +++++ .../dao/CapacitySchedulerQueueInfo.java | 6 +++++ .../TestConfigurationMutationACLPolicies.java | 2 +- .../TestSchedulerApplicationAttempt.java | 2 +- .../scheduler/capacity/TestLeafQueue.java | 2 +- .../TestRMWebServicesCapacitySched.java | 14 ++++++++--- .../src/site/markdown/ResourceManagerRest.md | 19 +++++++++++++++ 15 files changed, 88 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java index eb703cdb2ff..707554cd1c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java @@ -60,7 +60,8 @@ public abstract class QueueInfo { List childQueues, List applications, QueueState queueState, Set accessibleNodeLabels, String defaultNodeLabelExpression, QueueStatistics queueStatistics, - boolean preemptionDisabled, float weight) { + boolean preemptionDisabled, float weight, + int maxParallelApps) { QueueInfo queueInfo = Records.newRecord(QueueInfo.class); queueInfo.setQueueName(queueName); queueInfo.setQueuePath(queuePath); @@ -75,6 +76,7 @@ public abstract class QueueInfo { queueInfo.setQueueStatistics(queueStatistics); queueInfo.setPreemptionDisabled(preemptionDisabled); queueInfo.setWeight(weight); + queueInfo.setMaxParallelApps(maxParallelApps); return queueInfo; } @@ -86,14 +88,14 @@ public abstract class QueueInfo { List childQueues, List applications, QueueState queueState, Set accessibleNodeLabels, String defaultNodeLabelExpression, QueueStatistics queueStatistics, - boolean preemptionDisabled, float weight, + boolean preemptionDisabled, float weight, int maxParallelApps, Map queueConfigurations) { QueueInfo queueInfo = QueueInfo.newInstance(queueName, queuePath, capacity, maximumCapacity, currentCapacity, childQueues, applications, queueState, accessibleNodeLabels, defaultNodeLabelExpression, queueStatistics, - preemptionDisabled, weight); + preemptionDisabled, weight, maxParallelApps); queueInfo.setQueueConfigurations(queueConfigurations); return queueInfo; } @@ -106,7 +108,7 @@ public abstract class QueueInfo { List childQueues, List applications, QueueState queueState, Set accessibleNodeLabels, String defaultNodeLabelExpression, QueueStatistics queueStatistics, - boolean preemptionDisabled, float weight, + boolean preemptionDisabled, float weight, int maxParallelApps, Map queueConfigurations, boolean intraQueuePreemptionDisabled) { QueueInfo queueInfo = QueueInfo.newInstance(queueName, queuePath, capacity, @@ -114,7 +116,7 @@ public abstract class QueueInfo { childQueues, applications, queueState, accessibleNodeLabels, defaultNodeLabelExpression, queueStatistics, - preemptionDisabled, weight, queueConfigurations); + preemptionDisabled, weight, maxParallelApps, queueConfigurations); queueInfo.setIntraQueuePreemptionDisabled(intraQueuePreemptionDisabled); return queueInfo; } @@ -166,6 +168,18 @@ public abstract class QueueInfo { @Private @Unstable public abstract void setWeight(float weight); + + /** + * Get the configured max parallel apps of the queue. + * @return configured max parallel apps of the queue + */ + @Public + @Stable + public abstract int getMaxParallelApps(); + + @Private + @Unstable + public abstract void setMaxParallelApps(int maxParallelApps); /** * Get the maximum capacity of the queue. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 558e724f8ac..95827ce56dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -617,6 +617,7 @@ message QueueInfoProto { optional bool intraQueuePreemptionDisabled = 13; optional float weight = 14; optional string queuePath = 15; + optional int32 maxParallelApps = 16; } message QueueConfigurationsProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java index c3566437b4a..aa4bb37b56c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java @@ -139,6 +139,8 @@ public class QueueCLI extends YarnCLI { writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%"); writer.print("\tWeight : "); writer.println(df.format(queueInfo.getWeight())); + writer.print("\tMaximum Parallel Apps : "); + writer.println(queueInfo.getMaxParallelApps()); writer.print("\tDefault Node Label expression : "); String nodeLabelExpression = queueInfo.getDefaultNodeLabelExpression(); nodeLabelExpression = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index 3972ca67ad3..53005aa4b75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -670,7 +670,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes { public QueueInfo createFakeQueueInfo() { return QueueInfo.newInstance("root", "root", 100f, 100f, 50f, null, createFakeAppReports(), QueueState.RUNNING, null, - null, null, false, -1.0f, + null, null, false, -1.0f, 10, null, false); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index dc9137e8fac..621540677cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -1722,8 +1722,8 @@ public class TestYarnCLI { newInstance("queueA", "root.queueA", 0.4f, 0.8f, 0.5f, null, null, QueueState.RUNNING, nodeLabels, - "GPU", null, false, -1.0f, null, - false); + "GPU", null, false, -1.0f, 10, + null, false); when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo); int result = cli.run(new String[] { "-status", "queueA" }); assertEquals(0, result); @@ -1738,6 +1738,7 @@ public class TestYarnCLI { pw.println("\tCurrent Capacity : " + "50.00%"); pw.println("\tMaximum Capacity : " + "80.00%"); pw.println("\tWeight : " + "-1.00"); + pw.println("\tMaximum Parallel Apps : " + "10"); pw.println("\tDefault Node Label expression : " + "GPU"); pw.println("\tAccessible Node Labels : " + "JDK_7,GPU"); pw.println("\tPreemption : " + "enabled"); @@ -1895,7 +1896,7 @@ public class TestYarnCLI { newInstance("queueA", "root.queueA", 0.4f, 0.8f, 0.5f, null, null, QueueState.RUNNING, null, null, null, - true, -1.0f, null, true); + true, -1.0f, 10, null, true); when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo); int result = cli.run(new String[] { "-status", "queueA" }); assertEquals(0, result); @@ -1910,6 +1911,7 @@ public class TestYarnCLI { pw.println("\tCurrent Capacity : " + "50.00%"); pw.println("\tMaximum Capacity : " + "80.00%"); pw.println("\tWeight : " + "-1.00"); + pw.println("\tMaximum Parallel Apps : " + "10"); pw.println("\tDefault Node Label expression : " + NodeLabel.DEFAULT_NODE_LABEL_PARTITION); pw.println("\tAccessible Node Labels : "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java index c69fddbc4c9..5643ff30d4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java @@ -142,6 +142,18 @@ public class QueueInfoPBImpl extends QueueInfo { builder.setWeight(weight); } + @Override + public int getMaxParallelApps() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasMaxParallelApps()) ? p.getMaxParallelApps() : -1; + } + + @Override + public void setMaxParallelApps(int weight) { + maybeInitBuilder(); + builder.setMaxParallelApps(weight); + } + @Override public void setChildQueues(List childQueues) { if (childQueues == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 96624d66dee..ad3725278a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -435,7 +435,7 @@ public class TestPBImplRecords extends BasePBImplRecordsTest { typeValueCache.put(QueueInfo.class, QueueInfo. newInstance("root", "root", 1.0f, 1.0f, 0.1f, null, null, QueueState.RUNNING, ImmutableSet.of("x", "y"), - "x && y", null, false, -1.0f, null, false)); + "x && y", null, false, -1.0f, 10, null, false)); generateByNewInstance(QueueStatistics.class); generateByNewInstance(QueueUserACLInfo.class); generateByNewInstance(YarnClusterMetrics.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 7ddc7655707..1f5820d807f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -774,6 +774,7 @@ public abstract class AbstractCSQueue implements CSQueue { getIntraQueuePreemptionDisabled()); queueInfo.setQueueConfigurations(getQueueConfigurations()); queueInfo.setWeight(queueCapacities.getWeight()); + queueInfo.setMaxParallelApps(maxParallelApps); return queueInfo; } @@ -1494,6 +1495,7 @@ public abstract class AbstractCSQueue implements CSQueue { this.maxParallelApps = maxParallelApps; } + @Override public int getMaxParallelApps() { return maxParallelApps; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 47bdeba45df..6bf5feaeb4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -151,6 +151,12 @@ public interface CSQueue extends SchedulerQueue { * @return current run-state */ public QueueState getState(); + + /** + * Get the max-parallel-applications property of the queue + * @return max-parallel-applications + */ + public int getMaxParallelApps(); /** * Get child queues diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 452fa503434..4cbb24d9d86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -65,6 +65,7 @@ public class CapacitySchedulerQueueInfo { protected float weight; protected float normalizedWeight; protected int numApplications; + protected int maxParallelApps; protected String queueName; protected boolean isAbsoluteResource; protected QueueState state; @@ -120,6 +121,7 @@ public class CapacitySchedulerQueueInfo { weight = q.getQueueCapacities().getWeight(); normalizedWeight = q.getQueueCapacities().getNormalizedWeight(); numApplications = q.getNumApplications(); + maxParallelApps = q.getMaxParallelApps(); allocatedContainers = q.getMetrics().getAllocatedContainers(); pendingContainers = q.getMetrics().getPendingContainers(); reservedContainers = q.getMetrics().getReservedContainers(); @@ -352,6 +354,10 @@ public class CapacitySchedulerQueueInfo { return normalizedWeight; } + public int getMaxParallelApps() { + return maxParallelApps; + } + public String getDefaultNodeLabelExpression() { return defaultNodeLabelExpression; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java index 1304a88d02d..f59c4376162 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java @@ -70,7 +70,7 @@ public class TestConfigurationMutationACLPolicies { QueueInfo queueInfo = QueueInfo. newInstance(queueName, queuePath, 0, 0, 0, null, null, - null, null, null, null, false, -1.0f, null, false); + null, null, null, null, false, -1.0f, 10, null, false); when(confScheduler.getQueueInfo(eq(queueName), anyBoolean(), anyBoolean())) .thenReturn(queueInfo); Queue queue = mock(Queue.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java index 65eed96c925..eae0bb3342b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java @@ -219,7 +219,7 @@ public class TestSchedulerApplicationAttempt { QueueMetrics metrics = QueueMetrics.forQueue(name, parent, false, conf); QueueInfo queueInfo = QueueInfo.newInstance(name, "root." + name, capacity, 1.0f, 0, null, - null, QueueState.RUNNING, null, "", null, false, -1.0f, null, false); + null, QueueState.RUNNING, null, "", null, false, -1.0f, 10, null, false); ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics); Queue queue = mock(Queue.class); when(queue.getMetrics()).thenReturn(metrics); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 7a310138da8..5bb12a44341 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -5101,7 +5101,7 @@ public class TestLeafQueue { CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf()); QueueInfo queueInfo = QueueInfo. newInstance(name, path, capacity, 1.0f, 0, null, - null, QueueState.RUNNING, null, "", null, false, -1.0f, null, false); + null, QueueState.RUNNING, null, "", null, false, -1.0f, 10, null, false); ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics); AbstractCSQueue queue = mock(AbstractCSQueue.class); when(queue.getMetrics()).thenReturn(metrics); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index b472e099f4b..d4e7ca63f22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAX_PARALLEL_APPLICATIONS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertFalse; @@ -83,6 +84,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { float absoluteMaxCapacity; float absoluteUsedCapacity; int numApplications; + int maxParallelApps; String queueName; private String queuePath; String state; @@ -140,6 +142,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { final String A = CapacitySchedulerConfiguration.ROOT + ".a"; config.setCapacity(A, 10.5f); config.setMaximumCapacity(A, 50); + config.setInt(CapacitySchedulerConfiguration.getQueuePrefix(A) + MAX_PARALLEL_APPLICATIONS, 42); final String B = CapacitySchedulerConfiguration.ROOT + ".b"; config.setCapacity(B, 89.5f); @@ -311,6 +314,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { WebServicesTestUtils.getXmlFloat(qElem, "absoluteUsedCapacity"); qi.numApplications = WebServicesTestUtils.getXmlInt(qElem, "numApplications"); + qi.maxParallelApps = + WebServicesTestUtils.getXmlInt(qElem, "maxParallelApps"); qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName"); qi.queuePath = WebServicesTestUtils.getXmlString(qElem, "queuePath"); qi.state = WebServicesTestUtils.getXmlString(qElem, "state"); @@ -424,10 +429,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 37; + int numExpectedElements = 38; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 55; + numExpectedElements = 56; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); @@ -440,6 +445,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { qi.absoluteMaxCapacity = (float) info.getDouble("absoluteMaxCapacity"); qi.absoluteUsedCapacity = (float) info.getDouble("absoluteUsedCapacity"); qi.numApplications = info.getInt("numApplications"); + qi.maxParallelApps = info.getInt("maxParallelApps"); qi.queueName = info.getString("queueName"); qi.queuePath = info.getString("queuePath"); qi.state = info.getString("state"); @@ -526,7 +532,9 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { + " queue is not configured in Absolute resource", info.isAbsoluteResource); } - + assertEquals("maxParallelApps doesn't match " + q, + (q.equals("root.a") ? 42 : Integer.MAX_VALUE), + info.maxParallelApps); } private void verifyLeafQueueGeneric(String q, LeafQueueInfo info) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 375e69ac8cd..e0f29a2679d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -324,6 +324,7 @@ The capacity scheduler supports hierarchical queues. This one request will print | absoluteMaxCapacity | float | Absolute maximum capacity percentage this queue can use of the entire cluster | | absoluteUsedCapacity | float | Absolute used capacity percentage this queue is using of the entire cluster | | numApplications | int | The number of applications currently in the queue | +| maxParallelApps | int | Maximum number of applications that can run at the same time | | usedResources | string | A string describing the current resources used by the queue | | queueName | string | The name of the queue | | state | string of QueueState | The state of the queue | @@ -423,6 +424,7 @@ Response Body: "capacity": 10.5, "maxCapacity": 50.0, "numApplications": 0, + "maxParallelApps": 2147483647, "queueName": "a", "queues": { "queue": [ @@ -433,6 +435,7 @@ Response Body: "capacity": 30.000002, "maxCapacity": 50.0, "numApplications": 0, + "maxParallelApps": 2147483647, "queueName": "a1", "queues": { "queue": [ @@ -448,6 +451,7 @@ Response Body: "maxCapacity": 100.0, "numActiveApplications": 0, "numApplications": 0, + "maxParallelApps": 2147483647, "numContainers": 0, "numPendingApplications": 0, "queueName": "a1a", @@ -475,6 +479,7 @@ Response Body: "maxCapacity": 100.0, "numActiveApplications": 0, "numApplications": 0, + "maxParallelApps": 2147483647, "numContainers": 0, "numPendingApplications": 0, "queueName": "a1b", @@ -512,6 +517,7 @@ Response Body: "maxCapacity": 100.0, "numActiveApplications": 0, "numApplications": 0, + "maxParallelApps": 2147483647, "numContainers": 0, "numPendingApplications": 0, "queueName": "a2", @@ -544,6 +550,7 @@ Response Body: "capacity": 89.5, "maxCapacity": 100.0, "numApplications": 2, + "maxParallelApps": 2147483647, "queueName": "b", "queues": { "queue": [ @@ -559,6 +566,7 @@ Response Body: "maxCapacity": 100.0, "numActiveApplications": 1, "numApplications": 2, + "maxParallelApps": 2147483647, "numContainers": 0, "numPendingApplications": 1, "queueName": "b1", @@ -607,6 +615,7 @@ Response Body: "maxCapacity": 100.0, "numActiveApplications": 0, "numApplications": 0, + "maxParallelApps": 2147483647, "numContainers": 0, "numPendingApplications": 0, "queueName": "b2", @@ -634,6 +643,7 @@ Response Body: "maxCapacity": 100.0, "numActiveApplications": 0, "numApplications": 0, + "maxParallelApps": 2147483647, "numContainers": 0, "numPendingApplications": 0, "queueName": "b3", @@ -756,6 +766,7 @@ Response Body: 50.0 0.0 0 + 2147483647 <memory:0, vCores:0> a RUNNING @@ -768,6 +779,7 @@ Response Body: 25.0 0.0 0 + 2147483647 <memory:0, vCores:0> a1 RUNNING @@ -780,6 +792,7 @@ Response Body: 25.0 0.0 0 + 2147483647 <memory:0, vCores:0> a1a RUNNING @@ -806,6 +819,7 @@ Response Body: 25.0 0.0 0 + 2147483647 <memory:0, vCores:0> a1b RUNNING @@ -838,6 +852,7 @@ Response Body: 50.0 0.0 0 + 2147483647 <memory:0, vCores:0> a2 RUNNING @@ -870,6 +885,7 @@ Response Body: 100.0 0.0 2 + 2147483647 <memory:0, vCores:0> b RUNNING @@ -882,6 +898,7 @@ Response Body: 100.0 0.0 2 + 2147483647 <memory:0, vCores:0> b1 RUNNING @@ -927,6 +944,7 @@ Response Body: 100.0 0.0 0 + 2147483647 <memory:0, vCores:0> b2 RUNNING @@ -953,6 +971,7 @@ Response Body: 100.0 0.0 0 + 2147483647 <memory:0, vCores:0> b3 RUNNING