From e19c00925f38346dc1291cdced108ecc0206fb74 Mon Sep 17 00:00:00 2001 From: Peter Bacsko Date: Fri, 5 Mar 2021 13:18:06 +0100 Subject: [PATCH] YARN-10639. Queueinfo related capacity, should adjusted to weight mode. Contributed by Qi Zhu. --- .../hadoop/yarn/api/records/QueueInfo.java | 23 +++++++--- .../src/main/proto/yarn_protos.proto | 1 + .../hadoop/yarn/client/cli/QueueCLI.java | 2 + .../yarn/client/ProtocolHATestBase.java | 5 ++- .../hadoop/yarn/client/cli/TestYarnCLI.java | 8 +++- .../api/records/impl/pb/QueueInfoPBImpl.java | 12 +++++ .../hadoop/yarn/api/TestPBImplRecords.java | 2 +- .../scheduler/capacity/AbstractCSQueue.java | 1 + .../TestConfigurationMutationACLPolicies.java | 2 +- .../TestSchedulerApplicationAttempt.java | 2 +- .../TestCapacitySchedulerWeightMode.java | 45 ++++++++++++++++--- .../scheduler/capacity/TestLeafQueue.java | 2 +- 12 files changed, 87 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java index 57ea9bfe44c..803adad002b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java @@ -59,7 +59,7 @@ public abstract class QueueInfo { List childQueues, List applications, QueueState queueState, Set accessibleNodeLabels, String defaultNodeLabelExpression, QueueStatistics queueStatistics, - boolean preemptionDisabled) { + boolean preemptionDisabled, float weight) { QueueInfo queueInfo = Records.newRecord(QueueInfo.class); queueInfo.setQueueName(queueName); queueInfo.setCapacity(capacity); @@ -72,6 +72,7 @@ public abstract class QueueInfo { queueInfo.setDefaultNodeLabelExpression(defaultNodeLabelExpression); queueInfo.setQueueStatistics(queueStatistics); queueInfo.setPreemptionDisabled(preemptionDisabled); + queueInfo.setWeight(weight); return queueInfo; } @@ -82,14 +83,14 @@ public abstract class QueueInfo { List childQueues, List applications, QueueState queueState, Set accessibleNodeLabels, String defaultNodeLabelExpression, QueueStatistics queueStatistics, - boolean preemptionDisabled, + boolean preemptionDisabled, float weight, Map queueConfigurations) { QueueInfo queueInfo = QueueInfo.newInstance(queueName, capacity, maximumCapacity, currentCapacity, childQueues, applications, queueState, accessibleNodeLabels, defaultNodeLabelExpression, queueStatistics, - preemptionDisabled); + preemptionDisabled, weight); queueInfo.setQueueConfigurations(queueConfigurations); return queueInfo; } @@ -101,7 +102,7 @@ public abstract class QueueInfo { List childQueues, List applications, QueueState queueState, Set accessibleNodeLabels, String defaultNodeLabelExpression, QueueStatistics queueStatistics, - boolean preemptionDisabled, + boolean preemptionDisabled, float weight, Map queueConfigurations, boolean intraQueuePreemptionDisabled) { QueueInfo queueInfo = QueueInfo.newInstance(queueName, capacity, @@ -109,7 +110,7 @@ public abstract class QueueInfo { childQueues, applications, queueState, accessibleNodeLabels, defaultNodeLabelExpression, queueStatistics, - preemptionDisabled, queueConfigurations); + preemptionDisabled, weight, queueConfigurations); queueInfo.setIntraQueuePreemptionDisabled(intraQueuePreemptionDisabled); return queueInfo; } @@ -137,6 +138,18 @@ public abstract class QueueInfo { @Private @Unstable public abstract void setCapacity(float capacity); + + /** + * Get the configured weight of the queue. + * @return configured weight of the queue + */ + @Public + @Stable + public abstract float getWeight(); + + @Private + @Unstable + public abstract void setWeight(float weight); /** * Get the maximum capacity of the queue. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index d7ca2a563ea..467f26aa111 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -609,6 +609,7 @@ message QueueInfoProto { optional bool preemptionDisabled = 11; repeated QueueConfigurationsMapProto queueConfigurationsMap = 12; optional bool intraQueuePreemptionDisabled = 13; + optional float weight = 14; } message QueueConfigurationsProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java index 550b929d37e..98f714fed81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java @@ -135,6 +135,8 @@ public class QueueCLI extends YarnCLI { writer.println(df.format(queueInfo.getCurrentCapacity() * 100) + "%"); writer.print("\tMaximum Capacity : "); writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%"); + writer.print("\tWeight : "); + writer.println(df.format(queueInfo.getWeight())); writer.print("\tDefault Node Label expression : "); String nodeLabelExpression = queueInfo.getDefaultNodeLabelExpression(); nodeLabelExpression = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index c923446a008..8c89cf88325 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -669,8 +669,9 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes { public QueueInfo createFakeQueueInfo() { return QueueInfo.newInstance("root", 100f, 100f, 50f, null, - createFakeAppReports(), QueueState.RUNNING, null, null, null, false, - null, false); + createFakeAppReports(), QueueState.RUNNING, null, + null, null, false, -1.0f, + null, false); } public List createFakeQueueUserACLInfoList() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index b5606018a8b..90e2a75c2e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -1719,7 +1719,8 @@ public class TestYarnCLI { nodeLabels.add("GPU"); nodeLabels.add("JDK_7"); QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f, - null, null, QueueState.RUNNING, nodeLabels, "GPU", null, false, null, + null, null, QueueState.RUNNING, nodeLabels, + "GPU", null, false, -1.0f, null, false); when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo); int result = cli.run(new String[] { "-status", "queueA" }); @@ -1733,6 +1734,7 @@ public class TestYarnCLI { pw.println("\tCapacity : " + "40.00%"); pw.println("\tCurrent Capacity : " + "50.00%"); pw.println("\tMaximum Capacity : " + "80.00%"); + pw.println("\tWeight : " + "-1.00"); pw.println("\tDefault Node Label expression : " + "GPU"); pw.println("\tAccessible Node Labels : " + "JDK_7,GPU"); pw.println("\tPreemption : " + "enabled"); @@ -1887,7 +1889,8 @@ public class TestYarnCLI { public void testGetQueueInfoWithEmptyNodeLabel() throws Exception { QueueCLI cli = createAndGetQueueCLI(); QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f, - null, null, QueueState.RUNNING, null, null, null, true, null, true); + null, null, QueueState.RUNNING, null, null, null, + true, -1.0f, null, true); when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo); int result = cli.run(new String[] { "-status", "queueA" }); assertEquals(0, result); @@ -1900,6 +1903,7 @@ public class TestYarnCLI { pw.println("\tCapacity : " + "40.00%"); pw.println("\tCurrent Capacity : " + "50.00%"); pw.println("\tMaximum Capacity : " + "80.00%"); + pw.println("\tWeight : " + "-1.00"); pw.println("\tDefault Node Label expression : " + NodeLabel.DEFAULT_NODE_LABEL_PARTITION); pw.println("\tAccessible Node Labels : "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java index db9ece4dce1..cb3757bd992 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java @@ -124,6 +124,18 @@ public class QueueInfoPBImpl extends QueueInfo { builder.setCapacity(capacity); } + @Override + public float getWeight() { + QueueInfoProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasWeight()) ? p.getWeight() : -1; + } + + @Override + public void setWeight(float weight) { + maybeInitBuilder(); + builder.setWeight(weight); + } + @Override public void setChildQueues(List childQueues) { if (childQueues == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 980ec0da9f1..7792cad2a84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -434,7 +434,7 @@ public class TestPBImplRecords extends BasePBImplRecordsTest { // it is recursive(has sub queues) typeValueCache.put(QueueInfo.class, QueueInfo.newInstance("root", 1.0f, 1.0f, 0.1f, null, null, QueueState.RUNNING, ImmutableSet.of("x", "y"), - "x && y", null, false, null, false)); + "x && y", null, false, -1.0f, null, false)); generateByNewInstance(QueueStatistics.class); generateByNewInstance(QueueUserACLInfo.class); generateByNewInstance(YarnClusterMetrics.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index e5380fa9528..7aa8e012df6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -727,6 +727,7 @@ public abstract class AbstractCSQueue implements CSQueue { queueInfo.setIntraQueuePreemptionDisabled( getIntraQueuePreemptionDisabled()); queueInfo.setQueueConfigurations(getQueueConfigurations()); + queueInfo.setWeight(queueCapacities.getWeight()); return queueInfo; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java index 8741432cc0a..90e12ac416b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java @@ -67,7 +67,7 @@ public class TestConfigurationMutationACLPolicies { private void mockQueue(String queueName, MutableConfScheduler scheduler) throws IOException { QueueInfo queueInfo = QueueInfo.newInstance(queueName, 0, 0, 0, null, null, - null, null, null, null, false, null, false); + null, null, null, null, false, -1.0f, null, false); when(scheduler.getQueueInfo(eq(queueName), anyBoolean(), anyBoolean())) .thenReturn(queueInfo); Queue queue = mock(Queue.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java index b1080f7d5fe..93d52bdb790 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java @@ -217,7 +217,7 @@ public class TestSchedulerApplicationAttempt { private Queue createQueue(String name, Queue parent, float capacity) { QueueMetrics metrics = QueueMetrics.forQueue(name, parent, false, conf); QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null, - null, QueueState.RUNNING, null, "", null, false, null, false); + null, QueueState.RUNNING, null, "", null, false, -1.0f, null, false); ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics); Queue queue = mock(Queue.class); when(queue.getMetrics()).thenReturn(metrics); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java index 1742ae45f35..77c8e053e3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java @@ -257,7 +257,7 @@ public class TestCapacitySchedulerWeightMode { */ @Test(timeout = 300000) public void testContainerAllocateWithComplexLabelsWeightOnly() throws Exception { - internalTestContainerAlloationWithNodeLabel( + internalTestContainerAllocationWithNodeLabel( getCSConfWithQueueLabelsWeightOnly(conf)); } @@ -270,7 +270,7 @@ public class TestCapacitySchedulerWeightMode { */ @Test(timeout = 300000) public void testContainerAllocateWithComplexLabelsWeightAndPercentMixed1() throws Exception { - internalTestContainerAlloationWithNodeLabel( + internalTestContainerAllocationWithNodeLabel( getCSConfWithLabelsParentUseWeightChildUsePct(conf)); } @@ -283,7 +283,7 @@ public class TestCapacitySchedulerWeightMode { */ @Test(timeout = 300000) public void testContainerAllocateWithComplexLabelsWeightAndPercentMixed2() throws Exception { - internalTestContainerAlloationWithNodeLabel( + internalTestContainerAllocationWithNodeLabel( getCSConfWithLabelsParentUsePctChildUseWeight(conf)); } @@ -338,8 +338,43 @@ public class TestCapacitySchedulerWeightMode { } } - private void internalTestContainerAlloationWithNodeLabel(Configuration csConf) - throws Exception { + @Test + public void testQueueInfoWeight() throws Exception { + MockRM rm = new MockRM(conf); + rm.init(conf); + rm.start(); + + CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration( + conf); + csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] {"a", "b", "default"}); + csConf.setNonLabeledQueueWeight("root.a", 1); + csConf.setNonLabeledQueueWeight("root.b", 2); + csConf.setNonLabeledQueueWeight("root.default", 3); + + // Check queue info capacity + CapacityScheduler cs = + (CapacityScheduler)rm.getRMContext().getScheduler(); + cs.reinitialize(csConf, rm.getRMContext()); + + LeafQueue a = (LeafQueue) + cs.getQueue("root.a"); + Assert.assertNotNull(a); + Assert.assertEquals(a.getQueueCapacities().getWeight(), + a.getQueueInfo(false, + false).getWeight(), 1e-6); + + LeafQueue b = (LeafQueue) + cs.getQueue("root.b"); + Assert.assertNotNull(b); + Assert.assertEquals(b.getQueueCapacities().getWeight(), + b.getQueueInfo(false, + false).getWeight(), 1e-6); + rm.close(); + } + + private void internalTestContainerAllocationWithNodeLabel( + Configuration csConf) throws Exception { /* * Queue structure: * root (*) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 889da07adda..1a45908ee91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -4905,7 +4905,7 @@ public class TestLeafQueue { float absCap, Resource res) { CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf()); QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null, - null, QueueState.RUNNING, null, "", null, false, null, false); + null, QueueState.RUNNING, null, "", null, false, -1.0f, null, false); ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics); AbstractCSQueue queue = mock(AbstractCSQueue.class); when(queue.getMetrics()).thenReturn(metrics);