From 79ca399a30d48b15b6b8d9eae101f016b69ad7a9 Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Thu, 5 Sep 2019 14:01:42 -0700 Subject: [PATCH] YARN-9810. Add queue capacity/maxcapacity percentage metrics. Contributed by Shubham Gupta (cherry picked from commit 0ccf4b0fe16a8c879a560f2a612a3185eb2df72b) --- .../scheduler/capacity/CSQueueMetrics.java | 39 +++++++++++++++++++ .../scheduler/capacity/CSQueueUtils.java | 6 +++ .../capacity/TestCapacityScheduler.java | 18 +++++++++ 3 files changed, 63 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java index 83826650414..c50a1b145d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java @@ -55,6 +55,14 @@ public class CSQueueMetrics extends QueueMetrics { MutableGaugeLong maxCapacityMB; @Metric("Maximum CPU in virtual cores") MutableGaugeInt maxCapacityVCores; + @Metric("Guaranteed capacity in percentage relative to parent") + private MutableGaugeFloat guaranteedCapacity; + @Metric("Guaranteed capacity in percentage relative to total partition") + private MutableGaugeFloat guaranteedAbsoluteCapacity; + @Metric("Maximum capacity in percentage relative to parent") + private MutableGaugeFloat maxCapacity; + @Metric("Maximum capacity in percentage relative to total partition") + private MutableGaugeFloat maxAbsoluteCapacity; CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { @@ -204,4 +212,35 @@ public class CSQueueMetrics extends QueueMetrics { return metrics; } + public float getGuaranteedCapacity() { + return guaranteedCapacity.value(); + } + + public float getGuaranteedAbsoluteCapacity() { + return guaranteedAbsoluteCapacity.value(); + } + + public void setGuaranteedCapacities(String partition, float capacity, + float absoluteCapacity) { + if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) { + guaranteedCapacity.set(capacity); + guaranteedAbsoluteCapacity.set(absoluteCapacity); + } + } + + public float getMaxCapacity() { + return maxCapacity.value(); + } + + public float getMaxAbsoluteCapacity() { + return maxAbsoluteCapacity.value(); + } + + public void setMaxCapacities(String partition, float capacity, + float absoluteCapacity) { + if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) { + maxCapacity.set(capacity); + maxAbsoluteCapacity.set(absoluteCapacity); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 5c5f30f329c..715f80bc366 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -336,5 +336,11 @@ public class CSQueueUtils { queue.getMetrics().setMaxCapacityResources(partition, rc.multiplyAndNormalizeDown( partitionResource, queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition), queue.getMinimumAllocation())); + queue.getMetrics().setGuaranteedCapacities(partition, + queue.getQueueCapacities().getCapacity(partition), + queue.getQueueCapacities().getAbsoluteCapacity(partition)); + queue.getMetrics().setMaxCapacities(partition, + queue.getQueueCapacities().getMaximumCapacity(partition), + queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 56882a70f96..51900b576b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -184,6 +184,8 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { private ResourceManager resourceManager = null; private RMContext mockContext; + private static final double DELTA = 0.000001; + @Before public void setUp() throws Exception { resourceManager = new ResourceManager() { @@ -5196,6 +5198,22 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { assertEquals(35840, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB()); assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB()); assertEquals(51200, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB()); + assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a") + .getMetrics()).getGuaranteedCapacity(), DELTA); + assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a") + .getMetrics()).getGuaranteedAbsoluteCapacity(), DELTA); + assertEquals(B1_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("b1") + .getMetrics()).getGuaranteedCapacity(), DELTA); + assertEquals((B_CAPACITY / 100) * (B1_CAPACITY / 100), ((CSQueueMetrics)cs + .getQueue("b1").getMetrics()).getGuaranteedAbsoluteCapacity(), DELTA); + assertEquals(1, ((CSQueueMetrics)cs.getQueue("a").getMetrics()) + .getMaxCapacity(), DELTA); + assertEquals(1, ((CSQueueMetrics)cs.getQueue("a").getMetrics()) + .getMaxAbsoluteCapacity(), DELTA); + assertEquals(1, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()) + .getMaxCapacity(), DELTA); + assertEquals(1, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()) + .getMaxAbsoluteCapacity(), DELTA); // Add child queue to a, and reinitialize. Metrics should be updated conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", new String[] {"a1", "a2", "a3"} );