From c19326c051103d0985c0f949654487cfa684335a Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Fri, 5 Feb 2021 17:34:22 +0100 Subject: [PATCH] YARN-10610. Add queuePath to RESTful API for CapacityScheduler consistent with FairScheduler queuePath. Contributed by Qi Zhu --- .../webapp/dao/CapacitySchedulerInfo.java | 6 +++++ .../dao/CapacitySchedulerQueueInfo.java | 2 -- .../TestRMWebServicesCapacitySched.java | 26 ++++++++++++++----- .../TestRMWebServicesForCSWithPartitions.java | 2 +- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 9143817f695..fc18d4d6e44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -49,6 +49,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected float weight; protected float normalizedWeight; protected String queueName; + private String queuePath; protected CapacitySchedulerQueueInfoList queues; protected QueueCapacitiesInfo capacities; protected CapacitySchedulerHealthInfo health; @@ -69,6 +70,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { this.queueName = parent.getQueueName(); + this.queuePath = parent.getQueuePath(); this.usedCapacity = parent.getUsedCapacity() * 100; this.capacity = parent.getCapacity() * 100; float max = parent.getMaximumCapacity(); @@ -134,6 +136,10 @@ public class CapacitySchedulerInfo extends SchedulerInfo { return this.queueName; } + public String getQueuePath() { + return this.queuePath; + } + public ResourceInfo getMaximumAllocation() { return maximumAllocation; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 91265662f87..3865cce909f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -57,9 +57,7 @@ public class CapacitySchedulerQueueInfo { @XmlTransient static final float EPSILON = 1e-8f; - @XmlTransient protected String queuePath; - protected float capacity; protected float usedCapacity; protected float maxCapacity; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 9ad75d63c98..a0a088497b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -84,9 +84,14 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { float absoluteUsedCapacity; int numApplications; String queueName; + private String queuePath; String state; boolean isAbsoluteResource; boolean autoCreateChildQueueEnabled; + + public String getQueuePath() { + return queuePath; + } } private class LeafQueueInfo extends QueueInfo { @@ -261,7 +266,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { WebServicesTestUtils.getXmlFloat(element, "usedCapacity"), WebServicesTestUtils.getXmlFloat(element, "capacity"), WebServicesTestUtils.getXmlFloat(element, "maxCapacity"), - WebServicesTestUtils.getXmlString(element, "queueName")); + WebServicesTestUtils.getXmlString(element, "queueName"), + WebServicesTestUtils.getXmlString(element, "queuePath")); NodeList children = element.getChildNodes(); for (int j = 0; j < children.getLength(); j++) { @@ -306,6 +312,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { qi.numApplications = WebServicesTestUtils.getXmlInt(qElem, "numApplications"); qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName"); + qi.queuePath = WebServicesTestUtils.getXmlString(qElem, "queuePath"); qi.state = WebServicesTestUtils.getXmlString(qElem, "state"); qi.autoCreateChildQueueEnabled = WebServicesTestUtils.getXmlBoolean(qElem, "autoCreateChildQueueEnabled"); @@ -362,11 +369,13 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements in: " + info, 18, info.length()); + assertEquals("incorrect number of elements in: " + info, 19, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), - (float) info.getDouble("maxCapacity"), info.getString("queueName")); + (float) info.getDouble("maxCapacity"), + info.getString("queueName"), + info.getString("queuePath")); JSONObject health = info.getJSONObject("health"); assertNotNull(health); assertEquals("incorrect number of elements in: " + health, 3, @@ -401,22 +410,24 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { } private void verifyClusterSchedulerGeneric(String type, float usedCapacity, - float capacity, float maxCapacity, String queueName) throws Exception { + float capacity, float maxCapacity, String queueName, String queuePath) + throws Exception { assertTrue("type doesn't match", "capacityScheduler".matches(type)); assertEquals("usedCapacity doesn't match", 0, usedCapacity, 1e-3f); assertEquals("capacity doesn't match", 100, capacity, 1e-3f); assertEquals("maxCapacity doesn't match", 100, maxCapacity, 1e-3f); assertTrue("queueName doesn't match", "root".matches(queueName)); + assertTrue("queuePath doesn't match", "root".matches(queuePath)); } private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 33; + int numExpectedElements = 34; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 51; + numExpectedElements = 52; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); @@ -430,6 +441,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { qi.absoluteUsedCapacity = (float) info.getDouble("absoluteUsedCapacity"); qi.numApplications = info.getInt("numApplications"); qi.queueName = info.getString("queueName"); + qi.queuePath = info.getString("queuePath"); qi.state = info.getString("state"); verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity); @@ -502,6 +514,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { assertEquals("numApplications doesn't match", 0, info.numApplications); assertTrue("queueName doesn't match, got: " + info.queueName + " expected: " + q, qshortName.matches(info.queueName)); + assertTrue("queuePath doesn't match, got: " + info.getQueuePath() + + " expected: " + q, q.matches(info.getQueuePath())); assertTrue("state doesn't match", (csConf.getState(q).toString()).matches(info.state)); if (q.equals("c")) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index 8d9c453731c..8ada04c0592 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -574,7 +574,7 @@ public class TestRMWebServicesForCSWithPartitions extends JerseyTestBase { JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 18, info.length()); + assertEquals("incorrect number of elements", 19, info.length()); JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES); JSONArray partitionsCapsArray = capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);