YARN-10237. Add isAbsoluteResource config for queue in scheduler response. Contributed by Prabhu Joseph

This commit is contained in:
Szilard Nemeth 2020-04-29 09:36:37 +02:00
parent 08dbe32122
commit 75433a658d
3 changed files with 34 additions and 3 deletions

View File

@ -377,7 +377,8 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "^\\[[\\w\\.,\\-_=\\ /]+\\]$"; public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "^\\[[\\w\\.,\\-_=\\ /]+\\]$";
private static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE); public static final Pattern RESOURCE_PATTERN =
Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE);
/** /**
* Different resource types supported. * Different resource types supported.

View File

@ -42,6 +42,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQu
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
CapacitySchedulerConfiguration.RESOURCE_PATTERN;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
CapacitySchedulerConfiguration.CAPACITY;
@XmlRootElement @XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD) @XmlAccessorType(XmlAccessType.FIELD)
@XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class}) @XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class})
@ -61,6 +66,7 @@ public class CapacitySchedulerQueueInfo {
protected float absoluteUsedCapacity; protected float absoluteUsedCapacity;
protected int numApplications; protected int numApplications;
protected String queueName; protected String queueName;
protected boolean isAbsoluteResource;
protected QueueState state; protected QueueState state;
protected CapacitySchedulerQueueInfoList queues; protected CapacitySchedulerQueueInfoList queues;
protected ResourceInfo resourcesUsed; protected ResourceInfo resourcesUsed;
@ -151,6 +157,11 @@ public class CapacitySchedulerQueueInfo {
orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy()
.getConfigName(); .getConfigName();
} }
String configuredCapacity = conf.get(
CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY);
isAbsoluteResource = (configuredCapacity != null)
&& RESOURCE_PATTERN.matcher(configuredCapacity).find();
} }
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@ -203,6 +214,10 @@ public class CapacitySchedulerQueueInfo {
return pendingContainers; return pendingContainers;
} }
public boolean isAbsoluteResource() {
return isAbsoluteResource;
}
public String getQueueName() { public String getQueueName() {
return this.queueName; return this.queueName;
} }

View File

@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
@ -79,6 +80,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
int numApplications; int numApplications;
String queueName; String queueName;
String state; String state;
boolean isAbsoluteResource;
} }
private class LeafQueueInfo extends QueueInfo { private class LeafQueueInfo extends QueueInfo {
@ -131,6 +133,9 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
final String B = CapacitySchedulerConfiguration.ROOT + ".b"; final String B = CapacitySchedulerConfiguration.ROOT + ".b";
config.setCapacity(B, 89.5f); config.setCapacity(B, 89.5f);
final String C = CapacitySchedulerConfiguration.ROOT + ".c";
config.setCapacity(C, "[memory=1024]");
// Define 2nd-level queues // Define 2nd-level queues
final String A1 = A + ".a1"; final String A1 = A + ".a1";
final String A2 = A + ".a2"; final String A2 = A + ".a2";
@ -290,6 +295,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
WebServicesTestUtils.getXmlInt(qElem, "numApplications"); WebServicesTestUtils.getXmlInt(qElem, "numApplications");
qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName"); qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName");
qi.state = WebServicesTestUtils.getXmlString(qElem, "state"); qi.state = WebServicesTestUtils.getXmlString(qElem, "state");
qi.isAbsoluteResource = WebServicesTestUtils.getXmlBoolean(qElem,
"isAbsoluteResource");
verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity); verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity);
if (hasSubQueues) { if (hasSubQueues) {
for (int j = 0; j < children.getLength(); j++) { for (int j = 0; j < children.getLength(); j++) {
@ -384,10 +391,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifySubQueue(JSONObject info, String q, private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity) float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception { throws JSONException, Exception {
int numExpectedElements = 24; int numExpectedElements = 25;
boolean isParentQueue = true; boolean isParentQueue = true;
if (!info.has("queues")) { if (!info.has("queues")) {
numExpectedElements = 42; numExpectedElements = 43;
isParentQueue = false; isParentQueue = false;
} }
assertEquals("incorrect number of elements", numExpectedElements, info.length()); assertEquals("incorrect number of elements", numExpectedElements, info.length());
@ -471,6 +478,14 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
+ " expected: " + q, qshortName.matches(info.queueName)); + " expected: " + q, qshortName.matches(info.queueName));
assertTrue("state doesn't match", assertTrue("state doesn't match",
(csConf.getState(q).toString()).matches(info.state)); (csConf.getState(q).toString()).matches(info.state));
if (q.equals("c")) {
assertTrue("c queue is not configured in Absolute resource",
info.isAbsoluteResource);
} else {
assertFalse(info.queueName
+ " queue is not configured in Absolute resource",
info.isAbsoluteResource);
}
} }