Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-10746. RmWebApp add default-node-label-expression to the queue info. Contributed by Gergely Pollak

This commit is contained in:
Szilard Nemeth 2021-04-23 16:12:12 +02:00
parent f76a2a7606
commit 919daec36b
4 changed files with 65 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
View File

@ -61,6 +61,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected String queueType;
protected String creationMethod;
protected String autoCreationEligibility;
protected String defaultNodeLabelExpression;
@XmlTransient
static final float EPSILON = 1e-8f;
@ -114,6 +115,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
creationMethod = CapacitySchedulerInfoHelper.getCreationMethod(parent);
autoCreationEligibility = CapacitySchedulerInfoHelper
.getAutoCreationEligibility(parent);
defaultNodeLabelExpression = parent.getDefaultNodeLabelExpression();
}
public float getCapacity() {

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
View File

@ -52,7 +52,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected ResourceInfo userAMResourceLimit;
protected boolean preemptionDisabled;
protected boolean intraQueuePreemptionDisabled;
protected String defaultNodeLabelExpression;
protected int defaultPriority;
protected boolean isAutoCreatedLeafQueue;
protected long maxApplicationLifetime;
@ -81,7 +80,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
intraQueuePreemptionDisabled = q.getIntraQueuePreemptionDisabled();
orderingPolicyDisplayName = q.getOrderingPolicy().getInfo();
orderingPolicyInfo = q.getOrderingPolicy().getConfigName();
defaultNodeLabelExpression = q.getDefaultNodeLabelExpression();
defaultPriority = q.getDefaultApplicationPriority().getPriority();
ArrayList<UserInfo> usersList = users.getUsersList();
if (usersList.isEmpty()) {
@ -148,17 +146,17 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public float getConfiguredMaxAMResourceLimit() {
return configuredMaxAMResourceLimit;
}
public ResourceInfo getAMResourceLimit() {
return AMResourceLimit;
}
public ResourceInfo getUsedAMResource() {
return usedAMResource;
}
public ResourceInfo getUserAMResourceLimit() {
return userAMResourceLimit;
return userAMResourceLimit;
}
public boolean getPreemptionDisabled() {
@ -172,10 +170,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public String getOrderingPolicyDisplayName() {
return orderingPolicyDisplayName;
}
public String getDefaultNodeLabelExpression() {
return defaultNodeLabelExpression;
}
public int getDefaultApplicationPriority() {
return defaultPriority;

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -91,6 +91,7 @@ public class CapacitySchedulerQueueInfo {
protected String queueType;
protected String creationMethod;
protected String autoCreationEligibility;
protected String defaultNodeLabelExpression;
CapacitySchedulerQueueInfo() {
};
@ -120,6 +121,7 @@ public class CapacitySchedulerQueueInfo {
reservedContainers = q.getMetrics().getReservedContainers();
queueName = q.getQueueName();
state = q.getState();
defaultNodeLabelExpression = q.getDefaultNodeLabelExpression();
resourcesUsed = new ResourceInfo(q.getUsedResources());
if (q instanceof PlanQueue && !((PlanQueue) q).showReservationsAsQueues()) {
hideReservationQueues = true;
@ -271,7 +273,7 @@ public class CapacitySchedulerQueueInfo {
static float cap(float val, float low, float hi) {
return Math.min(Math.max(val, low), hi);
}
public ArrayList<String> getNodeLabels() {
return this.nodeLabels;
}
@ -335,4 +337,8 @@ public class CapacitySchedulerQueueInfo {
public float getNormalizedWeight() {
return normalizedWeight;
}
public String getDefaultNodeLabelExpression() {
return defaultNodeLabelExpression;
}
}

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -661,6 +661,58 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
}
}
@Test
public void testNodeLabelDefaultAPI() throws Exception {
CapacitySchedulerConfiguration config =
((CapacityScheduler)rm.getResourceScheduler()).getConfiguration();
config.setDefaultNodeLabelExpression("root", "ROOT-INHERITED");
config.setDefaultNodeLabelExpression("root.a", "root-a-default-label");
rm.getResourceScheduler().reinitialize(config, rm.getRMContext());
//Start RM so that it accepts app submissions
rm.start();
try {
//Get the XML from ws/v1/cluster/scheduler
WebResource r = resource();
ClientResponse response = r.path("ws/v1/cluster/scheduler")
.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
String xml = response.getEntity(String.class);
DocumentBuilder db = DocumentBuilderFactory.newInstance()
.newDocumentBuilder();
InputSource is = new InputSource();
is.setCharacterStream(new StringReader(xml));
//Parse the XML we got
Document dom = db.parse(is);
NodeList allQueues = dom.getElementsByTagName("queue");
for (int i = 0; i < allQueues.getLength(); ++i) {
Node queueNode = allQueues.item(i);
Node queuePathNode = getChildNodeByName(queueNode, "queuePath");
if (queuePathNode == null) {
continue;
}
String queuePath = queuePathNode.getTextContent();
if (queuePath != null) {
if (queuePath.startsWith("root.a")) {
assertEquals("root-a-default-label",
getChildNodeByName(queueNode, "defaultNodeLabelExpression")
.getTextContent());
} else {
assertEquals("ROOT-INHERITED",
getChildNodeByName(queueNode, "defaultNodeLabelExpression")
.getTextContent());
}
}
}
} finally {
rm.stop();
}
}
private void checkResourcesUsed(JSONObject queue) throws JSONException {
queue.getJSONObject("resourcesUsed").getInt("memory");
queue.getJSONObject("resourcesUsed").getInt("vCores");