Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-10598. CS Flexible Auto Queue Creation: Modify RM /scheduler endpoint to extend the creation type with additional information. Contributed by Benjamin Teke

This commit is contained in:
Szilard Nemeth 2021-01-27 18:14:06 +01:00
parent 28cc912a5c
commit 9628aa87bf
6 changed files with 159 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
View File

@ -58,6 +58,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected String orderingPolicyInfo;
protected String mode;
protected String queueType;
protected String creationMethod;
protected String autoCreationEligibility;
@XmlTransient
static final float EPSILON = 1e-8f;
@ -107,6 +109,9 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
}
mode = CapacitySchedulerInfoHelper.getMode(parent);
queueType = CapacitySchedulerInfoHelper.getQueueType(parent);
creationMethod = CapacitySchedulerInfoHelper.getCreationMethod(parent);
autoCreationEligibility = CapacitySchedulerInfoHelper
.getAutoCreationEligibility(parent);
}
public float getCapacity() {

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -91,6 +91,8 @@ public class CapacitySchedulerQueueInfo {
protected LeafQueueTemplateInfo leafQueueTemplate;
protected String mode;
protected String queueType;
protected String creationMethod;
protected String autoCreationEligibility;
CapacitySchedulerQueueInfo() {
};
@ -137,6 +139,9 @@ public class CapacitySchedulerQueueInfo {
mode = CapacitySchedulerInfoHelper.getMode(q);
queueType = CapacitySchedulerInfoHelper.getQueueType(q);
creationMethod = CapacitySchedulerInfoHelper.getCreationMethod(q);
autoCreationEligibility = CapacitySchedulerInfoHelper
.getAutoCreationEligibility(q);
ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
populateQueueResourceUsage(queueResourceUsage);

71
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
View File

@ -18,17 +18,44 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractManagedParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
/**
* Helper class to describe a queue's type, its creation method and its
* eligibility of having auto created children.
*
* queueType: a queue can be a parent or a leaf.
*
* creationMethod: the creation method of the queue. Can be: static,
* dynamicLegacy or dynamicFlexible. When the legacy way of queue auto-creation
* (before YARN-10506) is used, a parent can only be static (ManagedParent)
* and a leaf queue can only be dynamicLegacy (no static child queues are
* allowed under ManagedParents). When the flexible auto queue creation is used
* both a parent and a leaf can be either static or dynamicFlexible.
*
* autoCreationEligibility: describes whether a queue can have dynamically
* created children. Can be: off, legacy or flexible. Every leaf will have this
* field with the value off, as they can't have children. When the legacy way
* of queue auto-creation (before YARN-10506) is used a ManagedParent will have
* the legacy value. When the flexible auto queue creation is used a static
* parent can have the value flexible if it is configured to allow auto queue
* creation, or off if it is not. A dynamic parent implicitly will have the
* value flexible, as a dynamically created parent cannot have static children.
*/
public class CapacitySchedulerInfoHelper {
private static final String AUTO_CREATED_LEAF = "autoCreatedLeaf";
private static final String STATIC_LEAF = "staticLeaf";
private static final String AUTO_CREATED_PARENT = "autoCreatedParent";
private static final String STATIC_PARENT = "staticParent";
private static final String PARENT_QUEUE = "parent";
private static final String LEAF_QUEUE = "leaf";
private static final String UNKNOWN_QUEUE = "unknown";
private static final String STATIC_QUEUE = "static";
private static final String LEGACY_DYNAMIC_QUEUE = "dynamicLegacy";
private static final String FLEXIBLE_DYNAMIC_QUEUE = "dynamicFlexible";
private static final String AUTO_CREATION_OFF = "off";
private static final String AUTO_CREATION_LEGACY = "legacy";
private static final String AUTO_CREATION_FLEXIBLE = "flexible";
private CapacitySchedulerInfoHelper() {}
@ -52,19 +79,31 @@ public class CapacitySchedulerInfoHelper {
public static String getQueueType(CSQueue queue) {
if (queue instanceof LeafQueue) {
if (((AbstractCSQueue)queue).isDynamicQueue()) {
return AUTO_CREATED_LEAF;
} else {
return STATIC_LEAF;
}
return LEAF_QUEUE;
} else if (queue instanceof ParentQueue) {
if (((AbstractCSQueue)queue).isDynamicQueue()) {
return AUTO_CREATED_PARENT;
} else {
//A ParentQueue with isDynamic=false or an AbstractManagedParentQueue
return STATIC_PARENT;
}
return PARENT_QUEUE;
}
return UNKNOWN_QUEUE;
}
public static String getCreationMethod(CSQueue queue) {
if (queue instanceof AutoCreatedLeafQueue) {
return LEGACY_DYNAMIC_QUEUE;
} else if (((AbstractCSQueue)queue).isDynamicQueue()) {
return FLEXIBLE_DYNAMIC_QUEUE;
} else {
return STATIC_QUEUE;
}
}
public static String getAutoCreationEligibility(CSQueue queue) {
if (queue instanceof ManagedParentQueue) {
return AUTO_CREATION_LEGACY;
} else if (queue instanceof ParentQueue &&
((ParentQueue)queue).isEligibleForAutoQueueCreation()) {
return AUTO_CREATION_FLEXIBLE;
} else {
return AUTO_CREATION_OFF;
}
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -362,7 +362,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements in: " + info, 16, info.length());
assertEquals("incorrect number of elements in: " + info, 18, info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
@ -413,10 +413,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
int numExpectedElements = 31;
int numExpectedElements = 33;
boolean isParentQueue = true;
if (!info.has("queues")) {
numExpectedElements = 49;
numExpectedElements = 51;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());

122
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
View File

@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData;
import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils;
@ -73,11 +75,14 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
private static final float EXP_ROOT_WEIGHT_IN_WEIGHT_MODE = 1.0F;
private static final float EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE = 1.0F;
private static final double DELTA = 0.00001;
private static final String STATIC_PARENT = "staticParent";
private static final String STATIC_LEAF = "staticLeaf";
private static final String PARENT_QUEUE = "parent";
private static final String LEAF_QUEUE = "leaf";
private static final String STATIC_QUEUE = "static";
private static final String FLEXIBLE_DYNAMIC_QUEUE = "dynamicFlexible";
private static final String AUTO_CREATION_OFF = "off";
private static final String AUTO_CREATION_LEGACY = "legacy";
private static final String AUTO_CREATION_FLEXIBLE = "flexible";
private static final int GB = 1024;
private static final String AUTO_CREATED_LEAF = "autoCreatedLeaf";
private static final String AUTO_CREATED_PARENT = "autoCreatedParent";
protected static MockRM RM;
private CapacitySchedulerAutoQueueHandler autoQueueHandler;
@ -88,13 +93,18 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
public final float weight;
public final float normalizedWeight;
private String queueType;
private String creationMethod;
private String autoCreationEligibility;
public ExpectedQueueWithProperties(String path, float weight,
float normalizedWeight, String queueType) {
float normalizedWeight, String queueType, String creationMethod,
String autoCreationEligibility) {
this.path = path;
this.weight = weight;
this.normalizedWeight = normalizedWeight;
this.queueType = queueType;
this.creationMethod = creationMethod;
this.autoCreationEligibility = autoCreationEligibility;
}
}
@ -161,16 +171,41 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
validateSchedulerInfo(json, "percentage",
new ExpectedQueueWithProperties("root",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
STATIC_PARENT),
PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.default",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
STATIC_LEAF),
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test1",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
STATIC_LEAF),
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test2",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
STATIC_LEAF));
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
}
@Test
public void testSchedulerResponsePercentageModeLegacyAutoCreation()
throws Exception {
Configuration config = CSConfigGenerator
.createPercentageConfigLegacyAutoCreation();
config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
YarnConfiguration.MEMORY_CONFIGURATION_STORE);
initResourceManager(config);
JSONObject json = sendRequestToSchedulerEndpoint();
validateSchedulerInfo(json, "percentage",
new ExpectedQueueWithProperties("root",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.default",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test1",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.managedtest2",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_LEGACY));
}
@Test
@ -186,16 +221,16 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
validateSchedulerInfo(json, "absolute",
new ExpectedQueueWithProperties("root",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
STATIC_PARENT),
PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.default",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
STATIC_LEAF),
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test1",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
STATIC_LEAF),
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test2",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
STATIC_LEAF));
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
}
@Test
@ -211,13 +246,13 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
validateSchedulerInfo(json, "weight",
new ExpectedQueueWithProperties("root",
EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
STATIC_PARENT),
PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.default", 10.0f, 0.5f,
STATIC_LEAF),
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test1", 4.0f, 0.2f,
STATIC_LEAF),
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test2", 6.0f, 0.3f,
STATIC_LEAF));
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
}
@Test
@ -234,13 +269,13 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
validateSchedulerInfo(json, "weight",
new ExpectedQueueWithProperties("root",
EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
STATIC_PARENT),
PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.default", 10.0f, 0.5f,
STATIC_LEAF),
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test1", 4.0f, 0.2f,
STATIC_LEAF),
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test2", 6.0f, 0.3f,
STATIC_LEAF));
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
//Now create some auto created queues
createQueue("root.auto1");
@ -255,42 +290,42 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
int sumOfWeights = 24;
ExpectedQueueWithProperties expectedRootQ =
new ExpectedQueueWithProperties("root",
EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
STATIC_PARENT);
EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF);
validateSchedulerInfo(json, "weight",
expectedRootQ,
new ExpectedQueueWithProperties("root.auto1",
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
AUTO_CREATED_LEAF),
LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.auto2",
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
AUTO_CREATED_LEAF),
LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.auto3",
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
AUTO_CREATED_LEAF),
LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.autoParent1",
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
AUTO_CREATED_PARENT),
PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE),
new ExpectedQueueWithProperties("root.default", 10.0f,
10.0f / sumOfWeights,
STATIC_LEAF),
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test1", 4.0f,
4.0f / sumOfWeights,
STATIC_LEAF),
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
new ExpectedQueueWithProperties("root.test2", 6.0f,
6.0f / sumOfWeights,
STATIC_LEAF));
LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
validateChildrenOfParent(json, "root.autoParent1", "weight",
expectedRootQ,
new ExpectedQueueWithProperties("root.autoParent1.auto4",
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
AUTO_CREATED_LEAF));
LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF));
}
private void initAutoQueueHandler() throws Exception {
@ -433,9 +468,18 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
Float.parseFloat(obj.getString("normalizedWeight")), DELTA);
//validate queue creation type
Assert.assertEquals("Queue creation type does not match for queue " +
Assert.assertEquals("Queue type does not match for queue " +
queuePath,
expectedQueue.queueType, obj.getString("queueType"));
Assert.assertEquals("Queue creation type does not match for queue " +
queuePath,
expectedQueue.creationMethod, obj.getString("creationMethod"));
Assert.assertEquals("Queue auto creation eligibility does not " +
"match for queue " + queuePath,
expectedQueue.autoCreationEligibility,
obj.getString("autoCreationEligibility"));
}
//Validate queue paths and modes
@ -470,6 +514,20 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
return createConfiguration(conf);
}
public static Configuration createPercentageConfigLegacyAutoCreation() {
Map<String, String> conf = new HashMap<>();
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, " +
"managedtest2");
conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
conf.put("yarn.scheduler.capacity.root.managedtest2.capacity", "50");
conf.put("yarn.scheduler.capacity.root.test1.maximum-capacity", "100");
conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
conf.put("yarn.scheduler.capacity.root.managedtest2.state", "RUNNING");
conf.put("yarn.scheduler.capacity.root.managedtest2." +
"auto-create-child-queue.enabled", "true");
return createConfiguration(conf);
}
public static Configuration createAbsoluteConfig() {
Map<String, String> conf = new HashMap<>();
conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
View File

@ -574,7 +574,7 @@ public class TestRMWebServicesForCSWithPartitions extends JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements", 16, info.length());
assertEquals("incorrect number of elements", 18, info.length());
JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
JSONArray partitionsCapsArray =
capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);