Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-10636. CS Auto Queue creation should reject submissions with empty path parts. Contributed by Gergely Pollak.

This commit is contained in:
Peter Bacsko 2021-02-19 13:57:30 +01:00
parent 4383726d19
commit d28b6f90c8
2 changed files with 62 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
View File

@ -962,6 +962,17 @@ public class CapacityScheduler extends
if (placementContext == null) {
fallbackContext = CSQueueUtils.extractQueuePath(queueName);
}
//we need to make sure there is no empty path parts present
String path = fallbackContext.getFullQueuePath();
String[] pathParts = path.split("\\.");
for (int i = 0; i < pathParts.length; i++) {
if ("".equals(pathParts[i])) {
LOG.error("Application submitted to invalid path: '{}'", path);
return null;
}
}
if (fallbackContext.hasParentQueue()) {
try {
return autoCreateLeafQueue(fallbackContext);
@ -1022,8 +1033,8 @@ public class CapacityScheduler extends
}
//Could be a potential auto-created leaf queue
CSQueue queue = getOrCreateQueueFromPlacementContext(applicationId, user,
queueName, placementContext, false);
CSQueue queue = getOrCreateQueueFromPlacementContext(
applicationId, user, queueName, placementContext, false);
if (queue == null) {
String message;

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
@ -89,6 +90,7 @@ import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager
.NO_LABEL;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@ -535,6 +537,53 @@ public class TestCapacitySchedulerAutoQueueCreation
}
}
@Test
public void testAutoQueueCreationFailsForEmptyPathWithAQCAndWeightMode()
throws Exception {
if (mockRM != null) {
mockRM.stop();
}
//We need a special configuration we only need a V2 queue auto creation
//And weight mode, to allow dynamic auto queue creation for root
CapacitySchedulerConfiguration conf = setupSchedulerConfiguration();
conf.setAutoQueueCreationV2Enabled(ROOT, true);
conf.setCapacity("root.default", "1w");
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
//Just a regular mockRM and CapacityScheduler instance
MockRM newMockRM = new MockRM(conf);
newMockRM.start();
((CapacityScheduler) newMockRM.getResourceScheduler()).start();
CapacityScheduler newCS =
(CapacityScheduler) newMockRM.getResourceScheduler();
try {
//submitting to root..user, this should fail WITHOUT crashing the RM
submitApp(newCS, USER0, "user", "root.");
RMContext rmContext = mock(RMContext.class);
when(rmContext.getDispatcher()).thenReturn(dispatcher);
newCS.setRMContext(rmContext);
ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(
appId, "user", "root.");
newCS.handle(addAppEvent);
RMAppEvent event = new RMAppEvent(appId, RMAppEventType.APP_REJECTED,
"error");
dispatcher.spyOnNextEvent(event, 10000);
} finally {
((CapacityScheduler) newMockRM.getResourceScheduler()).stop();
newMockRM.stop();
}
}
/**
* This test case checks if a mapping rule can put an application to an auto
* created queue even if an other queue with the same leaf name already