YARN-10636. CS Auto Queue creation should reject submissions with empty path parts. Contributed by Gergely Pollak.

This commit is contained in:
Peter Bacsko 2021-02-19 13:57:30 +01:00
parent 4383726d19
commit d28b6f90c8
2 changed files with 62 additions and 2 deletions

View File

@ -962,6 +962,17 @@ public class CapacityScheduler extends
if (placementContext == null) { if (placementContext == null) {
fallbackContext = CSQueueUtils.extractQueuePath(queueName); fallbackContext = CSQueueUtils.extractQueuePath(queueName);
} }
//we need to make sure there is no empty path parts present
String path = fallbackContext.getFullQueuePath();
String[] pathParts = path.split("\\.");
for (int i = 0; i < pathParts.length; i++) {
if ("".equals(pathParts[i])) {
LOG.error("Application submitted to invalid path: '{}'", path);
return null;
}
}
if (fallbackContext.hasParentQueue()) { if (fallbackContext.hasParentQueue()) {
try { try {
return autoCreateLeafQueue(fallbackContext); return autoCreateLeafQueue(fallbackContext);
@ -1022,8 +1033,8 @@ public class CapacityScheduler extends
} }
//Could be a potential auto-created leaf queue //Could be a potential auto-created leaf queue
CSQueue queue = getOrCreateQueueFromPlacementContext(applicationId, user, CSQueue queue = getOrCreateQueueFromPlacementContext(
queueName, placementContext, false); applicationId, user, queueName, placementContext, false);
if (queue == null) { if (queue == null) {
String message; String message;

View File

@ -17,6 +17,7 @@
*/ */
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
@ -89,6 +90,7 @@ import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager
.NO_LABEL; .NO_LABEL;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNull;
@ -535,6 +537,53 @@ public class TestCapacitySchedulerAutoQueueCreation
} }
} }
@Test
public void testAutoQueueCreationFailsForEmptyPathWithAQCAndWeightMode()
throws Exception {
if (mockRM != null) {
mockRM.stop();
}
//We need a special configuration we only need a V2 queue auto creation
//And weight mode, to allow dynamic auto queue creation for root
CapacitySchedulerConfiguration conf = setupSchedulerConfiguration();
conf.setAutoQueueCreationV2Enabled(ROOT, true);
conf.setCapacity("root.default", "1w");
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
//Just a regular mockRM and CapacityScheduler instance
MockRM newMockRM = new MockRM(conf);
newMockRM.start();
((CapacityScheduler) newMockRM.getResourceScheduler()).start();
CapacityScheduler newCS =
(CapacityScheduler) newMockRM.getResourceScheduler();
try {
//submitting to root..user, this should fail WITHOUT crashing the RM
submitApp(newCS, USER0, "user", "root.");
RMContext rmContext = mock(RMContext.class);
when(rmContext.getDispatcher()).thenReturn(dispatcher);
newCS.setRMContext(rmContext);
ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(
appId, "user", "root.");
newCS.handle(addAppEvent);
RMAppEvent event = new RMAppEvent(appId, RMAppEventType.APP_REJECTED,
"error");
dispatcher.spyOnNextEvent(event, 10000);
} finally {
((CapacityScheduler) newMockRM.getResourceScheduler()).stop();
newMockRM.stop();
}
}
/** /**
* This test case checks if a mapping rule can put an application to an auto * This test case checks if a mapping rule can put an application to an auto
* created queue even if an other queue with the same leaf name already * created queue even if an other queue with the same leaf name already