YARN-10622. Fix preemption policy to exclude childless ParentQueues. Contributed by Andras Gyori
This commit is contained in:
parent
852aac34f2
commit
bad6038a48
|
@ -17,6 +17,7 @@
|
|||
*/
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
|
||||
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
@ -409,9 +410,11 @@ public class ProportionalCapacityPreemptionPolicy
|
|||
}
|
||||
|
||||
private Set<String> getLeafQueueNames(TempQueuePerPartition q) {
|
||||
// If its a ManagedParentQueue, it might not have any children
|
||||
if ((q.children == null || q.children.isEmpty())
|
||||
&& !(q.parentQueue instanceof ManagedParentQueue)) {
|
||||
// Also exclude ParentQueues, which might be without children
|
||||
if (CollectionUtils.isEmpty(q.children)
|
||||
&& !(q.parentQueue instanceof ManagedParentQueue)
|
||||
&& (q.parentQueue == null
|
||||
|| !q.parentQueue.isEligibleForAutoQueueCreation())) {
|
||||
return ImmutableSet.of(q.queueName);
|
||||
}
|
||||
|
||||
|
|
|
@ -81,6 +81,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
|
|||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
|
@ -1071,6 +1072,47 @@ public class TestProportionalCapacityPreemptionPolicy {
|
|||
assertEquals(newObserveOnly, policy.isObserveOnly());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLeafQueueNameExtraction() throws Exception {
|
||||
ProportionalCapacityPreemptionPolicy policy =
|
||||
buildPolicy(Q_DATA_FOR_IGNORE);
|
||||
ParentQueue root = (ParentQueue) mCS.getRootQueue();
|
||||
root.addDynamicParentQueue("childlessFlexible");
|
||||
List<CSQueue> queues = root.getChildQueues();
|
||||
ArrayList<CSQueue> extendedQueues = new ArrayList<>();
|
||||
LinkedList<ParentQueue> pqs = new LinkedList<>();
|
||||
ParentQueue dynamicParent = mockParentQueue(
|
||||
null, 0, pqs);
|
||||
when(dynamicParent.getQueuePath()).thenReturn("root.dynamicParent");
|
||||
when(dynamicParent.getQueueCapacities()).thenReturn(
|
||||
new QueueCapacities(false));
|
||||
QueueResourceQuotas dynamicParentQr = new QueueResourceQuotas();
|
||||
dynamicParentQr.setEffectiveMaxResource(Resource.newInstance(1, 1));
|
||||
dynamicParentQr.setEffectiveMinResource(Resources.createResource(1));
|
||||
dynamicParentQr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
|
||||
Resource.newInstance(1, 1));
|
||||
dynamicParentQr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL,
|
||||
Resources.createResource(1));
|
||||
when(dynamicParent.getQueueResourceQuotas()).thenReturn(dynamicParentQr);
|
||||
when(dynamicParent.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
|
||||
.thenReturn(Resources.createResource(1));
|
||||
when(dynamicParent.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
|
||||
.thenReturn(Resource.newInstance(1, 1));
|
||||
ResourceUsage resUsage = new ResourceUsage();
|
||||
resUsage.setUsed(Resources.createResource(1024));
|
||||
resUsage.setReserved(Resources.createResource(1024));
|
||||
when(dynamicParent.getQueueResourceUsage()).thenReturn(resUsage);
|
||||
when(dynamicParent.isEligibleForAutoQueueCreation()).thenReturn(true);
|
||||
extendedQueues.add(dynamicParent);
|
||||
extendedQueues.addAll(queues);
|
||||
when(root.getChildQueues()).thenReturn(extendedQueues);
|
||||
|
||||
policy.editSchedule();
|
||||
|
||||
assertFalse("dynamicParent should not be a LeafQueue " +
|
||||
"candidate", policy.getLeafQueueNames().contains("root.dynamicParent"));
|
||||
}
|
||||
|
||||
static class IsPreemptionRequestFor
|
||||
implements ArgumentMatcher<ContainerPreemptEvent> {
|
||||
private final ApplicationAttemptId appAttId;
|
||||
|
|
Loading…
Reference in New Issue