YARN-10622. Fix preemption policy to exclude childless ParentQueues. Contributed by Andras Gyori
This commit is contained in:
parent
852aac34f2
commit
bad6038a48
|
@ -17,6 +17,7 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
|
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
|
||||||
|
|
||||||
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
@ -409,9 +410,11 @@ public class ProportionalCapacityPreemptionPolicy
|
||||||
}
|
}
|
||||||
|
|
||||||
private Set<String> getLeafQueueNames(TempQueuePerPartition q) {
|
private Set<String> getLeafQueueNames(TempQueuePerPartition q) {
|
||||||
// If its a ManagedParentQueue, it might not have any children
|
// Also exclude ParentQueues, which might be without children
|
||||||
if ((q.children == null || q.children.isEmpty())
|
if (CollectionUtils.isEmpty(q.children)
|
||||||
&& !(q.parentQueue instanceof ManagedParentQueue)) {
|
&& !(q.parentQueue instanceof ManagedParentQueue)
|
||||||
|
&& (q.parentQueue == null
|
||||||
|
|| !q.parentQueue.isEligibleForAutoQueueCreation())) {
|
||||||
return ImmutableSet.of(q.queueName);
|
return ImmutableSet.of(q.queueName);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -81,6 +81,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE;
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE;
|
||||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION;
|
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertFalse;
|
||||||
import static org.junit.Assert.assertNotNull;
|
import static org.junit.Assert.assertNotNull;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
|
@ -1071,6 +1072,47 @@ public class TestProportionalCapacityPreemptionPolicy {
|
||||||
assertEquals(newObserveOnly, policy.isObserveOnly());
|
assertEquals(newObserveOnly, policy.isObserveOnly());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testLeafQueueNameExtraction() throws Exception {
|
||||||
|
ProportionalCapacityPreemptionPolicy policy =
|
||||||
|
buildPolicy(Q_DATA_FOR_IGNORE);
|
||||||
|
ParentQueue root = (ParentQueue) mCS.getRootQueue();
|
||||||
|
root.addDynamicParentQueue("childlessFlexible");
|
||||||
|
List<CSQueue> queues = root.getChildQueues();
|
||||||
|
ArrayList<CSQueue> extendedQueues = new ArrayList<>();
|
||||||
|
LinkedList<ParentQueue> pqs = new LinkedList<>();
|
||||||
|
ParentQueue dynamicParent = mockParentQueue(
|
||||||
|
null, 0, pqs);
|
||||||
|
when(dynamicParent.getQueuePath()).thenReturn("root.dynamicParent");
|
||||||
|
when(dynamicParent.getQueueCapacities()).thenReturn(
|
||||||
|
new QueueCapacities(false));
|
||||||
|
QueueResourceQuotas dynamicParentQr = new QueueResourceQuotas();
|
||||||
|
dynamicParentQr.setEffectiveMaxResource(Resource.newInstance(1, 1));
|
||||||
|
dynamicParentQr.setEffectiveMinResource(Resources.createResource(1));
|
||||||
|
dynamicParentQr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
|
||||||
|
Resource.newInstance(1, 1));
|
||||||
|
dynamicParentQr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL,
|
||||||
|
Resources.createResource(1));
|
||||||
|
when(dynamicParent.getQueueResourceQuotas()).thenReturn(dynamicParentQr);
|
||||||
|
when(dynamicParent.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
|
||||||
|
.thenReturn(Resources.createResource(1));
|
||||||
|
when(dynamicParent.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
|
||||||
|
.thenReturn(Resource.newInstance(1, 1));
|
||||||
|
ResourceUsage resUsage = new ResourceUsage();
|
||||||
|
resUsage.setUsed(Resources.createResource(1024));
|
||||||
|
resUsage.setReserved(Resources.createResource(1024));
|
||||||
|
when(dynamicParent.getQueueResourceUsage()).thenReturn(resUsage);
|
||||||
|
when(dynamicParent.isEligibleForAutoQueueCreation()).thenReturn(true);
|
||||||
|
extendedQueues.add(dynamicParent);
|
||||||
|
extendedQueues.addAll(queues);
|
||||||
|
when(root.getChildQueues()).thenReturn(extendedQueues);
|
||||||
|
|
||||||
|
policy.editSchedule();
|
||||||
|
|
||||||
|
assertFalse("dynamicParent should not be a LeafQueue " +
|
||||||
|
"candidate", policy.getLeafQueueNames().contains("root.dynamicParent"));
|
||||||
|
}
|
||||||
|
|
||||||
static class IsPreemptionRequestFor
|
static class IsPreemptionRequestFor
|
||||||
implements ArgumentMatcher<ContainerPreemptEvent> {
|
implements ArgumentMatcher<ContainerPreemptEvent> {
|
||||||
private final ApplicationAttemptId appAttId;
|
private final ApplicationAttemptId appAttId;
|
||||||
|
|
Loading…
Reference in New Issue