YARN-10918. Simplify method: CapacitySchedulerQueueManager#parseQueue. Contributed by Andras Gyori
This commit is contained in:
parent
2ece95064b
commit
db8ae4b654
|
@ -95,6 +95,13 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
|
|||
*/
|
||||
QueuePath getQueuePathObject();
|
||||
|
||||
/**
|
||||
* Checks whether the queue is a dynamic queue (created dynamically in the fashion of auto queue
|
||||
* creation v2).
|
||||
* @return true, if it is a dynamic queue, false otherwise
|
||||
*/
|
||||
boolean isDynamicQueue();
|
||||
|
||||
public PrivilegedEntity getPrivilegedEntity();
|
||||
|
||||
Resource getMaximumAllocation();
|
||||
|
|
|
@ -43,13 +43,11 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
|
|||
import org.apache.hadoop.yarn.security.Permission;
|
||||
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueueManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
|
||||
|
||||
import org.apache.hadoop.classification.VisibleForTesting;
|
||||
|
@ -231,99 +229,62 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
|
|||
* @throws IOException
|
||||
*/
|
||||
static CSQueue parseQueue(
|
||||
CapacitySchedulerQueueContext queueContext,
|
||||
CapacitySchedulerConfiguration conf,
|
||||
CSQueue parent, String queueName,
|
||||
CSQueueStore newQueues,
|
||||
CSQueueStore oldQueues,
|
||||
CapacitySchedulerQueueContext queueContext, CapacitySchedulerConfiguration conf,
|
||||
CSQueue parent, String queueName, CSQueueStore newQueues, CSQueueStore oldQueues,
|
||||
QueueHook hook) throws IOException {
|
||||
CSQueue queue;
|
||||
String fullQueueName = (parent == null) ?
|
||||
queueName :
|
||||
(parent.getQueuePath() + "." + queueName);
|
||||
String fullQueueName = (parent == null) ? queueName :
|
||||
(QueuePath.createFromQueues(parent.getQueuePath(), queueName).getFullPath());
|
||||
String[] staticChildQueueNames = conf.getQueues(fullQueueName);
|
||||
List<String> childQueueNames = staticChildQueueNames != null ?
|
||||
Arrays.asList(staticChildQueueNames) : Collections.emptyList();
|
||||
CSQueue oldQueue = oldQueues.get(fullQueueName);
|
||||
|
||||
boolean isReservableQueue = conf.isReservable(fullQueueName);
|
||||
boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(
|
||||
fullQueueName);
|
||||
// if a queue is eligible for auto queue creation v2
|
||||
// it must be a ParentQueue (even if it is empty)
|
||||
boolean isAutoQueueCreationV2Enabled = conf.isAutoQueueCreationV2Enabled(
|
||||
fullQueueName);
|
||||
boolean isDynamicParent = false;
|
||||
boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(fullQueueName);
|
||||
// if a queue is eligible for auto queue creation v2 it must be a ParentQueue
|
||||
// (even if it is empty)
|
||||
final boolean isDynamicParent = oldQueue instanceof ParentQueue && oldQueue.isDynamicQueue();
|
||||
boolean isAutoQueueCreationEnabledParent = isDynamicParent || conf.isAutoQueueCreationV2Enabled(
|
||||
fullQueueName) || isAutoCreateEnabled;
|
||||
|
||||
// Auto created parent queues might not have static children, but they
|
||||
// must be kept as a ParentQueue
|
||||
CSQueue oldQueue = oldQueues.get(fullQueueName);
|
||||
if (oldQueue instanceof ParentQueue) {
|
||||
isDynamicParent = ((ParentQueue) oldQueue).isDynamicQueue();
|
||||
}
|
||||
|
||||
if (childQueueNames.size() == 0 && !isDynamicParent &&
|
||||
!isAutoQueueCreationV2Enabled) {
|
||||
if (null == parent) {
|
||||
throw new IllegalStateException(
|
||||
"Queue configuration missing child queue names for " + queueName);
|
||||
}
|
||||
// Check if the queue will be dynamically managed by the Reservation
|
||||
// system
|
||||
if (childQueueNames.size() == 0 && !isAutoQueueCreationEnabledParent) {
|
||||
validateParent(parent, queueName);
|
||||
// Check if the queue will be dynamically managed by the Reservation system
|
||||
if (isReservableQueue) {
|
||||
queue = new PlanQueue(queueContext, queueName, parent,
|
||||
oldQueues.get(fullQueueName));
|
||||
|
||||
//initializing the "internal" default queue, for SLS compatibility
|
||||
String defReservationId =
|
||||
queueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
|
||||
|
||||
List<CSQueue> childQueues = new ArrayList<>();
|
||||
ReservationQueue resQueue = new ReservationQueue(queueContext,
|
||||
defReservationId, (PlanQueue) queue);
|
||||
try {
|
||||
resQueue.setEntitlement(new QueueEntitlement(1.0f, 1.0f));
|
||||
} catch (SchedulerDynamicEditException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
childQueues.add(resQueue);
|
||||
((PlanQueue) queue).setChildQueues(childQueues);
|
||||
newQueues.add(resQueue);
|
||||
|
||||
} else if (isAutoCreateEnabled) {
|
||||
queue = new ManagedParentQueue(queueContext, queueName, parent,
|
||||
oldQueues.get(fullQueueName));
|
||||
|
||||
} else{
|
||||
queue = new LeafQueue(queueContext, queueName, parent,
|
||||
oldQueues.get(fullQueueName));
|
||||
// Used only for unit tests
|
||||
queue = hook.hook(queue);
|
||||
queue = new PlanQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName));
|
||||
ReservationQueue defaultResQueue = ((PlanQueue) queue).initializeDefaultInternalQueue();
|
||||
newQueues.add(defaultResQueue);
|
||||
} else {
|
||||
queue = new LeafQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName));
|
||||
}
|
||||
} else{
|
||||
|
||||
queue = hook.hook(queue);
|
||||
} else {
|
||||
if (isReservableQueue) {
|
||||
throw new IllegalStateException(
|
||||
"Only Leaf Queues can be reservable for " + fullQueueName);
|
||||
throw new IllegalStateException("Only Leaf Queues can be reservable for " + fullQueueName);
|
||||
}
|
||||
|
||||
ParentQueue parentQueue;
|
||||
if (isAutoCreateEnabled) {
|
||||
parentQueue = new ManagedParentQueue(queueContext, queueName, parent,
|
||||
oldQueues.get(fullQueueName));
|
||||
} else{
|
||||
parentQueue = new ParentQueue(queueContext, queueName, parent,
|
||||
oldQueues.get(fullQueueName));
|
||||
parentQueue = new ManagedParentQueue(queueContext, queueName, parent, oldQueues.get(
|
||||
fullQueueName));
|
||||
} else {
|
||||
parentQueue = new ParentQueue(queueContext, queueName, parent, oldQueues.get(
|
||||
fullQueueName));
|
||||
}
|
||||
|
||||
// Used only for unit tests
|
||||
queue = hook.hook(parentQueue);
|
||||
|
||||
List<CSQueue> childQueues = new ArrayList<>();
|
||||
for (String childQueueName : childQueueNames) {
|
||||
CSQueue childQueue = parseQueue(queueContext, conf, queue, childQueueName,
|
||||
newQueues, oldQueues, hook);
|
||||
CSQueue childQueue = parseQueue(queueContext, conf, queue, childQueueName, newQueues,
|
||||
oldQueues, hook);
|
||||
childQueues.add(childQueue);
|
||||
}
|
||||
parentQueue.setChildQueues(childQueues);
|
||||
|
||||
if (!childQueues.isEmpty()) {
|
||||
parentQueue.setChildQueues(childQueues);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
@ -721,4 +682,11 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
|
|||
// that existingQueues contain valid dynamic queues.
|
||||
return !isDynamicQueue(parent);
|
||||
}
|
||||
|
||||
private static void validateParent(CSQueue parent, String queueName) {
|
||||
if (parent == null) {
|
||||
throw new IllegalStateException("Queue configuration missing child queue names for "
|
||||
+ queueName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -21,8 +21,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
|
|||
import java.io.IOException;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
@ -123,6 +125,23 @@ public class PlanQueue extends AbstractManagedParentQueue {
|
|||
}
|
||||
}
|
||||
|
||||
public ReservationQueue initializeDefaultInternalQueue() throws IOException {
|
||||
//initializing the "internal" default queue, for SLS compatibility
|
||||
String defReservationId =
|
||||
getQueueName() + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
|
||||
|
||||
ReservationQueue resQueue = new ReservationQueue(queueContext,
|
||||
defReservationId, this);
|
||||
try {
|
||||
resQueue.initializeEntitlements();
|
||||
} catch (SchedulerDynamicEditException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
childQueues.add(resQueue);
|
||||
|
||||
return resQueue;
|
||||
}
|
||||
|
||||
private void updateQuotas(float newUserLimit, float newUserLimitFactor,
|
||||
int newMaxAppsForReservation, int newMaxAppsPerUserForReservation) {
|
||||
this.userLimit = newUserLimit;
|
||||
|
|
|
@ -22,6 +22,8 @@ import java.io.IOException;
|
|||
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
@ -74,6 +76,10 @@ public class ReservationQueue extends AbstractAutoCreatedLeafQueue {
|
|||
}
|
||||
}
|
||||
|
||||
public void initializeEntitlements() throws SchedulerDynamicEditException {
|
||||
setEntitlement(new QueueEntitlement(1.0f, 1.0f));
|
||||
}
|
||||
|
||||
private void updateQuotas(float userLimit, float userLimitFactor,
|
||||
int maxAppsForReservation, int maxAppsPerUserForReservation) {
|
||||
setUserLimit(userLimit);
|
||||
|
|
Loading…
Reference in New Issue