YARN-9970. Refactor TestUserGroupMappingPlacementRule#verifyQueueMapping. Contributed by Manikandan R
This commit is contained in:
parent
a2fdd7c2b5
commit
7c5cecc3b3
|
@ -0,0 +1,183 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||||
|
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Queue Mapping class to hold the queue mapping information.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
@Private
|
||||||
|
public class QueueMapping {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Builder class for QueueMapping.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public static class QueueMappingBuilder {
|
||||||
|
|
||||||
|
private MappingType type;
|
||||||
|
private String source;
|
||||||
|
private String queue;
|
||||||
|
private String parentQueue;
|
||||||
|
|
||||||
|
public QueueMappingBuilder() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static QueueMappingBuilder create() {
|
||||||
|
return new QueueMappingBuilder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMappingBuilder type(MappingType mappingType) {
|
||||||
|
this.type = mappingType;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMappingBuilder source(String mappingSource) {
|
||||||
|
this.source = mappingSource;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMappingBuilder queue(String mappingQueue) {
|
||||||
|
this.queue = mappingQueue;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMappingBuilder parentQueue(String mappingParentQueue) {
|
||||||
|
this.parentQueue = mappingParentQueue;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMapping build() {
|
||||||
|
return new QueueMapping(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private QueueMapping(QueueMappingBuilder builder) {
|
||||||
|
this.type = builder.type;
|
||||||
|
this.source = builder.source;
|
||||||
|
this.queue = builder.queue;
|
||||||
|
this.parentQueue = builder.parentQueue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Different types of mapping.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public enum MappingType {
|
||||||
|
|
||||||
|
USER("u"), GROUP("g");
|
||||||
|
|
||||||
|
private final String type;
|
||||||
|
|
||||||
|
MappingType(String type) {
|
||||||
|
this.type = type;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String toString() {
|
||||||
|
return type;
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
private MappingType type;
|
||||||
|
private String source;
|
||||||
|
private String queue;
|
||||||
|
private String parentQueue;
|
||||||
|
|
||||||
|
private final static String DELIMITER = ":";
|
||||||
|
|
||||||
|
public String getQueue() {
|
||||||
|
return queue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getParentQueue() {
|
||||||
|
return parentQueue;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean hasParentQueue() {
|
||||||
|
return parentQueue != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public MappingType getType() {
|
||||||
|
return type;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSource() {
|
||||||
|
return source;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
final int prime = 31;
|
||||||
|
int result = 1;
|
||||||
|
result =
|
||||||
|
prime * result + ((parentQueue == null) ? 0 : parentQueue.hashCode());
|
||||||
|
result = prime * result + ((queue == null) ? 0 : queue.hashCode());
|
||||||
|
result = prime * result + ((source == null) ? 0 : source.hashCode());
|
||||||
|
result = prime * result + ((type == null) ? 0 : type.hashCode());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object obj) {
|
||||||
|
if (this == obj) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (obj == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (getClass() != obj.getClass()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
QueueMapping other = (QueueMapping) obj;
|
||||||
|
if (parentQueue == null) {
|
||||||
|
if (other.parentQueue != null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else if (!parentQueue.equals(other.parentQueue)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (queue == null) {
|
||||||
|
if (other.queue != null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else if (!queue.equals(other.queue)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (source == null) {
|
||||||
|
if (other.source != null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else if (!source.equals(other.source)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (type != other.type) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return type.toString() + DELIMITER + source + DELIMITER
|
||||||
|
+ (parentQueue != null ? parentQueue + "." + queue : queue);
|
||||||
|
}
|
||||||
|
}
|
|
@ -23,18 +23,15 @@ import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.security.Groups;
|
import org.apache.hadoop.security.Groups;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
|
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||||
|
@ -44,6 +41,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
|
||||||
public class UserGroupMappingPlacementRule extends PlacementRule {
|
public class UserGroupMappingPlacementRule extends PlacementRule {
|
||||||
private static final Logger LOG = LoggerFactory
|
private static final Logger LOG = LoggerFactory
|
||||||
|
@ -60,91 +61,6 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
|
||||||
private Groups groups;
|
private Groups groups;
|
||||||
private CapacitySchedulerQueueManager queueManager;
|
private CapacitySchedulerQueueManager queueManager;
|
||||||
|
|
||||||
@Private
|
|
||||||
public static class QueueMapping {
|
|
||||||
|
|
||||||
public enum MappingType {
|
|
||||||
|
|
||||||
USER("u"), GROUP("g");
|
|
||||||
private final String type;
|
|
||||||
|
|
||||||
private MappingType(String type) {
|
|
||||||
this.type = type;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String toString() {
|
|
||||||
return type;
|
|
||||||
}
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
MappingType type;
|
|
||||||
String source;
|
|
||||||
String queue;
|
|
||||||
String parentQueue;
|
|
||||||
|
|
||||||
public final static String DELIMITER = ":";
|
|
||||||
|
|
||||||
public QueueMapping(MappingType type, String source, String queue) {
|
|
||||||
this.type = type;
|
|
||||||
this.source = source;
|
|
||||||
this.queue = queue;
|
|
||||||
this.parentQueue = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public QueueMapping(MappingType type, String source,
|
|
||||||
String queue, String parentQueue) {
|
|
||||||
this.type = type;
|
|
||||||
this.source = source;
|
|
||||||
this.queue = queue;
|
|
||||||
this.parentQueue = parentQueue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getQueue() {
|
|
||||||
return queue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getParentQueue() {
|
|
||||||
return parentQueue;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean hasParentQueue() {
|
|
||||||
return parentQueue != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public MappingType getType() {
|
|
||||||
return type;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getSource() {
|
|
||||||
return source;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int hashCode() {
|
|
||||||
return super.hashCode();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean equals(Object obj) {
|
|
||||||
if (obj instanceof QueueMapping) {
|
|
||||||
QueueMapping other = (QueueMapping) obj;
|
|
||||||
return (other.type.equals(type) &&
|
|
||||||
other.source.equals(source) &&
|
|
||||||
other.queue.equals(queue));
|
|
||||||
} else {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public String toString() {
|
|
||||||
return type.toString() + DELIMITER + source + DELIMITER +
|
|
||||||
(parentQueue != null ?
|
|
||||||
parentQueue + "." + queue :
|
|
||||||
queue);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public UserGroupMappingPlacementRule(){
|
public UserGroupMappingPlacementRule(){
|
||||||
this(false, null, null);
|
this(false, null, null);
|
||||||
}
|
}
|
||||||
|
@ -179,15 +95,19 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
|
||||||
private ApplicationPlacementContext getPlacementForUser(String user)
|
private ApplicationPlacementContext getPlacementForUser(String user)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
for (QueueMapping mapping : mappings) {
|
for (QueueMapping mapping : mappings) {
|
||||||
if (mapping.type == MappingType.USER) {
|
if (mapping.getType() == MappingType.USER) {
|
||||||
if (mapping.source.equals(CURRENT_USER_MAPPING)) {
|
if (mapping.getSource().equals(CURRENT_USER_MAPPING)) {
|
||||||
if (mapping.getParentQueue() != null
|
if (mapping.getParentQueue() != null
|
||||||
&& mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
|
&& mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
|
||||||
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
||||||
if (this.queueManager
|
if (this.queueManager
|
||||||
.getQueue(groups.getGroups(user).get(0)) != null) {
|
.getQueue(groups.getGroups(user).get(0)) != null) {
|
||||||
QueueMapping queueMapping = new QueueMapping(mapping.getType(),
|
QueueMapping queueMapping =
|
||||||
mapping.getSource(), user, groups.getGroups(user).get(0));
|
QueueMappingBuilder.create()
|
||||||
|
.type(mapping.getType())
|
||||||
|
.source(mapping.getSource()).queue(user)
|
||||||
|
.parentQueue(groups.getGroups(user).get(0))
|
||||||
|
.build();
|
||||||
validateQueueMapping(queueMapping);
|
validateQueueMapping(queueMapping);
|
||||||
return getPlacementContext(queueMapping, user);
|
return getPlacementContext(queueMapping, user);
|
||||||
} else {
|
} else {
|
||||||
|
@ -198,16 +118,21 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
|
||||||
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
||||||
String secondaryGroup = getSecondaryGroup(user);
|
String secondaryGroup = getSecondaryGroup(user);
|
||||||
if (secondaryGroup != null) {
|
if (secondaryGroup != null) {
|
||||||
QueueMapping queueMapping = new QueueMapping(mapping.getType(),
|
QueueMapping queueMapping =
|
||||||
mapping.getSource(), user, secondaryGroup);
|
QueueMappingBuilder.create()
|
||||||
|
.type(mapping.getType())
|
||||||
|
.source(mapping.getSource())
|
||||||
|
.queue(user)
|
||||||
|
.parentQueue(secondaryGroup)
|
||||||
|
.build();
|
||||||
validateQueueMapping(queueMapping);
|
validateQueueMapping(queueMapping);
|
||||||
return getPlacementContext(queueMapping, user);
|
return getPlacementContext(queueMapping, user);
|
||||||
} else {
|
} else {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
} else if (mapping.queue.equals(CURRENT_USER_MAPPING)) {
|
} else if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
||||||
return getPlacementContext(mapping, user);
|
return getPlacementContext(mapping, user);
|
||||||
} else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) {
|
} else if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
|
||||||
if (this.queueManager
|
if (this.queueManager
|
||||||
.getQueue(groups.getGroups(user).get(0)) != null) {
|
.getQueue(groups.getGroups(user).get(0)) != null) {
|
||||||
return getPlacementContext(mapping,
|
return getPlacementContext(mapping,
|
||||||
|
@ -215,7 +140,7 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
|
||||||
} else {
|
} else {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
} else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) {
|
} else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
|
||||||
String secondaryGroup = getSecondaryGroup(user);
|
String secondaryGroup = getSecondaryGroup(user);
|
||||||
if (secondaryGroup != null) {
|
if (secondaryGroup != null) {
|
||||||
return getPlacementContext(mapping, secondaryGroup);
|
return getPlacementContext(mapping, secondaryGroup);
|
||||||
|
@ -226,10 +151,10 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
|
||||||
return getPlacementContext(mapping);
|
return getPlacementContext(mapping);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (user.equals(mapping.source)) {
|
if (user.equals(mapping.getSource())) {
|
||||||
if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) {
|
if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
|
||||||
return getPlacementContext(mapping, groups.getGroups(user).get(0));
|
return getPlacementContext(mapping, groups.getGroups(user).get(0));
|
||||||
} else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) {
|
} else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
|
||||||
String secondaryGroup = getSecondaryGroup(user);
|
String secondaryGroup = getSecondaryGroup(user);
|
||||||
if (secondaryGroup != null) {
|
if (secondaryGroup != null) {
|
||||||
return getPlacementContext(mapping, secondaryGroup);
|
return getPlacementContext(mapping, secondaryGroup);
|
||||||
|
@ -241,10 +166,10 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (mapping.type == MappingType.GROUP) {
|
if (mapping.getType() == MappingType.GROUP) {
|
||||||
for (String userGroups : groups.getGroups(user)) {
|
for (String userGroups : groups.getGroups(user)) {
|
||||||
if (userGroups.equals(mapping.source)) {
|
if (userGroups.equals(mapping.getSource())) {
|
||||||
if (mapping.queue.equals(CURRENT_USER_MAPPING)) {
|
if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
||||||
return getPlacementContext(mapping, user);
|
return getPlacementContext(mapping, user);
|
||||||
}
|
}
|
||||||
return getPlacementContext(mapping);
|
return getPlacementContext(mapping);
|
||||||
|
@ -292,7 +217,7 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
|
||||||
|
|
||||||
private ApplicationPlacementContext getPlacementContext(QueueMapping mapping,
|
private ApplicationPlacementContext getPlacementContext(QueueMapping mapping,
|
||||||
String leafQueueName) {
|
String leafQueueName) {
|
||||||
if (!StringUtils.isEmpty(mapping.parentQueue)) {
|
if (!StringUtils.isEmpty(mapping.getParentQueue())) {
|
||||||
return new ApplicationPlacementContext(leafQueueName,
|
return new ApplicationPlacementContext(leafQueueName,
|
||||||
mapping.getParentQueue());
|
mapping.getParentQueue());
|
||||||
} else{
|
} else{
|
||||||
|
@ -426,16 +351,24 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
|
||||||
&& (queuePath.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
|
&& (queuePath.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
|
||||||
|| queuePath.getParentQueue().equals(SECONDARY_GROUP_MAPPING))) {
|
|| queuePath.getParentQueue().equals(SECONDARY_GROUP_MAPPING))) {
|
||||||
// dynamic parent queue
|
// dynamic parent queue
|
||||||
return new QueueMapping(mapping.getType(), mapping.getSource(),
|
return QueueMappingBuilder.create()
|
||||||
queuePath.getLeafQueue(), queuePath.getParentQueue());
|
.type(mapping.getType())
|
||||||
|
.source(mapping.getSource())
|
||||||
|
.queue(queuePath.getLeafQueue())
|
||||||
|
.parentQueue(queuePath.getParentQueue())
|
||||||
|
.build();
|
||||||
} else if (queuePath.hasParentQueue()) {
|
} else if (queuePath.hasParentQueue()) {
|
||||||
//if parent queue is specified,
|
//if parent queue is specified,
|
||||||
// then it should exist and be an instance of ManagedParentQueue
|
// then it should exist and be an instance of ManagedParentQueue
|
||||||
QueuePlacementRuleUtils.validateQueueMappingUnderParentQueue(
|
QueuePlacementRuleUtils.validateQueueMappingUnderParentQueue(
|
||||||
queueManager.getQueue(queuePath.getParentQueue()),
|
queueManager.getQueue(queuePath.getParentQueue()),
|
||||||
queuePath.getParentQueue(), queuePath.getLeafQueue());
|
queuePath.getParentQueue(), queuePath.getLeafQueue());
|
||||||
return new QueueMapping(mapping.getType(), mapping.getSource(),
|
return QueueMappingBuilder.create()
|
||||||
queuePath.getLeafQueue(), queuePath.getParentQueue());
|
.type(mapping.getType())
|
||||||
|
.source(mapping.getSource())
|
||||||
|
.queue(queuePath.getLeafQueue())
|
||||||
|
.parentQueue(queuePath.getParentQueue())
|
||||||
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
|
|
|
@ -39,8 +39,9 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||||
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.security.AccessType;
|
import org.apache.hadoop.yarn.security.AccessType;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity;
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping;
|
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
|
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AppPriorityACLConfigurationParser.AppPriorityACLKeyType;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AppPriorityACLConfigurationParser.AppPriorityACLKeyType;
|
||||||
|
@ -1119,10 +1120,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
|
||||||
throw new IllegalArgumentException(
|
throw new IllegalArgumentException(
|
||||||
"unknown mapping prefix " + mapping[0]);
|
"unknown mapping prefix " + mapping[0]);
|
||||||
}
|
}
|
||||||
m = new QueueMapping(
|
m = QueueMappingBuilder.create()
|
||||||
mappingType,
|
.type(mappingType)
|
||||||
mapping[1],
|
.source(mapping[1])
|
||||||
mapping[2]);
|
.queue(mapping[2])
|
||||||
|
.build();
|
||||||
} catch (Throwable t) {
|
} catch (Throwable t) {
|
||||||
throw new IllegalArgumentException(
|
throw new IllegalArgumentException(
|
||||||
"Illegal queue mapping " + mappingValue);
|
"Illegal queue mapping " + mappingValue);
|
||||||
|
|
|
@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||||
|
@ -70,11 +72,13 @@ public class TestPlacementManager {
|
||||||
.getQueuePlacementManager();
|
.getQueuePlacementManager();
|
||||||
|
|
||||||
List<PlacementRule> queuePlacementRules = new ArrayList<>();
|
List<PlacementRule> queuePlacementRules = new ArrayList<>();
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping =
|
QueueMapping userQueueMapping = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source(USER1)
|
||||||
USER1,
|
.queue(
|
||||||
getQueueMapping(PARENT_QUEUE, USER1));
|
getQueueMapping(PARENT_QUEUE,
|
||||||
|
USER1))
|
||||||
|
.build();
|
||||||
|
|
||||||
UserGroupMappingPlacementRule ugRule = new UserGroupMappingPlacementRule(
|
UserGroupMappingPlacementRule ugRule = new UserGroupMappingPlacementRule(
|
||||||
false, Arrays.asList(userQueueMapping), null);
|
false, Arrays.asList(userQueueMapping), null);
|
||||||
|
|
|
@ -31,8 +31,9 @@ import org.apache.hadoop.security.NullGroupsMapping;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping;
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.TestUserGroupMappingPlacementRule.QueueMappingTestData.QueueMappingTestDataBuilder;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
|
||||||
|
@ -52,21 +53,16 @@ public class TestUserGroupMappingPlacementRule {
|
||||||
SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
|
SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void verifyQueueMapping(QueueMapping queueMapping, String inputUser,
|
private void verifyQueueMapping(QueueMappingTestData queueMappingTestData)
|
||||||
String expectedQueue) throws YarnException {
|
throws YarnException {
|
||||||
verifyQueueMapping(queueMapping, inputUser,
|
|
||||||
YarnConfiguration.DEFAULT_QUEUE_NAME, expectedQueue, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void verifyQueueMapping(QueueMapping queueMapping, String inputUser,
|
QueueMapping queueMapping = queueMappingTestData.queueMapping;
|
||||||
String inputQueue, String expectedQueue, boolean overwrite) throws YarnException {
|
String inputUser = queueMappingTestData.inputUser;
|
||||||
verifyQueueMapping(queueMapping, inputUser, inputQueue, expectedQueue,
|
String inputQueue = queueMappingTestData.inputQueue;
|
||||||
overwrite, null);
|
String expectedQueue = queueMappingTestData.expectedQueue;
|
||||||
}
|
boolean overwrite = queueMappingTestData.overwrite;
|
||||||
|
String expectedParentQueue = queueMappingTestData.expectedParentQueue;
|
||||||
|
|
||||||
private void verifyQueueMapping(QueueMapping queueMapping, String inputUser,
|
|
||||||
String inputQueue, String expectedQueue, boolean overwrite,
|
|
||||||
String expectedParentQueue) throws YarnException {
|
|
||||||
Groups groups = new Groups(conf);
|
Groups groups = new Groups(conf);
|
||||||
UserGroupMappingPlacementRule rule = new UserGroupMappingPlacementRule(
|
UserGroupMappingPlacementRule rule = new UserGroupMappingPlacementRule(
|
||||||
overwrite, Arrays.asList(queueMapping), groups);
|
overwrite, Arrays.asList(queueMapping), groups);
|
||||||
|
@ -109,16 +105,28 @@ public class TestUserGroupMappingPlacementRule {
|
||||||
@Test
|
@Test
|
||||||
public void testSecondaryGroupMapping() throws YarnException {
|
public void testSecondaryGroupMapping() throws YarnException {
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a",
|
QueueMappingTestDataBuilder.create()
|
||||||
"asubgroup2");
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.USER)
|
||||||
|
.source("%user")
|
||||||
|
.queue("%secondary_group").build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("asubgroup2")
|
||||||
|
.build());
|
||||||
|
|
||||||
// PrimaryGroupMapping.class returns only primary group, no secondary groups
|
// PrimaryGroupMapping.class returns only primary group, no secondary groups
|
||||||
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
|
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
|
||||||
PrimaryGroupMapping.class, GroupMappingServiceProvider.class);
|
PrimaryGroupMapping.class, GroupMappingServiceProvider.class);
|
||||||
|
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a",
|
QueueMappingTestDataBuilder.create()
|
||||||
"default");
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.USER)
|
||||||
|
.source("%user")
|
||||||
|
.queue("%secondary_group")
|
||||||
|
.build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("default")
|
||||||
|
.build());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -127,8 +135,15 @@ public class TestUserGroupMappingPlacementRule {
|
||||||
NullGroupsMapping.class, GroupMappingServiceProvider.class);
|
NullGroupsMapping.class, GroupMappingServiceProvider.class);
|
||||||
try {
|
try {
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a",
|
QueueMappingTestDataBuilder.create()
|
||||||
"default");
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.USER)
|
||||||
|
.source("%user")
|
||||||
|
.queue("%secondary_group")
|
||||||
|
.build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("default")
|
||||||
|
.build());
|
||||||
fail("No Groups for user 'a'");
|
fail("No Groups for user 'a'");
|
||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
// Exception is expected as there are no groups for given user
|
// Exception is expected as there are no groups for given user
|
||||||
|
@ -137,59 +152,253 @@ public class TestUserGroupMappingPlacementRule {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testMapping() throws YarnException {
|
public void testMapping() throws YarnException {
|
||||||
|
|
||||||
verifyQueueMapping(new QueueMapping(MappingType.USER, "a", "q1"), "a", "q1");
|
|
||||||
verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", "q1"),
|
|
||||||
"a", "q1");
|
|
||||||
verifyQueueMapping(new QueueMapping(MappingType.USER, "%user", "q2"), "a",
|
|
||||||
"q2");
|
|
||||||
verifyQueueMapping(new QueueMapping(MappingType.USER, "%user", "%user"),
|
|
||||||
"a", "a");
|
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
new QueueMapping(MappingType.USER, "%user", "%primary_group"), "a",
|
QueueMappingTestDataBuilder.create()
|
||||||
"agroup");
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
// Queue "bgroup" is not configured, hence "default" should be used
|
.type(MappingType.USER)
|
||||||
|
.source("a")
|
||||||
|
.queue("q1")
|
||||||
|
.build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("q1")
|
||||||
|
.build());
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
new QueueMapping(MappingType.USER, "%user", "%primary_group"), "b",
|
QueueMappingTestDataBuilder.create()
|
||||||
"default");
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.GROUP)
|
||||||
|
.source("agroup")
|
||||||
|
.queue("q1")
|
||||||
|
.build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("q1")
|
||||||
|
.build());
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
new QueueMapping(MappingType.USER, "%user", "%user", "%primary_group"),
|
QueueMappingTestDataBuilder.create()
|
||||||
"a", YarnConfiguration.DEFAULT_QUEUE_NAME, "a", false, "agroup");
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.USER)
|
||||||
|
.source("%user")
|
||||||
|
.queue("q2")
|
||||||
|
.build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("q2")
|
||||||
|
.build());
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
new QueueMapping(MappingType.USER, "%user", "%user",
|
QueueMappingTestDataBuilder.create()
|
||||||
"%secondary_group"),
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
"b", YarnConfiguration.DEFAULT_QUEUE_NAME, "b", false, "bsubgroup2");
|
.type(MappingType.USER)
|
||||||
verifyQueueMapping(new QueueMapping(MappingType.GROUP, "asubgroup1", "q1"),
|
.source("%user")
|
||||||
"a", "q1");
|
.queue("%user")
|
||||||
|
.build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("a")
|
||||||
|
.build());
|
||||||
|
verifyQueueMapping(
|
||||||
|
QueueMappingTestDataBuilder.create()
|
||||||
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.USER)
|
||||||
|
.source("%user")
|
||||||
|
.queue("%primary_group")
|
||||||
|
.build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("agroup")
|
||||||
|
.build());
|
||||||
|
verifyQueueMapping(
|
||||||
|
QueueMappingTestDataBuilder.create()
|
||||||
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.USER)
|
||||||
|
.source("%user")
|
||||||
|
.queue("%user")
|
||||||
|
.parentQueue("%primary_group")
|
||||||
|
.build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("a")
|
||||||
|
.expectedParentQueue("agroup")
|
||||||
|
.build());
|
||||||
|
verifyQueueMapping(
|
||||||
|
QueueMappingTestDataBuilder.create()
|
||||||
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.USER)
|
||||||
|
.source("%user")
|
||||||
|
.queue("%user")
|
||||||
|
.parentQueue("%secondary_group")
|
||||||
|
.build())
|
||||||
|
.inputUser("b")
|
||||||
|
.expectedQueue("b")
|
||||||
|
.expectedParentQueue("bsubgroup2")
|
||||||
|
.build());
|
||||||
|
verifyQueueMapping(
|
||||||
|
QueueMappingTestDataBuilder.create()
|
||||||
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.GROUP)
|
||||||
|
.source("asubgroup1")
|
||||||
|
.queue("q1")
|
||||||
|
.build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("q1")
|
||||||
|
.build());
|
||||||
|
|
||||||
// specify overwritten, and see if user specified a queue, and it will be
|
// specify overwritten, and see if user specified a queue, and it will be
|
||||||
// overridden
|
// overridden
|
||||||
verifyQueueMapping(new QueueMapping(MappingType.USER, "user", "q1"), "user",
|
verifyQueueMapping(
|
||||||
"q2", "q1", true);
|
QueueMappingTestDataBuilder.create()
|
||||||
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.USER)
|
||||||
|
.source("user")
|
||||||
|
.queue("q1")
|
||||||
|
.build())
|
||||||
|
.inputUser("user")
|
||||||
|
.inputQueue("q2")
|
||||||
|
.expectedQueue("q1")
|
||||||
|
.overwrite(true)
|
||||||
|
.build());
|
||||||
|
|
||||||
// if overwritten not specified, it should be which user specified
|
// if overwritten not specified, it should be which user specified
|
||||||
verifyQueueMapping(new QueueMapping(MappingType.USER, "user", "q1"), "user",
|
verifyQueueMapping(
|
||||||
"q2", "q2", false);
|
QueueMappingTestDataBuilder.create()
|
||||||
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.USER)
|
||||||
|
.source("user")
|
||||||
|
.queue("q1")
|
||||||
|
.build())
|
||||||
|
.inputUser("user")
|
||||||
|
.inputQueue("q2")
|
||||||
|
.expectedQueue("q2")
|
||||||
|
.build());
|
||||||
|
|
||||||
// if overwritten not specified, it should be which user specified
|
// if overwritten not specified, it should be which user specified
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
new QueueMapping(MappingType.GROUP, "usergroup", "%user", "usergroup"),
|
QueueMappingTestDataBuilder.create()
|
||||||
"user", "default", "user", false);
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.GROUP)
|
||||||
|
.source("usergroup")
|
||||||
|
.queue("%user")
|
||||||
|
.parentQueue("usergroup")
|
||||||
|
.build())
|
||||||
|
.inputUser("user")
|
||||||
|
.inputQueue("default")
|
||||||
|
.expectedQueue("user")
|
||||||
|
.build());
|
||||||
|
|
||||||
// if overwritten not specified, it should be which user specified
|
// if overwritten not specified, it should be which user specified
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
new QueueMapping(MappingType.GROUP, "usergroup", "%user", "usergroup"),
|
QueueMappingTestDataBuilder.create()
|
||||||
"user", "agroup", "user", true);
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.GROUP)
|
||||||
|
.source("usergroup")
|
||||||
|
.queue("%user")
|
||||||
|
.parentQueue("usergroup")
|
||||||
|
.build())
|
||||||
|
.inputUser("user")
|
||||||
|
.inputQueue("agroup")
|
||||||
|
.expectedQueue("user")
|
||||||
|
.overwrite(true)
|
||||||
|
.build());
|
||||||
|
|
||||||
//If user specific queue is enabled for a specified group under a given
|
// If user specific queue is enabled for a specified group under a given
|
||||||
// parent queue
|
// parent queue
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
new QueueMapping(MappingType.GROUP, "agroup", "%user", "parent1"), "a",
|
QueueMappingTestDataBuilder.create()
|
||||||
"a");
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.GROUP)
|
||||||
|
.source("agroup")
|
||||||
|
.queue("%user")
|
||||||
|
.parentQueue("parent1")
|
||||||
|
.build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("a")
|
||||||
|
.build());
|
||||||
|
|
||||||
//If user specific queue is enabled for a specified group without parent
|
// If user specific queue is enabled for a specified group without parent
|
||||||
// queue
|
// queue
|
||||||
verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", "%user"),
|
verifyQueueMapping(
|
||||||
"a", "a");
|
QueueMappingTestDataBuilder.create()
|
||||||
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.GROUP)
|
||||||
|
.source("agroup")
|
||||||
|
.queue("%user")
|
||||||
|
.build())
|
||||||
|
.inputUser("a")
|
||||||
|
.expectedQueue("a")
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Queue Mapping test class to prepare the test data.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public static final class QueueMappingTestData {
|
||||||
|
|
||||||
|
private QueueMapping queueMapping;
|
||||||
|
private String inputUser;
|
||||||
|
private String inputQueue;
|
||||||
|
private String expectedQueue;
|
||||||
|
private boolean overwrite;
|
||||||
|
private String expectedParentQueue;
|
||||||
|
|
||||||
|
private QueueMappingTestData(QueueMappingTestDataBuilder builder) {
|
||||||
|
this.queueMapping = builder.queueMapping;
|
||||||
|
this.inputUser = builder.inputUser;
|
||||||
|
this.inputQueue = builder.inputQueue;
|
||||||
|
this.expectedQueue = builder.expectedQueue;
|
||||||
|
this.overwrite = builder.overwrite;
|
||||||
|
this.expectedParentQueue = builder.expectedParentQueue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Builder class to prepare the Queue Mapping test data.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public static class QueueMappingTestDataBuilder {
|
||||||
|
|
||||||
|
private QueueMapping queueMapping = null;
|
||||||
|
private String inputUser = null;
|
||||||
|
private String inputQueue = YarnConfiguration.DEFAULT_QUEUE_NAME;
|
||||||
|
private String expectedQueue = null;
|
||||||
|
private boolean overwrite = false;
|
||||||
|
private String expectedParentQueue = null;
|
||||||
|
|
||||||
|
public QueueMappingTestDataBuilder() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public static QueueMappingTestDataBuilder create() {
|
||||||
|
return new QueueMappingTestDataBuilder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMappingTestDataBuilder queueMapping(QueueMapping mapping) {
|
||||||
|
this.queueMapping = mapping;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMappingTestDataBuilder inputUser(String user) {
|
||||||
|
this.inputUser = user;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMappingTestDataBuilder inputQueue(String queue) {
|
||||||
|
this.inputQueue = queue;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMappingTestDataBuilder expectedQueue(String outputQueue) {
|
||||||
|
this.expectedQueue = outputQueue;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMappingTestDataBuilder overwrite(boolean overwriteMappings) {
|
||||||
|
this.overwrite = overwriteMappings;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMappingTestDataBuilder expectedParentQueue(
|
||||||
|
String outputParentQueue) {
|
||||||
|
this.expectedParentQueue = outputParentQueue;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public QueueMappingTestData build() {
|
||||||
|
return new QueueMappingTestData(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -46,8 +46,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement
|
import org.apache.hadoop.yarn.server.resourcemanager.placement
|
||||||
.ApplicationPlacementContext;
|
.ApplicationPlacementContext;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
|
||||||
.UserGroupMappingPlacementRule;
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
|
||||||
|
@ -268,19 +268,19 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
|
||||||
queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
|
queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
|
||||||
conf.setQueuePlacementRules(queuePlacementRules);
|
conf.setQueuePlacementRules(queuePlacementRules);
|
||||||
|
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> existingMappings =
|
List<QueueMapping> existingMappings = conf.getQueueMappings();
|
||||||
conf.getQueueMappings();
|
|
||||||
|
|
||||||
//set queue mapping
|
//set queue mapping
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> queueMappings =
|
List<QueueMapping> queueMappings = new ArrayList<>();
|
||||||
new ArrayList<>();
|
|
||||||
for (int i = 0; i < userIds.length; i++) {
|
for (int i = 0; i < userIds.length; i++) {
|
||||||
//Set C as parent queue name for auto queue creation
|
//Set C as parent queue name for auto queue creation
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping =
|
QueueMapping userQueueMapping = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source(USER + userIds[i])
|
||||||
USER + userIds[i],
|
.queue(
|
||||||
getQueueMapping(parentQueue, USER + userIds[i]));
|
getQueueMapping(parentQueue,
|
||||||
|
USER + userIds[i]))
|
||||||
|
.build();
|
||||||
queueMappings.add(userQueueMapping);
|
queueMappings.add(userQueueMapping);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -295,12 +295,10 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
|
||||||
(String parentQueue, CapacitySchedulerConfiguration conf, String
|
(String parentQueue, CapacitySchedulerConfiguration conf, String
|
||||||
leafQueueName) {
|
leafQueueName) {
|
||||||
|
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> existingMappings =
|
List<QueueMapping> existingMappings = conf.getQueueMappings();
|
||||||
conf.getQueueMappings();
|
|
||||||
|
|
||||||
//set queue mapping
|
//set queue mapping
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> queueMappings =
|
List<QueueMapping> queueMappings = new ArrayList<>();
|
||||||
new ArrayList<>();
|
|
||||||
|
|
||||||
//setup group mapping
|
//setup group mapping
|
||||||
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
|
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
|
||||||
|
@ -309,11 +307,13 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
|
||||||
TEST_GROUPUSER +"=" + TEST_GROUP + ";invalid_user=invalid_group");
|
TEST_GROUPUSER +"=" + TEST_GROUP + ";invalid_user=invalid_group");
|
||||||
Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf);
|
Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf);
|
||||||
|
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping =
|
QueueMapping userQueueMapping = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.GROUP)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.GROUP,
|
.source(TEST_GROUP)
|
||||||
TEST_GROUP,
|
.queue(
|
||||||
getQueueMapping(parentQueue, leafQueueName));
|
getQueueMapping(parentQueue,
|
||||||
|
leafQueueName))
|
||||||
|
.build();
|
||||||
|
|
||||||
queueMappings.add(userQueueMapping);
|
queueMappings.add(userQueueMapping);
|
||||||
existingMappings.addAll(queueMappings);
|
existingMappings.addAll(queueMappings);
|
||||||
|
@ -514,13 +514,14 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
|
||||||
return rmApp.getApplicationId();
|
return rmApp.getApplicationId();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected List<UserGroupMappingPlacementRule.QueueMapping> setupQueueMapping(
|
protected List<QueueMapping> setupQueueMapping(
|
||||||
CapacityScheduler newCS, String user, String parentQueue, String queue) {
|
CapacityScheduler newCS, String user, String parentQueue, String queue) {
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> queueMappings =
|
List<QueueMapping> queueMappings = new ArrayList<>();
|
||||||
new ArrayList<>();
|
queueMappings.add(QueueMappingBuilder.create()
|
||||||
queueMappings.add(new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, user,
|
.source(user)
|
||||||
getQueueMapping(parentQueue, queue)));
|
.queue(getQueueMapping(parentQueue, queue))
|
||||||
|
.build());
|
||||||
newCS.getConfiguration().setQueueMappings(queueMappings);
|
newCS.getConfiguration().setQueueMappings(queueMappings);
|
||||||
return queueMappings;
|
return queueMappings;
|
||||||
}
|
}
|
||||||
|
|
|
@ -26,6 +26,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity;
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||||
|
@ -62,19 +65,19 @@ public class TestCapacitySchedulerQueueMappingFactory {
|
||||||
|
|
||||||
conf.setQueuePlacementRules(queuePlacementRules);
|
conf.setQueuePlacementRules(queuePlacementRules);
|
||||||
|
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> existingMappingsForUG =
|
List<QueueMapping> existingMappingsForUG = conf.getQueueMappings();
|
||||||
conf.getQueueMappings();
|
|
||||||
|
|
||||||
//set queue mapping
|
//set queue mapping
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG =
|
List<QueueMapping> queueMappingsForUG = new ArrayList<>();
|
||||||
new ArrayList<>();
|
|
||||||
for (int i = 0; i < sourceIds.length; i++) {
|
for (int i = 0; i < sourceIds.length; i++) {
|
||||||
//Set C as parent queue name for auto queue creation
|
//Set C as parent queue name for auto queue creation
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping =
|
QueueMapping userQueueMapping = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source(USER + sourceIds[i])
|
||||||
USER + sourceIds[i],
|
.queue(
|
||||||
getQueueMapping(parentQueue, USER + sourceIds[i]));
|
getQueueMapping(parentQueue,
|
||||||
|
USER + sourceIds[i]))
|
||||||
|
.build();
|
||||||
queueMappingsForUG.add(userQueueMapping);
|
queueMappingsForUG.add(userQueueMapping);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -152,23 +155,25 @@ public class TestCapacitySchedulerQueueMappingFactory {
|
||||||
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
|
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
|
||||||
conf.setQueuePlacementRules(queuePlacementRules);
|
conf.setQueuePlacementRules(queuePlacementRules);
|
||||||
|
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> existingMappingsForUG =
|
List<QueueMapping> existingMappingsForUG = conf.getQueueMappings();
|
||||||
conf.getQueueMappings();
|
|
||||||
|
|
||||||
// set queue mapping
|
// set queue mapping
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG =
|
List<QueueMapping> queueMappingsForUG = new ArrayList<>();
|
||||||
new ArrayList<>();
|
|
||||||
|
|
||||||
// u:user1:b1
|
// u:user1:b1
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 =
|
QueueMapping userQueueMapping1 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "user1",
|
.source("user1")
|
||||||
"b1");
|
.queue("b1")
|
||||||
|
.build();
|
||||||
|
|
||||||
// u:%user:parentqueue.%user
|
// u:%user:parentqueue.%user
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 =
|
QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "%user",
|
.source("%user")
|
||||||
getQueueMapping("c", "%user"));
|
.queue(getQueueMapping("c", "%user"))
|
||||||
|
.build();
|
||||||
|
|
||||||
queueMappingsForUG.add(userQueueMapping1);
|
queueMappingsForUG.add(userQueueMapping1);
|
||||||
queueMappingsForUG.add(userQueueMapping2);
|
queueMappingsForUG.add(userQueueMapping2);
|
||||||
|
|
||||||
|
@ -221,26 +226,32 @@ public class TestCapacitySchedulerQueueMappingFactory {
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// set queue mapping
|
// set queue mapping
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG =
|
List<QueueMapping> queueMappingsForUG = new ArrayList<>();
|
||||||
new ArrayList<>();
|
|
||||||
|
|
||||||
// u:%user:%primary_group.%user
|
// u:%user:%primary_group.%user
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 =
|
QueueMapping userQueueMapping1 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source("%user")
|
||||||
"%user", getQueueMapping("%primary_group", "%user"));
|
.queue(
|
||||||
|
getQueueMapping("%primary_group",
|
||||||
|
"%user"))
|
||||||
|
.build();
|
||||||
|
|
||||||
// u:%user:%secondary_group.%user
|
// u:%user:%secondary_group.%user
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 =
|
QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source("%user")
|
||||||
"%user", getQueueMapping("%secondary_group", "%user"));
|
.queue(
|
||||||
|
getQueueMapping("%secondary_group",
|
||||||
|
"%user"))
|
||||||
|
.build();
|
||||||
|
|
||||||
// u:b4:%secondary_group
|
// u:b4:%secondary_group
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 =
|
QueueMapping userQueueMapping3 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source("b4")
|
||||||
"b4", "%secondary_group");
|
.queue("%secondary_group")
|
||||||
|
.build();
|
||||||
queueMappingsForUG.add(userQueueMapping1);
|
queueMappingsForUG.add(userQueueMapping1);
|
||||||
queueMappingsForUG.add(userQueueMapping2);
|
queueMappingsForUG.add(userQueueMapping2);
|
||||||
queueMappingsForUG.add(userQueueMapping3);
|
queueMappingsForUG.add(userQueueMapping3);
|
||||||
|
@ -276,20 +287,26 @@ public class TestCapacitySchedulerQueueMappingFactory {
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// set queue mapping
|
// set queue mapping
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG =
|
List<QueueMapping> queueMappingsForUG = new ArrayList<>();
|
||||||
new ArrayList<>();
|
|
||||||
|
|
||||||
// u:%user:%primary_group.%user
|
// u:%user:%primary_group.%user
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 =
|
QueueMapping userQueueMapping1 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source("%user")
|
||||||
"%user", getQueueMapping("%primary_group", "%user"));
|
.queue(
|
||||||
|
getQueueMapping("%primary_group",
|
||||||
|
"%user"))
|
||||||
|
.build();
|
||||||
|
|
||||||
// u:%user:%secondary_group.%user
|
// u:%user:%secondary_group.%user
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 =
|
QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source("%user")
|
||||||
"%user", getQueueMapping("%secondary_group", "%user"));
|
.queue(
|
||||||
|
getQueueMapping(
|
||||||
|
"%secondary_group", "%user")
|
||||||
|
)
|
||||||
|
.build();
|
||||||
|
|
||||||
queueMappingsForUG.add(userQueueMapping2);
|
queueMappingsForUG.add(userQueueMapping2);
|
||||||
queueMappingsForUG.add(userQueueMapping1);
|
queueMappingsForUG.add(userQueueMapping1);
|
||||||
|
@ -298,8 +315,7 @@ public class TestCapacitySchedulerQueueMappingFactory {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void testNestedUserQueueWithDynamicParentQueue(
|
private void testNestedUserQueueWithDynamicParentQueue(
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> mapping, boolean primary,
|
List<QueueMapping> mapping, boolean primary, String user)
|
||||||
String user)
|
|
||||||
throws Exception {
|
throws Exception {
|
||||||
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
|
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
|
||||||
setupQueueConfiguration(conf);
|
setupQueueConfiguration(conf);
|
||||||
|
@ -312,8 +328,7 @@ public class TestCapacitySchedulerQueueMappingFactory {
|
||||||
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
|
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
|
||||||
conf.setQueuePlacementRules(queuePlacementRules);
|
conf.setQueuePlacementRules(queuePlacementRules);
|
||||||
|
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> existingMappingsForUG =
|
List<QueueMapping> existingMappingsForUG = conf.getQueueMappings();
|
||||||
conf.getQueueMappings();
|
|
||||||
|
|
||||||
existingMappingsForUG.addAll(mapping);
|
existingMappingsForUG.addAll(mapping);
|
||||||
conf.setQueueMappings(existingMappingsForUG);
|
conf.setQueueMappings(existingMappingsForUG);
|
||||||
|
@ -367,24 +382,24 @@ public class TestCapacitySchedulerQueueMappingFactory {
|
||||||
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
|
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
|
||||||
conf.setQueuePlacementRules(queuePlacementRules);
|
conf.setQueuePlacementRules(queuePlacementRules);
|
||||||
|
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> existingMappingsForUG =
|
List<QueueMapping> existingMappingsForUG = conf.getQueueMappings();
|
||||||
conf.getQueueMappings();
|
|
||||||
|
|
||||||
// set queue mapping
|
// set queue mapping
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG =
|
List<QueueMapping> queueMappingsForUG = new ArrayList<>();
|
||||||
new ArrayList<>();
|
|
||||||
|
|
||||||
// u:user1:b1
|
// u:user1:b1
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 =
|
QueueMapping userQueueMapping1 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source("user1")
|
||||||
"user1", "b1");
|
.queue("b1")
|
||||||
|
.build();
|
||||||
|
|
||||||
// u:user2:%primary_group
|
// u:user2:%primary_group
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 =
|
QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source("user2")
|
||||||
"user2", "%primary_group");
|
.queue("%primary_group")
|
||||||
|
.build();
|
||||||
|
|
||||||
queueMappingsForUG.add(userQueueMapping1);
|
queueMappingsForUG.add(userQueueMapping1);
|
||||||
queueMappingsForUG.add(userQueueMapping2);
|
queueMappingsForUG.add(userQueueMapping2);
|
||||||
|
@ -436,30 +451,31 @@ public class TestCapacitySchedulerQueueMappingFactory {
|
||||||
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
|
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
|
||||||
conf.setQueuePlacementRules(queuePlacementRules);
|
conf.setQueuePlacementRules(queuePlacementRules);
|
||||||
|
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> existingMappingsForUG =
|
List<QueueMapping> existingMappingsForUG = conf.getQueueMappings();
|
||||||
conf.getQueueMappings();
|
|
||||||
|
|
||||||
// set queue mapping
|
// set queue mapping
|
||||||
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG =
|
List<QueueMapping> queueMappingsForUG = new ArrayList<>();
|
||||||
new ArrayList<>();
|
|
||||||
|
|
||||||
// u:user1:b1
|
// u:user1:b1
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 =
|
QueueMapping userQueueMapping1 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source("user1")
|
||||||
"user1", "b1");
|
.queue("b1")
|
||||||
|
.build();
|
||||||
|
|
||||||
// u:user2:%primary_group
|
// u:user2:%primary_group
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 =
|
QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
|
.source("user2")
|
||||||
"user2", "%primary_group");
|
.queue("%primary_group")
|
||||||
|
.build();
|
||||||
|
|
||||||
// u:b4:%secondary_group
|
// u:b4:%secondary_group
|
||||||
UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 =
|
QueueMapping userQueueMapping3 = QueueMappingBuilder.create()
|
||||||
new UserGroupMappingPlacementRule.QueueMapping(
|
.type(QueueMapping.MappingType.USER)
|
||||||
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "b4",
|
.source("b4")
|
||||||
"%secondary_group");
|
.queue("%secondary_group")
|
||||||
|
.build();
|
||||||
|
|
||||||
queueMappingsForUG.add(userQueueMapping1);
|
queueMappingsForUG.add(userQueueMapping1);
|
||||||
queueMappingsForUG.add(userQueueMapping2);
|
queueMappingsForUG.add(userQueueMapping2);
|
||||||
|
|
|
@ -23,9 +23,10 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping;
|
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
|
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
@ -91,7 +92,12 @@ public class TestQueueMappings {
|
||||||
// space trimming
|
// space trimming
|
||||||
conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, " u : a : " + Q1);
|
conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, " u : a : " + Q1);
|
||||||
cs.reinitialize(conf, null);
|
cs.reinitialize(conf, null);
|
||||||
checkQMapping(new QueueMapping(MappingType.USER, "a", Q1));
|
checkQMapping(
|
||||||
|
QueueMappingBuilder.create()
|
||||||
|
.type(MappingType.USER)
|
||||||
|
.source("a")
|
||||||
|
.queue(Q1)
|
||||||
|
.build());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test (timeout = 60000)
|
@Test (timeout = 60000)
|
||||||
|
|
Loading…
Reference in New Issue