YARN-9970. Refactor TestUserGroupMappingPlacementRule#verifyQueueMapping. Contributed by Manikandan R

This commit is contained in:
Szilard Nemeth 2020-01-15 09:55:41 +01:00
parent a2fdd7c2b5
commit 7c5cecc3b3
8 changed files with 645 additions and 291 deletions

View File

@ -0,0 +1,183 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.placement;
import org.apache.hadoop.classification.InterfaceAudience.Private;
/**
* Queue Mapping class to hold the queue mapping information.
*
*/
@Private
public class QueueMapping {
/**
* Builder class for QueueMapping.
*
*/
public static class QueueMappingBuilder {
private MappingType type;
private String source;
private String queue;
private String parentQueue;
public QueueMappingBuilder() {
}
public static QueueMappingBuilder create() {
return new QueueMappingBuilder();
}
public QueueMappingBuilder type(MappingType mappingType) {
this.type = mappingType;
return this;
}
public QueueMappingBuilder source(String mappingSource) {
this.source = mappingSource;
return this;
}
public QueueMappingBuilder queue(String mappingQueue) {
this.queue = mappingQueue;
return this;
}
public QueueMappingBuilder parentQueue(String mappingParentQueue) {
this.parentQueue = mappingParentQueue;
return this;
}
public QueueMapping build() {
return new QueueMapping(this);
}
}
private QueueMapping(QueueMappingBuilder builder) {
this.type = builder.type;
this.source = builder.source;
this.queue = builder.queue;
this.parentQueue = builder.parentQueue;
}
/**
* Different types of mapping.
*
*/
public enum MappingType {
USER("u"), GROUP("g");
private final String type;
MappingType(String type) {
this.type = type;
}
public String toString() {
return type;
}
};
private MappingType type;
private String source;
private String queue;
private String parentQueue;
private final static String DELIMITER = ":";
public String getQueue() {
return queue;
}
public String getParentQueue() {
return parentQueue;
}
public boolean hasParentQueue() {
return parentQueue != null;
}
public MappingType getType() {
return type;
}
public String getSource() {
return source;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result =
prime * result + ((parentQueue == null) ? 0 : parentQueue.hashCode());
result = prime * result + ((queue == null) ? 0 : queue.hashCode());
result = prime * result + ((source == null) ? 0 : source.hashCode());
result = prime * result + ((type == null) ? 0 : type.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
QueueMapping other = (QueueMapping) obj;
if (parentQueue == null) {
if (other.parentQueue != null) {
return false;
}
} else if (!parentQueue.equals(other.parentQueue)) {
return false;
}
if (queue == null) {
if (other.queue != null) {
return false;
}
} else if (!queue.equals(other.queue)) {
return false;
}
if (source == null) {
if (other.source != null) {
return false;
}
} else if (!source.equals(other.source)) {
return false;
}
if (type != other.type) {
return false;
}
return true;
}
@Override
public String toString() {
return type.toString() + DELIMITER + source + DELIMITER
+ (parentQueue != null ? parentQueue + "." + queue : queue);
}
}

View File

@ -23,18 +23,15 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.Groups;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@ -44,6 +41,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
public class UserGroupMappingPlacementRule extends PlacementRule { public class UserGroupMappingPlacementRule extends PlacementRule {
private static final Logger LOG = LoggerFactory private static final Logger LOG = LoggerFactory
@ -60,91 +61,6 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
private Groups groups; private Groups groups;
private CapacitySchedulerQueueManager queueManager; private CapacitySchedulerQueueManager queueManager;
@Private
public static class QueueMapping {
public enum MappingType {
USER("u"), GROUP("g");
private final String type;
private MappingType(String type) {
this.type = type;
}
public String toString() {
return type;
}
};
MappingType type;
String source;
String queue;
String parentQueue;
public final static String DELIMITER = ":";
public QueueMapping(MappingType type, String source, String queue) {
this.type = type;
this.source = source;
this.queue = queue;
this.parentQueue = null;
}
public QueueMapping(MappingType type, String source,
String queue, String parentQueue) {
this.type = type;
this.source = source;
this.queue = queue;
this.parentQueue = parentQueue;
}
public String getQueue() {
return queue;
}
public String getParentQueue() {
return parentQueue;
}
public boolean hasParentQueue() {
return parentQueue != null;
}
public MappingType getType() {
return type;
}
public String getSource() {
return source;
}
@Override
public int hashCode() {
return super.hashCode();
}
@Override
public boolean equals(Object obj) {
if (obj instanceof QueueMapping) {
QueueMapping other = (QueueMapping) obj;
return (other.type.equals(type) &&
other.source.equals(source) &&
other.queue.equals(queue));
} else {
return false;
}
}
public String toString() {
return type.toString() + DELIMITER + source + DELIMITER +
(parentQueue != null ?
parentQueue + "." + queue :
queue);
}
}
public UserGroupMappingPlacementRule(){ public UserGroupMappingPlacementRule(){
this(false, null, null); this(false, null, null);
} }
@ -179,15 +95,19 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
private ApplicationPlacementContext getPlacementForUser(String user) private ApplicationPlacementContext getPlacementForUser(String user)
throws IOException { throws IOException {
for (QueueMapping mapping : mappings) { for (QueueMapping mapping : mappings) {
if (mapping.type == MappingType.USER) { if (mapping.getType() == MappingType.USER) {
if (mapping.source.equals(CURRENT_USER_MAPPING)) { if (mapping.getSource().equals(CURRENT_USER_MAPPING)) {
if (mapping.getParentQueue() != null if (mapping.getParentQueue() != null
&& mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING) && mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) { && mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
if (this.queueManager if (this.queueManager
.getQueue(groups.getGroups(user).get(0)) != null) { .getQueue(groups.getGroups(user).get(0)) != null) {
QueueMapping queueMapping = new QueueMapping(mapping.getType(), QueueMapping queueMapping =
mapping.getSource(), user, groups.getGroups(user).get(0)); QueueMappingBuilder.create()
.type(mapping.getType())
.source(mapping.getSource()).queue(user)
.parentQueue(groups.getGroups(user).get(0))
.build();
validateQueueMapping(queueMapping); validateQueueMapping(queueMapping);
return getPlacementContext(queueMapping, user); return getPlacementContext(queueMapping, user);
} else { } else {
@ -198,16 +118,21 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) { && mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
String secondaryGroup = getSecondaryGroup(user); String secondaryGroup = getSecondaryGroup(user);
if (secondaryGroup != null) { if (secondaryGroup != null) {
QueueMapping queueMapping = new QueueMapping(mapping.getType(), QueueMapping queueMapping =
mapping.getSource(), user, secondaryGroup); QueueMappingBuilder.create()
.type(mapping.getType())
.source(mapping.getSource())
.queue(user)
.parentQueue(secondaryGroup)
.build();
validateQueueMapping(queueMapping); validateQueueMapping(queueMapping);
return getPlacementContext(queueMapping, user); return getPlacementContext(queueMapping, user);
} else { } else {
return null; return null;
} }
} else if (mapping.queue.equals(CURRENT_USER_MAPPING)) { } else if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
return getPlacementContext(mapping, user); return getPlacementContext(mapping, user);
} else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) { } else if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
if (this.queueManager if (this.queueManager
.getQueue(groups.getGroups(user).get(0)) != null) { .getQueue(groups.getGroups(user).get(0)) != null) {
return getPlacementContext(mapping, return getPlacementContext(mapping,
@ -215,7 +140,7 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
} else { } else {
return null; return null;
} }
} else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) { } else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
String secondaryGroup = getSecondaryGroup(user); String secondaryGroup = getSecondaryGroup(user);
if (secondaryGroup != null) { if (secondaryGroup != null) {
return getPlacementContext(mapping, secondaryGroup); return getPlacementContext(mapping, secondaryGroup);
@ -226,10 +151,10 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
return getPlacementContext(mapping); return getPlacementContext(mapping);
} }
} }
if (user.equals(mapping.source)) { if (user.equals(mapping.getSource())) {
if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) { if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
return getPlacementContext(mapping, groups.getGroups(user).get(0)); return getPlacementContext(mapping, groups.getGroups(user).get(0));
} else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) { } else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
String secondaryGroup = getSecondaryGroup(user); String secondaryGroup = getSecondaryGroup(user);
if (secondaryGroup != null) { if (secondaryGroup != null) {
return getPlacementContext(mapping, secondaryGroup); return getPlacementContext(mapping, secondaryGroup);
@ -241,10 +166,10 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
} }
} }
} }
if (mapping.type == MappingType.GROUP) { if (mapping.getType() == MappingType.GROUP) {
for (String userGroups : groups.getGroups(user)) { for (String userGroups : groups.getGroups(user)) {
if (userGroups.equals(mapping.source)) { if (userGroups.equals(mapping.getSource())) {
if (mapping.queue.equals(CURRENT_USER_MAPPING)) { if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
return getPlacementContext(mapping, user); return getPlacementContext(mapping, user);
} }
return getPlacementContext(mapping); return getPlacementContext(mapping);
@ -292,7 +217,7 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
private ApplicationPlacementContext getPlacementContext(QueueMapping mapping, private ApplicationPlacementContext getPlacementContext(QueueMapping mapping,
String leafQueueName) { String leafQueueName) {
if (!StringUtils.isEmpty(mapping.parentQueue)) { if (!StringUtils.isEmpty(mapping.getParentQueue())) {
return new ApplicationPlacementContext(leafQueueName, return new ApplicationPlacementContext(leafQueueName,
mapping.getParentQueue()); mapping.getParentQueue());
} else{ } else{
@ -426,16 +351,24 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
&& (queuePath.getParentQueue().equals(PRIMARY_GROUP_MAPPING) && (queuePath.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
|| queuePath.getParentQueue().equals(SECONDARY_GROUP_MAPPING))) { || queuePath.getParentQueue().equals(SECONDARY_GROUP_MAPPING))) {
// dynamic parent queue // dynamic parent queue
return new QueueMapping(mapping.getType(), mapping.getSource(), return QueueMappingBuilder.create()
queuePath.getLeafQueue(), queuePath.getParentQueue()); .type(mapping.getType())
.source(mapping.getSource())
.queue(queuePath.getLeafQueue())
.parentQueue(queuePath.getParentQueue())
.build();
} else if (queuePath.hasParentQueue()) { } else if (queuePath.hasParentQueue()) {
//if parent queue is specified, //if parent queue is specified,
// then it should exist and be an instance of ManagedParentQueue // then it should exist and be an instance of ManagedParentQueue
QueuePlacementRuleUtils.validateQueueMappingUnderParentQueue( QueuePlacementRuleUtils.validateQueueMappingUnderParentQueue(
queueManager.getQueue(queuePath.getParentQueue()), queueManager.getQueue(queuePath.getParentQueue()),
queuePath.getParentQueue(), queuePath.getLeafQueue()); queuePath.getParentQueue(), queuePath.getLeafQueue());
return new QueueMapping(mapping.getType(), mapping.getSource(), return QueueMappingBuilder.create()
queuePath.getLeafQueue(), queuePath.getParentQueue()); .type(mapping.getType())
.source(mapping.getSource())
.queue(queuePath.getLeafQueue())
.parentQueue(queuePath.getParentQueue())
.build();
} }
return null; return null;

View File

@ -39,8 +39,9 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AppPriorityACLConfigurationParser.AppPriorityACLKeyType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AppPriorityACLConfigurationParser.AppPriorityACLKeyType;
@ -1119,10 +1120,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
throw new IllegalArgumentException( throw new IllegalArgumentException(
"unknown mapping prefix " + mapping[0]); "unknown mapping prefix " + mapping[0]);
} }
m = new QueueMapping( m = QueueMappingBuilder.create()
mappingType, .type(mappingType)
mapping[1], .source(mapping[1])
mapping[2]); .queue(mapping[2])
.build();
} catch (Throwable t) { } catch (Throwable t) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"Illegal queue mapping " + mappingValue); "Illegal queue mapping " + mappingValue);

View File

@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@ -70,11 +72,13 @@ public class TestPlacementManager {
.getQueuePlacementManager(); .getQueuePlacementManager();
List<PlacementRule> queuePlacementRules = new ArrayList<>(); List<PlacementRule> queuePlacementRules = new ArrayList<>();
UserGroupMappingPlacementRule.QueueMapping userQueueMapping = QueueMapping userQueueMapping = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source(USER1)
USER1, .queue(
getQueueMapping(PARENT_QUEUE, USER1)); getQueueMapping(PARENT_QUEUE,
USER1))
.build();
UserGroupMappingPlacementRule ugRule = new UserGroupMappingPlacementRule( UserGroupMappingPlacementRule ugRule = new UserGroupMappingPlacementRule(
false, Arrays.asList(userQueueMapping), null); false, Arrays.asList(userQueueMapping), null);

View File

@ -31,8 +31,9 @@ import org.apache.hadoop.security.NullGroupsMapping;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.placement.TestUserGroupMappingPlacementRule.QueueMappingTestData.QueueMappingTestDataBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
@ -52,21 +53,16 @@ public class TestUserGroupMappingPlacementRule {
SimpleGroupsMapping.class, GroupMappingServiceProvider.class); SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
} }
private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, private void verifyQueueMapping(QueueMappingTestData queueMappingTestData)
String expectedQueue) throws YarnException { throws YarnException {
verifyQueueMapping(queueMapping, inputUser,
YarnConfiguration.DEFAULT_QUEUE_NAME, expectedQueue, false);
}
private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, QueueMapping queueMapping = queueMappingTestData.queueMapping;
String inputQueue, String expectedQueue, boolean overwrite) throws YarnException { String inputUser = queueMappingTestData.inputUser;
verifyQueueMapping(queueMapping, inputUser, inputQueue, expectedQueue, String inputQueue = queueMappingTestData.inputQueue;
overwrite, null); String expectedQueue = queueMappingTestData.expectedQueue;
} boolean overwrite = queueMappingTestData.overwrite;
String expectedParentQueue = queueMappingTestData.expectedParentQueue;
private void verifyQueueMapping(QueueMapping queueMapping, String inputUser,
String inputQueue, String expectedQueue, boolean overwrite,
String expectedParentQueue) throws YarnException {
Groups groups = new Groups(conf); Groups groups = new Groups(conf);
UserGroupMappingPlacementRule rule = new UserGroupMappingPlacementRule( UserGroupMappingPlacementRule rule = new UserGroupMappingPlacementRule(
overwrite, Arrays.asList(queueMapping), groups); overwrite, Arrays.asList(queueMapping), groups);
@ -109,16 +105,28 @@ public class TestUserGroupMappingPlacementRule {
@Test @Test
public void testSecondaryGroupMapping() throws YarnException { public void testSecondaryGroupMapping() throws YarnException {
verifyQueueMapping( verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a", QueueMappingTestDataBuilder.create()
"asubgroup2"); .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%secondary_group").build())
.inputUser("a")
.expectedQueue("asubgroup2")
.build());
// PrimaryGroupMapping.class returns only primary group, no secondary groups // PrimaryGroupMapping.class returns only primary group, no secondary groups
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
PrimaryGroupMapping.class, GroupMappingServiceProvider.class); PrimaryGroupMapping.class, GroupMappingServiceProvider.class);
verifyQueueMapping( verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a", QueueMappingTestDataBuilder.create()
"default"); .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%secondary_group")
.build())
.inputUser("a")
.expectedQueue("default")
.build());
} }
@Test @Test
@ -127,8 +135,15 @@ public class TestUserGroupMappingPlacementRule {
NullGroupsMapping.class, GroupMappingServiceProvider.class); NullGroupsMapping.class, GroupMappingServiceProvider.class);
try { try {
verifyQueueMapping( verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a", QueueMappingTestDataBuilder.create()
"default"); .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%secondary_group")
.build())
.inputUser("a")
.expectedQueue("default")
.build());
fail("No Groups for user 'a'"); fail("No Groups for user 'a'");
} catch (YarnException e) { } catch (YarnException e) {
// Exception is expected as there are no groups for given user // Exception is expected as there are no groups for given user
@ -137,59 +152,253 @@ public class TestUserGroupMappingPlacementRule {
@Test @Test
public void testMapping() throws YarnException { public void testMapping() throws YarnException {
verifyQueueMapping(new QueueMapping(MappingType.USER, "a", "q1"), "a", "q1");
verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", "q1"),
"a", "q1");
verifyQueueMapping(new QueueMapping(MappingType.USER, "%user", "q2"), "a",
"q2");
verifyQueueMapping(new QueueMapping(MappingType.USER, "%user", "%user"),
"a", "a");
verifyQueueMapping( verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%primary_group"), "a", QueueMappingTestDataBuilder.create()
"agroup"); .queueMapping(QueueMappingBuilder.create()
// Queue "bgroup" is not configured, hence "default" should be used .type(MappingType.USER)
.source("a")
.queue("q1")
.build())
.inputUser("a")
.expectedQueue("q1")
.build());
verifyQueueMapping( verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%primary_group"), "b", QueueMappingTestDataBuilder.create()
"default"); .queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP)
.source("agroup")
.queue("q1")
.build())
.inputUser("a")
.expectedQueue("q1")
.build());
verifyQueueMapping( verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%user", "%primary_group"), QueueMappingTestDataBuilder.create()
"a", YarnConfiguration.DEFAULT_QUEUE_NAME, "a", false, "agroup"); .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("q2")
.build())
.inputUser("a")
.expectedQueue("q2")
.build());
verifyQueueMapping( verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%user", QueueMappingTestDataBuilder.create()
"%secondary_group"), .queueMapping(QueueMappingBuilder.create()
"b", YarnConfiguration.DEFAULT_QUEUE_NAME, "b", false, "bsubgroup2"); .type(MappingType.USER)
verifyQueueMapping(new QueueMapping(MappingType.GROUP, "asubgroup1", "q1"), .source("%user")
"a", "q1"); .queue("%user")
.build())
.inputUser("a")
.expectedQueue("a")
.build());
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%primary_group")
.build())
.inputUser("a")
.expectedQueue("agroup")
.build());
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.parentQueue("%primary_group")
.build())
.inputUser("a")
.expectedQueue("a")
.expectedParentQueue("agroup")
.build());
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.parentQueue("%secondary_group")
.build())
.inputUser("b")
.expectedQueue("b")
.expectedParentQueue("bsubgroup2")
.build());
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP)
.source("asubgroup1")
.queue("q1")
.build())
.inputUser("a")
.expectedQueue("q1")
.build());
// specify overwritten, and see if user specified a queue, and it will be // specify overwritten, and see if user specified a queue, and it will be
// overridden // overridden
verifyQueueMapping(new QueueMapping(MappingType.USER, "user", "q1"), "user", verifyQueueMapping(
"q2", "q1", true); QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
// if overwritten not specified, it should be which user specified .type(MappingType.USER)
verifyQueueMapping(new QueueMapping(MappingType.USER, "user", "q1"), "user", .source("user")
"q2", "q2", false); .queue("q1")
.build())
.inputUser("user")
.inputQueue("q2")
.expectedQueue("q1")
.overwrite(true)
.build());
// if overwritten not specified, it should be which user specified // if overwritten not specified, it should be which user specified
verifyQueueMapping( verifyQueueMapping(
new QueueMapping(MappingType.GROUP, "usergroup", "%user", "usergroup"), QueueMappingTestDataBuilder.create()
"user", "default", "user", false); .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("user")
.queue("q1")
.build())
.inputUser("user")
.inputQueue("q2")
.expectedQueue("q2")
.build());
// if overwritten not specified, it should be which user specified // if overwritten not specified, it should be which user specified
verifyQueueMapping( verifyQueueMapping(
new QueueMapping(MappingType.GROUP, "usergroup", "%user", "usergroup"), QueueMappingTestDataBuilder.create()
"user", "agroup", "user", true); .queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP)
.source("usergroup")
.queue("%user")
.parentQueue("usergroup")
.build())
.inputUser("user")
.inputQueue("default")
.expectedQueue("user")
.build());
//If user specific queue is enabled for a specified group under a given // if overwritten not specified, it should be which user specified
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP)
.source("usergroup")
.queue("%user")
.parentQueue("usergroup")
.build())
.inputUser("user")
.inputQueue("agroup")
.expectedQueue("user")
.overwrite(true)
.build());
// If user specific queue is enabled for a specified group under a given
// parent queue // parent queue
verifyQueueMapping( verifyQueueMapping(
new QueueMapping(MappingType.GROUP, "agroup", "%user", "parent1"), "a", QueueMappingTestDataBuilder.create()
"a"); .queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP)
.source("agroup")
.queue("%user")
.parentQueue("parent1")
.build())
.inputUser("a")
.expectedQueue("a")
.build());
//If user specific queue is enabled for a specified group without parent // If user specific queue is enabled for a specified group without parent
// queue // queue
verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", "%user"), verifyQueueMapping(
"a", "a"); QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP)
.source("agroup")
.queue("%user")
.build())
.inputUser("a")
.expectedQueue("a")
.build());
}
/**
* Queue Mapping test class to prepare the test data.
*
*/
public static final class QueueMappingTestData {
private QueueMapping queueMapping;
private String inputUser;
private String inputQueue;
private String expectedQueue;
private boolean overwrite;
private String expectedParentQueue;
private QueueMappingTestData(QueueMappingTestDataBuilder builder) {
this.queueMapping = builder.queueMapping;
this.inputUser = builder.inputUser;
this.inputQueue = builder.inputQueue;
this.expectedQueue = builder.expectedQueue;
this.overwrite = builder.overwrite;
this.expectedParentQueue = builder.expectedParentQueue;
}
/**
* Builder class to prepare the Queue Mapping test data.
*
*/
public static class QueueMappingTestDataBuilder {
private QueueMapping queueMapping = null;
private String inputUser = null;
private String inputQueue = YarnConfiguration.DEFAULT_QUEUE_NAME;
private String expectedQueue = null;
private boolean overwrite = false;
private String expectedParentQueue = null;
public QueueMappingTestDataBuilder() {
}
public static QueueMappingTestDataBuilder create() {
return new QueueMappingTestDataBuilder();
}
public QueueMappingTestDataBuilder queueMapping(QueueMapping mapping) {
this.queueMapping = mapping;
return this;
}
public QueueMappingTestDataBuilder inputUser(String user) {
this.inputUser = user;
return this;
}
public QueueMappingTestDataBuilder inputQueue(String queue) {
this.inputQueue = queue;
return this;
}
public QueueMappingTestDataBuilder expectedQueue(String outputQueue) {
this.expectedQueue = outputQueue;
return this;
}
public QueueMappingTestDataBuilder overwrite(boolean overwriteMappings) {
this.overwrite = overwriteMappings;
return this;
}
public QueueMappingTestDataBuilder expectedParentQueue(
String outputParentQueue) {
this.expectedParentQueue = outputParentQueue;
return this;
}
public QueueMappingTestData build() {
return new QueueMappingTestData(this);
}
}
} }
} }

View File

@ -46,8 +46,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.placement import org.apache.hadoop.yarn.server.resourcemanager.placement
.ApplicationPlacementContext; .ApplicationPlacementContext;
import org.apache.hadoop.yarn.server.resourcemanager.placement import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
.UserGroupMappingPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
@ -268,19 +268,19 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE); queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
conf.setQueuePlacementRules(queuePlacementRules); conf.setQueuePlacementRules(queuePlacementRules);
List<UserGroupMappingPlacementRule.QueueMapping> existingMappings = List<QueueMapping> existingMappings = conf.getQueueMappings();
conf.getQueueMappings();
//set queue mapping //set queue mapping
List<UserGroupMappingPlacementRule.QueueMapping> queueMappings = List<QueueMapping> queueMappings = new ArrayList<>();
new ArrayList<>();
for (int i = 0; i < userIds.length; i++) { for (int i = 0; i < userIds.length; i++) {
//Set C as parent queue name for auto queue creation //Set C as parent queue name for auto queue creation
UserGroupMappingPlacementRule.QueueMapping userQueueMapping = QueueMapping userQueueMapping = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source(USER + userIds[i])
USER + userIds[i], .queue(
getQueueMapping(parentQueue, USER + userIds[i])); getQueueMapping(parentQueue,
USER + userIds[i]))
.build();
queueMappings.add(userQueueMapping); queueMappings.add(userQueueMapping);
} }
@ -295,12 +295,10 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
(String parentQueue, CapacitySchedulerConfiguration conf, String (String parentQueue, CapacitySchedulerConfiguration conf, String
leafQueueName) { leafQueueName) {
List<UserGroupMappingPlacementRule.QueueMapping> existingMappings = List<QueueMapping> existingMappings = conf.getQueueMappings();
conf.getQueueMappings();
//set queue mapping //set queue mapping
List<UserGroupMappingPlacementRule.QueueMapping> queueMappings = List<QueueMapping> queueMappings = new ArrayList<>();
new ArrayList<>();
//setup group mapping //setup group mapping
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
@ -309,11 +307,13 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
TEST_GROUPUSER +"=" + TEST_GROUP + ";invalid_user=invalid_group"); TEST_GROUPUSER +"=" + TEST_GROUP + ";invalid_user=invalid_group");
Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf);
UserGroupMappingPlacementRule.QueueMapping userQueueMapping = QueueMapping userQueueMapping = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.GROUP)
UserGroupMappingPlacementRule.QueueMapping.MappingType.GROUP, .source(TEST_GROUP)
TEST_GROUP, .queue(
getQueueMapping(parentQueue, leafQueueName)); getQueueMapping(parentQueue,
leafQueueName))
.build();
queueMappings.add(userQueueMapping); queueMappings.add(userQueueMapping);
existingMappings.addAll(queueMappings); existingMappings.addAll(queueMappings);
@ -514,13 +514,14 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
return rmApp.getApplicationId(); return rmApp.getApplicationId();
} }
protected List<UserGroupMappingPlacementRule.QueueMapping> setupQueueMapping( protected List<QueueMapping> setupQueueMapping(
CapacityScheduler newCS, String user, String parentQueue, String queue) { CapacityScheduler newCS, String user, String parentQueue, String queue) {
List<UserGroupMappingPlacementRule.QueueMapping> queueMappings = List<QueueMapping> queueMappings = new ArrayList<>();
new ArrayList<>(); queueMappings.add(QueueMappingBuilder.create()
queueMappings.add(new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, user, .source(user)
getQueueMapping(parentQueue, queue))); .queue(getQueueMapping(parentQueue, queue))
.build());
newCS.getConfiguration().setQueueMappings(queueMappings); newCS.getConfiguration().setQueueMappings(queueMappings);
return queueMappings; return queueMappings;
} }

View File

@ -26,6 +26,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@ -62,19 +65,19 @@ public class TestCapacitySchedulerQueueMappingFactory {
conf.setQueuePlacementRules(queuePlacementRules); conf.setQueuePlacementRules(queuePlacementRules);
List<UserGroupMappingPlacementRule.QueueMapping> existingMappingsForUG = List<QueueMapping> existingMappingsForUG = conf.getQueueMappings();
conf.getQueueMappings();
//set queue mapping //set queue mapping
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG = List<QueueMapping> queueMappingsForUG = new ArrayList<>();
new ArrayList<>();
for (int i = 0; i < sourceIds.length; i++) { for (int i = 0; i < sourceIds.length; i++) {
//Set C as parent queue name for auto queue creation //Set C as parent queue name for auto queue creation
UserGroupMappingPlacementRule.QueueMapping userQueueMapping = QueueMapping userQueueMapping = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source(USER + sourceIds[i])
USER + sourceIds[i], .queue(
getQueueMapping(parentQueue, USER + sourceIds[i])); getQueueMapping(parentQueue,
USER + sourceIds[i]))
.build();
queueMappingsForUG.add(userQueueMapping); queueMappingsForUG.add(userQueueMapping);
} }
@ -152,23 +155,25 @@ public class TestCapacitySchedulerQueueMappingFactory {
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
conf.setQueuePlacementRules(queuePlacementRules); conf.setQueuePlacementRules(queuePlacementRules);
List<UserGroupMappingPlacementRule.QueueMapping> existingMappingsForUG = List<QueueMapping> existingMappingsForUG = conf.getQueueMappings();
conf.getQueueMappings();
// set queue mapping // set queue mapping
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG = List<QueueMapping> queueMappingsForUG = new ArrayList<>();
new ArrayList<>();
// u:user1:b1 // u:user1:b1
UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = QueueMapping userQueueMapping1 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "user1", .source("user1")
"b1"); .queue("b1")
.build();
// u:%user:parentqueue.%user // u:%user:parentqueue.%user
UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "%user", .source("%user")
getQueueMapping("c", "%user")); .queue(getQueueMapping("c", "%user"))
.build();
queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping1);
queueMappingsForUG.add(userQueueMapping2); queueMappingsForUG.add(userQueueMapping2);
@ -221,26 +226,32 @@ public class TestCapacitySchedulerQueueMappingFactory {
*/ */
// set queue mapping // set queue mapping
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG = List<QueueMapping> queueMappingsForUG = new ArrayList<>();
new ArrayList<>();
// u:%user:%primary_group.%user // u:%user:%primary_group.%user
UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = QueueMapping userQueueMapping1 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source("%user")
"%user", getQueueMapping("%primary_group", "%user")); .queue(
getQueueMapping("%primary_group",
"%user"))
.build();
// u:%user:%secondary_group.%user // u:%user:%secondary_group.%user
UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source("%user")
"%user", getQueueMapping("%secondary_group", "%user")); .queue(
getQueueMapping("%secondary_group",
"%user"))
.build();
// u:b4:%secondary_group // u:b4:%secondary_group
UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 = QueueMapping userQueueMapping3 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source("b4")
"b4", "%secondary_group"); .queue("%secondary_group")
.build();
queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping1);
queueMappingsForUG.add(userQueueMapping2); queueMappingsForUG.add(userQueueMapping2);
queueMappingsForUG.add(userQueueMapping3); queueMappingsForUG.add(userQueueMapping3);
@ -276,20 +287,26 @@ public class TestCapacitySchedulerQueueMappingFactory {
*/ */
// set queue mapping // set queue mapping
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG = List<QueueMapping> queueMappingsForUG = new ArrayList<>();
new ArrayList<>();
// u:%user:%primary_group.%user // u:%user:%primary_group.%user
UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = QueueMapping userQueueMapping1 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source("%user")
"%user", getQueueMapping("%primary_group", "%user")); .queue(
getQueueMapping("%primary_group",
"%user"))
.build();
// u:%user:%secondary_group.%user // u:%user:%secondary_group.%user
UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source("%user")
"%user", getQueueMapping("%secondary_group", "%user")); .queue(
getQueueMapping(
"%secondary_group", "%user")
)
.build();
queueMappingsForUG.add(userQueueMapping2); queueMappingsForUG.add(userQueueMapping2);
queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping1);
@ -298,8 +315,7 @@ public class TestCapacitySchedulerQueueMappingFactory {
} }
private void testNestedUserQueueWithDynamicParentQueue( private void testNestedUserQueueWithDynamicParentQueue(
List<UserGroupMappingPlacementRule.QueueMapping> mapping, boolean primary, List<QueueMapping> mapping, boolean primary, String user)
String user)
throws Exception { throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf); setupQueueConfiguration(conf);
@ -312,8 +328,7 @@ public class TestCapacitySchedulerQueueMappingFactory {
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
conf.setQueuePlacementRules(queuePlacementRules); conf.setQueuePlacementRules(queuePlacementRules);
List<UserGroupMappingPlacementRule.QueueMapping> existingMappingsForUG = List<QueueMapping> existingMappingsForUG = conf.getQueueMappings();
conf.getQueueMappings();
existingMappingsForUG.addAll(mapping); existingMappingsForUG.addAll(mapping);
conf.setQueueMappings(existingMappingsForUG); conf.setQueueMappings(existingMappingsForUG);
@ -367,24 +382,24 @@ public class TestCapacitySchedulerQueueMappingFactory {
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
conf.setQueuePlacementRules(queuePlacementRules); conf.setQueuePlacementRules(queuePlacementRules);
List<UserGroupMappingPlacementRule.QueueMapping> existingMappingsForUG = List<QueueMapping> existingMappingsForUG = conf.getQueueMappings();
conf.getQueueMappings();
// set queue mapping // set queue mapping
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG = List<QueueMapping> queueMappingsForUG = new ArrayList<>();
new ArrayList<>();
// u:user1:b1 // u:user1:b1
UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = QueueMapping userQueueMapping1 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source("user1")
"user1", "b1"); .queue("b1")
.build();
// u:user2:%primary_group // u:user2:%primary_group
UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source("user2")
"user2", "%primary_group"); .queue("%primary_group")
.build();
queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping1);
queueMappingsForUG.add(userQueueMapping2); queueMappingsForUG.add(userQueueMapping2);
@ -436,30 +451,31 @@ public class TestCapacitySchedulerQueueMappingFactory {
queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP);
conf.setQueuePlacementRules(queuePlacementRules); conf.setQueuePlacementRules(queuePlacementRules);
List<UserGroupMappingPlacementRule.QueueMapping> existingMappingsForUG = List<QueueMapping> existingMappingsForUG = conf.getQueueMappings();
conf.getQueueMappings();
// set queue mapping // set queue mapping
List<UserGroupMappingPlacementRule.QueueMapping> queueMappingsForUG = List<QueueMapping> queueMappingsForUG = new ArrayList<>();
new ArrayList<>();
// u:user1:b1 // u:user1:b1
UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = QueueMapping userQueueMapping1 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source("user1")
"user1", "b1"); .queue("b1")
.build();
// u:user2:%primary_group // u:user2:%primary_group
UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, .source("user2")
"user2", "%primary_group"); .queue("%primary_group")
.build();
// u:b4:%secondary_group // u:b4:%secondary_group
UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 = QueueMapping userQueueMapping3 = QueueMappingBuilder.create()
new UserGroupMappingPlacementRule.QueueMapping( .type(QueueMapping.MappingType.USER)
UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "b4", .source("b4")
"%secondary_group"); .queue("%secondary_group")
.build();
queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping1);
queueMappingsForUG.add(userQueueMapping2); queueMappingsForUG.add(userQueueMapping2);

View File

@ -23,9 +23,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@ -91,7 +92,12 @@ public class TestQueueMappings {
// space trimming // space trimming
conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, " u : a : " + Q1); conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, " u : a : " + Q1);
cs.reinitialize(conf, null); cs.reinitialize(conf, null);
checkQMapping(new QueueMapping(MappingType.USER, "a", Q1)); checkQMapping(
QueueMappingBuilder.create()
.type(MappingType.USER)
.source("a")
.queue(Q1)
.build());
} }
@Test (timeout = 60000) @Test (timeout = 60000)