diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java new file mode 100644 index 00000000000..bd0e0e296cd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +/** + * Queue Mapping class to hold the queue mapping information. + * + */ +@Private +public class QueueMapping { + + /** + * Builder class for QueueMapping. + * + */ + public static class QueueMappingBuilder { + + private MappingType type; + private String source; + private String queue; + private String parentQueue; + + public QueueMappingBuilder() { + } + + public static QueueMappingBuilder create() { + return new QueueMappingBuilder(); + } + + public QueueMappingBuilder type(MappingType mappingType) { + this.type = mappingType; + return this; + } + + public QueueMappingBuilder source(String mappingSource) { + this.source = mappingSource; + return this; + } + + public QueueMappingBuilder queue(String mappingQueue) { + this.queue = mappingQueue; + return this; + } + + public QueueMappingBuilder parentQueue(String mappingParentQueue) { + this.parentQueue = mappingParentQueue; + return this; + } + + public QueueMapping build() { + return new QueueMapping(this); + } + } + + private QueueMapping(QueueMappingBuilder builder) { + this.type = builder.type; + this.source = builder.source; + this.queue = builder.queue; + this.parentQueue = builder.parentQueue; + } + + /** + * Different types of mapping. + * + */ + public enum MappingType { + + USER("u"), GROUP("g"); + + private final String type; + + MappingType(String type) { + this.type = type; + } + + public String toString() { + return type; + } + + }; + + private MappingType type; + private String source; + private String queue; + private String parentQueue; + + private final static String DELIMITER = ":"; + + public String getQueue() { + return queue; + } + + public String getParentQueue() { + return parentQueue; + } + + public boolean hasParentQueue() { + return parentQueue != null; + } + + public MappingType getType() { + return type; + } + + public String getSource() { + return source; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = + prime * result + ((parentQueue == null) ? 0 : parentQueue.hashCode()); + result = prime * result + ((queue == null) ? 0 : queue.hashCode()); + result = prime * result + ((source == null) ? 0 : source.hashCode()); + result = prime * result + ((type == null) ? 0 : type.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + QueueMapping other = (QueueMapping) obj; + if (parentQueue == null) { + if (other.parentQueue != null) { + return false; + } + } else if (!parentQueue.equals(other.parentQueue)) { + return false; + } + if (queue == null) { + if (other.queue != null) { + return false; + } + } else if (!queue.equals(other.queue)) { + return false; + } + if (source == null) { + if (other.source != null) { + return false; + } + } else if (!source.equals(other.source)) { + return false; + } + if (type != other.type) { + return false; + } + return true; + } + + @Override + public String toString() { + return type.toString() + DELIMITER + source + DELIMITER + + (parentQueue != null ? parentQueue + "." + queue : queue); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java index c457b2d5281..465136bea9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java @@ -23,18 +23,14 @@ import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.security.Groups; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType; - -import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -43,6 +39,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; public class UserGroupMappingPlacementRule extends PlacementRule { private static final Logger LOG = LoggerFactory @@ -56,91 +56,6 @@ public class UserGroupMappingPlacementRule extends PlacementRule { private List mappings = null; private Groups groups; - @Private - public static class QueueMapping { - - public enum MappingType { - - USER("u"), GROUP("g"); - private final String type; - - private MappingType(String type) { - this.type = type; - } - - public String toString() { - return type; - } - - }; - - MappingType type; - String source; - String queue; - String parentQueue; - - public final static String DELIMITER = ":"; - - public QueueMapping(MappingType type, String source, String queue) { - this.type = type; - this.source = source; - this.queue = queue; - this.parentQueue = null; - } - - public QueueMapping(MappingType type, String source, - String queue, String parentQueue) { - this.type = type; - this.source = source; - this.queue = queue; - this.parentQueue = parentQueue; - } - - public String getQueue() { - return queue; - } - - public String getParentQueue() { - return parentQueue; - } - - public boolean hasParentQueue() { - return parentQueue != null; - } - - public MappingType getType() { - return type; - } - - public String getSource() { - return source; - } - - @Override - public int hashCode() { - return super.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof QueueMapping) { - QueueMapping other = (QueueMapping) obj; - return (other.type.equals(type) && - other.source.equals(source) && - other.queue.equals(queue)); - } else { - return false; - } - } - - public String toString() { - return type.toString() + DELIMITER + source + DELIMITER + - (parentQueue != null ? - parentQueue + "." + queue : - queue); - } - } - public UserGroupMappingPlacementRule(){ this(false, null, null); } @@ -155,24 +70,24 @@ public class UserGroupMappingPlacementRule extends PlacementRule { private ApplicationPlacementContext getPlacementForUser(String user) throws IOException { for (QueueMapping mapping : mappings) { - if (mapping.type == MappingType.USER) { - if (mapping.source.equals(CURRENT_USER_MAPPING)) { - if (mapping.queue.equals(CURRENT_USER_MAPPING)) { + if (mapping.getType() == MappingType.USER) { + if (mapping.getSource().equals(CURRENT_USER_MAPPING)) { + if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) { return getPlacementContext(mapping, user); - } else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) { + } else if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) { return getPlacementContext(mapping, groups.getGroups(user).get(0)); } else { return getPlacementContext(mapping); } } - if (user.equals(mapping.source)) { + if (user.equals(mapping.getSource())) { return getPlacementContext(mapping); } } - if (mapping.type == MappingType.GROUP) { + if (mapping.getType() == MappingType.GROUP) { for (String userGroups : groups.getGroups(user)) { - if (userGroups.equals(mapping.source)) { - if (mapping.queue.equals(CURRENT_USER_MAPPING)) { + if (userGroups.equals(mapping.getSource())) { + if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) { return getPlacementContext(mapping, user); } return getPlacementContext(mapping); @@ -220,7 +135,7 @@ public class UserGroupMappingPlacementRule extends PlacementRule { private ApplicationPlacementContext getPlacementContext(QueueMapping mapping, String leafQueueName) { - if (!StringUtils.isEmpty(mapping.parentQueue)) { + if (!StringUtils.isEmpty(mapping.getParentQueue())) { return new ApplicationPlacementContext(leafQueueName, mapping.getParentQueue()); } else{ @@ -357,8 +272,12 @@ public class UserGroupMappingPlacementRule extends PlacementRule { QueuePlacementRuleUtils.validateQueueMappingUnderParentQueue( queueManager.getQueue(queuePath.getParentQueue()), queuePath.getParentQueue(), queuePath.getLeafQueue()); - return new QueueMapping(mapping.getType(), mapping.getSource(), - queuePath.getLeafQueue(), queuePath.getParentQueue()); + return QueueMappingBuilder.create() + .type(mapping.getType()) + .source(mapping.getSource()) + .queue(queuePath.getLeafQueue()) + .parentQueue(queuePath.getParentQueue()) + .build(); } return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 962e3af5239..bd4c68f7066 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -38,8 +38,9 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AppPriorityACLConfigurationParser.AppPriorityACLKeyType; @@ -1128,10 +1129,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur throw new IllegalArgumentException( "unknown mapping prefix " + mapping[0]); } - m = new QueueMapping( - mappingType, - mapping[1], - mapping[2]); + m = QueueMappingBuilder.create() + .type(mappingType) + .source(mapping[1]) + .queue(mapping[2]) + .build(); } catch (Throwable t) { throw new IllegalArgumentException( "Illegal queue mapping " + mappingValue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementManager.java index db5cd60e580..3b85fdad08d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementManager.java @@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -70,11 +72,13 @@ public class TestPlacementManager { .getQueuePlacementManager(); List queuePlacementRules = new ArrayList<>(); - UserGroupMappingPlacementRule.QueueMapping userQueueMapping = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - USER1, - getQueueMapping(PARENT_QUEUE, USER1)); + QueueMapping userQueueMapping = QueueMappingBuilder.create() + .type(MappingType.USER) + .source(USER1) + .queue( + getQueueMapping(PARENT_QUEUE, + USER1)) + .build(); UserGroupMappingPlacementRule ugRule = new UserGroupMappingPlacementRule( false, Arrays.asList(userQueueMapping), null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java index a6143ba6cbb..660c3a0cd91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java @@ -26,8 +26,9 @@ import org.apache.hadoop.security.Groups; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; +import org.apache.hadoop.yarn.server.resourcemanager.placement.TestUserGroupMappingPlacementRule.QueueMappingTestData.QueueMappingTestDataBuilder; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SimpleGroupsMapping; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; @@ -43,14 +44,16 @@ public class TestUserGroupMappingPlacementRule { SimpleGroupsMapping.class, GroupMappingServiceProvider.class); } - private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, - String expectedQueue) throws YarnException { - verifyQueueMapping(queueMapping, inputUser, - YarnConfiguration.DEFAULT_QUEUE_NAME, expectedQueue, false); - } + private void verifyQueueMapping(QueueMappingTestData queueMappingTestData) + throws YarnException { + + QueueMapping queueMapping = queueMappingTestData.queueMapping; + String inputUser = queueMappingTestData.inputUser; + String inputQueue = queueMappingTestData.inputQueue; + String expectedQueue = queueMappingTestData.expectedQueue; + boolean overwrite = queueMappingTestData.overwrite; + String expectedParentQueue = queueMappingTestData.expectedParentQueue; - private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, - String inputQueue, String expectedQueue, boolean overwrite) throws YarnException { Groups groups = new Groups(conf); UserGroupMappingPlacementRule rule = new UserGroupMappingPlacementRule( overwrite, Arrays.asList(queueMapping), groups); @@ -58,53 +61,222 @@ public class TestUserGroupMappingPlacementRule { ApplicationSubmissionContext.class); asc.setQueue(inputQueue); ApplicationPlacementContext ctx = rule.getPlacementForApp(asc, inputUser); - Assert.assertEquals(expectedQueue, + Assert.assertEquals("Queue", expectedQueue, ctx != null ? ctx.getQueue() : inputQueue); + if (expectedParentQueue != null) { + Assert.assertEquals("Parent Queue", expectedParentQueue, + ctx.getParentQueue()); + } } @Test public void testMapping() throws YarnException { + // simple base case for mapping user to queue - verifyQueueMapping(new QueueMapping(MappingType.USER, "a", "q1"), "a", "q1"); - verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", "q1"), - "a", "q1"); - verifyQueueMapping(new QueueMapping(MappingType.USER, "%user", "q2"), "a", - "q2"); - verifyQueueMapping(new QueueMapping(MappingType.USER, "%user", "%user"), - "a", "a"); - verifyQueueMapping(new QueueMapping(MappingType.USER, "%user", - "%primary_group"), "a", "agroup"); - verifyQueueMapping(new QueueMapping(MappingType.GROUP, "asubgroup1", "q1"), - "a", "q1"); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("a") + .queue("q1").build()) + .inputUser("a") + .expectedQueue("q1") + .build()); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.GROUP) + .source("agroup") + .queue("q1").build()) + .inputUser("a") + .expectedQueue("q1") + .build()); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("q2").build()) + .inputUser("a") + .expectedQueue("q2") + .build()); + + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%user").build()) + .inputUser("a") + .expectedQueue("a") + .build()); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%primary_group").build()) + .inputUser("a") + .expectedQueue("agroup") + .build()); + + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.GROUP) + .source("asubgroup1") + .queue("q1").build()) + .inputUser("a") + .expectedQueue("q1") + .build()); // specify overwritten, and see if user specified a queue, and it will be // overridden - verifyQueueMapping(new QueueMapping(MappingType.USER, "user", "q1"), - "user", "q2", "q1", true); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("user") + .queue("q1").build()) + .inputUser("user") + .inputQueue("q2") + .expectedQueue("q1") + .overwrite(true) + .build()); // if overwritten not specified, it should be which user specified - verifyQueueMapping(new QueueMapping(MappingType.USER, "user", "q1"), - "user", "q2", "q2", false); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.GROUP) + .source("usergroup") + .queue("%user") + .parentQueue("usergroup") + .build()) + .inputUser("user") + .inputQueue("default") + .expectedQueue("user") + .build()); // if overwritten not specified, it should be which user specified - verifyQueueMapping(new QueueMapping(MappingType.GROUP, "usergroup", - "%user", "usergroup"), - "user", "default", "user", false); - - // if overwritten not specified, it should be which user specified - verifyQueueMapping(new QueueMapping(MappingType.GROUP, "usergroup", - "%user", "usergroup"), - "user", "agroup", "user", true); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create(). + type(MappingType.GROUP) + .source("usergroup") + .queue("%user") + .parentQueue("usergroup") + .build()) + .inputUser("user") + .inputQueue("agroup") + .expectedQueue("user") + .overwrite(true) + .build()); //If user specific queue is enabled for a specified group under a given // parent queue - verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", - "%user", "parent1"), - "a", "a"); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.GROUP) + .source("agroup") + .queue("%user") + .parentQueue("parent1") + .build()) + .inputUser("a") + .expectedQueue("a") + .build()); //If user specific queue is enabled for a specified group without parent // queue - verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", "%user"), - "a", "a"); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.GROUP) + .source("agroup") + .queue("%user").build()) + .inputUser("a") + .expectedQueue("a") + .build()); + } + + /** + * Queue Mapping test class to prepare the test data. + * + */ + public static final class QueueMappingTestData { + + private QueueMapping queueMapping; + private String inputUser; + private String inputQueue; + private String expectedQueue; + private boolean overwrite; + private String expectedParentQueue; + + private QueueMappingTestData(QueueMappingTestDataBuilder builder) { + this.queueMapping = builder.queueMapping; + this.inputUser = builder.inputUser; + this.inputQueue = builder.inputQueue; + this.expectedQueue = builder.expectedQueue; + this.overwrite = builder.overwrite; + this.expectedParentQueue = builder.expectedParentQueue; + } + + /** + * Builder class to prepare the Queue Mapping test data. + * + */ + public static class QueueMappingTestDataBuilder { + + private QueueMapping queueMapping = null; + private String inputUser = null; + private String inputQueue = YarnConfiguration.DEFAULT_QUEUE_NAME; + private String expectedQueue = null; + private boolean overwrite = false; + private String expectedParentQueue = null; + + public QueueMappingTestDataBuilder() { + + } + + public static QueueMappingTestDataBuilder create() { + return new QueueMappingTestDataBuilder(); + } + + public QueueMappingTestDataBuilder queueMapping(QueueMapping mapping) { + this.queueMapping = mapping; + return this; + } + + public QueueMappingTestDataBuilder inputUser(String user) { + this.inputUser = user; + return this; + } + + public QueueMappingTestDataBuilder inputQueue(String queue) { + this.inputQueue = queue; + return this; + } + + public QueueMappingTestDataBuilder expectedQueue(String outputQueue) { + this.expectedQueue = outputQueue; + return this; + } + + public QueueMappingTestDataBuilder overwrite(boolean overwriteMappings) { + this.overwrite = overwriteMappings; + return this; + } + + public QueueMappingTestDataBuilder expectedParentQueue( + String outputParentQueue) { + this.expectedParentQueue = outputParentQueue; + return this; + } + + public QueueMappingTestData build() { + return new QueueMappingTestData(this); + } + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index 97d854267ce..67463b1037e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -43,8 +43,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.placement .ApplicationPlacementContext; -import org.apache.hadoop.yarn.server.resourcemanager.placement - .UserGroupMappingPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; @@ -255,19 +255,19 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE); conf.setQueuePlacementRules(queuePlacementRules); - List existingMappings = - conf.getQueueMappings(); + List existingMappings = conf.getQueueMappings(); //set queue mapping - List queueMappings = - new ArrayList<>(); + List queueMappings = new ArrayList<>(); for (int i = 0; i < userIds.length; i++) { //Set C as parent queue name for auto queue creation - UserGroupMappingPlacementRule.QueueMapping userQueueMapping = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - USER + userIds[i], - getQueueMapping(parentQueue, USER + userIds[i])); + QueueMapping userQueueMapping = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source(USER + userIds[i]) + .queue( + getQueueMapping(parentQueue, + USER + userIds[i])) + .build(); queueMappings.add(userQueueMapping); } @@ -282,12 +282,10 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { (String parentQueue, CapacitySchedulerConfiguration conf, String leafQueueName) { - List existingMappings = - conf.getQueueMappings(); + List existingMappings = conf.getQueueMappings(); //set queue mapping - List queueMappings = - new ArrayList<>(); + List queueMappings = new ArrayList<>(); //setup group mapping conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, @@ -296,11 +294,13 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { TEST_GROUPUSER +"=" + TEST_GROUP + ";invalid_user=invalid_group"); Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); - UserGroupMappingPlacementRule.QueueMapping userQueueMapping = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.GROUP, - TEST_GROUP, - getQueueMapping(parentQueue, leafQueueName)); + QueueMapping userQueueMapping = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.GROUP) + .source(TEST_GROUP) + .queue( + getQueueMapping(parentQueue, + leafQueueName)) + .build(); queueMappings.add(userQueueMapping); existingMappings.addAll(queueMappings); @@ -482,13 +482,14 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { return rmApp.getApplicationId(); } - protected List setupQueueMapping( + protected List setupQueueMapping( CapacityScheduler newCS, String user, String parentQueue, String queue) { - List queueMappings = - new ArrayList<>(); - queueMappings.add(new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, user, - getQueueMapping(parentQueue, queue))); + List queueMappings = new ArrayList<>(); + queueMappings.add(QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source(user) + .queue(getQueueMapping(parentQueue, queue)) + .build()); newCS.getConfiguration().setQueueMappings(queueMappings); return queueMappings; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java index b4f97123c0f..9395052f14f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java @@ -21,8 +21,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.junit.Test; @@ -57,19 +59,19 @@ public class TestCapacitySchedulerQueueMappingFactory { conf.setQueuePlacementRules(queuePlacementRules); - List existingMappingsForUG = - conf.getQueueMappings(); + List existingMappingsForUG = conf.getQueueMappings(); //set queue mapping - List queueMappingsForUG = - new ArrayList<>(); + List queueMappingsForUG = new ArrayList<>(); for (int i = 0; i < sourceIds.length; i++) { //Set C as parent queue name for auto queue creation - UserGroupMappingPlacementRule.QueueMapping userQueueMapping = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - USER + sourceIds[i], - getQueueMapping(parentQueue, USER + sourceIds[i])); + QueueMapping userQueueMapping = QueueMappingBuilder.create() + .type(MappingType.USER) + .source(USER + sourceIds[i]) + .queue( + getQueueMapping(parentQueue, + USER + sourceIds[i])) + .build(); queueMappingsForUG.add(userQueueMapping); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java index fbf092f334c..2e7009eae65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java @@ -23,9 +23,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -91,7 +92,12 @@ public class TestQueueMappings { // space trimming conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, " u : a : " + Q1); cs.reinitialize(conf, null); - checkQMapping(new QueueMapping(MappingType.USER, "a", Q1)); + checkQMapping( + QueueMappingBuilder.create() + .type(MappingType.USER) + .source("a") + .queue(Q1) + .build()); } @Test (timeout = 60000)