From 7c5cecc3b3c00886a5bc39a9a8cad6ca1088b095 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Wed, 15 Jan 2020 09:55:41 +0100 Subject: [PATCH] YARN-9970. Refactor TestUserGroupMappingPlacementRule#verifyQueueMapping. Contributed by Manikandan R --- .../placement/QueueMapping.java | 183 ++++++++++ .../UserGroupMappingPlacementRule.java | 153 +++----- .../CapacitySchedulerConfiguration.java | 12 +- .../placement/TestPlacementManager.java | 14 +- .../TestUserGroupMappingPlacementRule.java | 339 ++++++++++++++---- ...CapacitySchedulerAutoCreatedQueueBase.java | 53 +-- ...tCapacitySchedulerQueueMappingFactory.java | 170 +++++---- .../scheduler/capacity/TestQueueMappings.java | 12 +- 8 files changed, 645 insertions(+), 291 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java new file mode 100644 index 00000000000..bd0e0e296cd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueueMapping.java @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +/** + * Queue Mapping class to hold the queue mapping information. + * + */ +@Private +public class QueueMapping { + + /** + * Builder class for QueueMapping. + * + */ + public static class QueueMappingBuilder { + + private MappingType type; + private String source; + private String queue; + private String parentQueue; + + public QueueMappingBuilder() { + } + + public static QueueMappingBuilder create() { + return new QueueMappingBuilder(); + } + + public QueueMappingBuilder type(MappingType mappingType) { + this.type = mappingType; + return this; + } + + public QueueMappingBuilder source(String mappingSource) { + this.source = mappingSource; + return this; + } + + public QueueMappingBuilder queue(String mappingQueue) { + this.queue = mappingQueue; + return this; + } + + public QueueMappingBuilder parentQueue(String mappingParentQueue) { + this.parentQueue = mappingParentQueue; + return this; + } + + public QueueMapping build() { + return new QueueMapping(this); + } + } + + private QueueMapping(QueueMappingBuilder builder) { + this.type = builder.type; + this.source = builder.source; + this.queue = builder.queue; + this.parentQueue = builder.parentQueue; + } + + /** + * Different types of mapping. + * + */ + public enum MappingType { + + USER("u"), GROUP("g"); + + private final String type; + + MappingType(String type) { + this.type = type; + } + + public String toString() { + return type; + } + + }; + + private MappingType type; + private String source; + private String queue; + private String parentQueue; + + private final static String DELIMITER = ":"; + + public String getQueue() { + return queue; + } + + public String getParentQueue() { + return parentQueue; + } + + public boolean hasParentQueue() { + return parentQueue != null; + } + + public MappingType getType() { + return type; + } + + public String getSource() { + return source; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = + prime * result + ((parentQueue == null) ? 0 : parentQueue.hashCode()); + result = prime * result + ((queue == null) ? 0 : queue.hashCode()); + result = prime * result + ((source == null) ? 0 : source.hashCode()); + result = prime * result + ((type == null) ? 0 : type.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + QueueMapping other = (QueueMapping) obj; + if (parentQueue == null) { + if (other.parentQueue != null) { + return false; + } + } else if (!parentQueue.equals(other.parentQueue)) { + return false; + } + if (queue == null) { + if (other.queue != null) { + return false; + } + } else if (!queue.equals(other.queue)) { + return false; + } + if (source == null) { + if (other.source != null) { + return false; + } + } else if (!source.equals(other.source)) { + return false; + } + if (type != other.type) { + return false; + } + return true; + } + + @Override + public String toString() { + return type.toString() + DELIMITER + source + DELIMITER + + (parentQueue != null ? parentQueue + "." + queue : queue); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java index 5221ace559a..6dce9c76c91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java @@ -23,18 +23,15 @@ import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.security.Groups; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType; - -import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -44,6 +41,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; public class UserGroupMappingPlacementRule extends PlacementRule { private static final Logger LOG = LoggerFactory @@ -60,91 +61,6 @@ public class UserGroupMappingPlacementRule extends PlacementRule { private Groups groups; private CapacitySchedulerQueueManager queueManager; - @Private - public static class QueueMapping { - - public enum MappingType { - - USER("u"), GROUP("g"); - private final String type; - - private MappingType(String type) { - this.type = type; - } - - public String toString() { - return type; - } - - }; - - MappingType type; - String source; - String queue; - String parentQueue; - - public final static String DELIMITER = ":"; - - public QueueMapping(MappingType type, String source, String queue) { - this.type = type; - this.source = source; - this.queue = queue; - this.parentQueue = null; - } - - public QueueMapping(MappingType type, String source, - String queue, String parentQueue) { - this.type = type; - this.source = source; - this.queue = queue; - this.parentQueue = parentQueue; - } - - public String getQueue() { - return queue; - } - - public String getParentQueue() { - return parentQueue; - } - - public boolean hasParentQueue() { - return parentQueue != null; - } - - public MappingType getType() { - return type; - } - - public String getSource() { - return source; - } - - @Override - public int hashCode() { - return super.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof QueueMapping) { - QueueMapping other = (QueueMapping) obj; - return (other.type.equals(type) && - other.source.equals(source) && - other.queue.equals(queue)); - } else { - return false; - } - } - - public String toString() { - return type.toString() + DELIMITER + source + DELIMITER + - (parentQueue != null ? - parentQueue + "." + queue : - queue); - } - } - public UserGroupMappingPlacementRule(){ this(false, null, null); } @@ -179,15 +95,19 @@ public class UserGroupMappingPlacementRule extends PlacementRule { private ApplicationPlacementContext getPlacementForUser(String user) throws IOException { for (QueueMapping mapping : mappings) { - if (mapping.type == MappingType.USER) { - if (mapping.source.equals(CURRENT_USER_MAPPING)) { + if (mapping.getType() == MappingType.USER) { + if (mapping.getSource().equals(CURRENT_USER_MAPPING)) { if (mapping.getParentQueue() != null && mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING) && mapping.getQueue().equals(CURRENT_USER_MAPPING)) { if (this.queueManager .getQueue(groups.getGroups(user).get(0)) != null) { - QueueMapping queueMapping = new QueueMapping(mapping.getType(), - mapping.getSource(), user, groups.getGroups(user).get(0)); + QueueMapping queueMapping = + QueueMappingBuilder.create() + .type(mapping.getType()) + .source(mapping.getSource()).queue(user) + .parentQueue(groups.getGroups(user).get(0)) + .build(); validateQueueMapping(queueMapping); return getPlacementContext(queueMapping, user); } else { @@ -198,16 +118,21 @@ public class UserGroupMappingPlacementRule extends PlacementRule { && mapping.getQueue().equals(CURRENT_USER_MAPPING)) { String secondaryGroup = getSecondaryGroup(user); if (secondaryGroup != null) { - QueueMapping queueMapping = new QueueMapping(mapping.getType(), - mapping.getSource(), user, secondaryGroup); + QueueMapping queueMapping = + QueueMappingBuilder.create() + .type(mapping.getType()) + .source(mapping.getSource()) + .queue(user) + .parentQueue(secondaryGroup) + .build(); validateQueueMapping(queueMapping); return getPlacementContext(queueMapping, user); } else { return null; } - } else if (mapping.queue.equals(CURRENT_USER_MAPPING)) { + } else if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) { return getPlacementContext(mapping, user); - } else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) { + } else if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) { if (this.queueManager .getQueue(groups.getGroups(user).get(0)) != null) { return getPlacementContext(mapping, @@ -215,7 +140,7 @@ public class UserGroupMappingPlacementRule extends PlacementRule { } else { return null; } - } else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) { + } else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) { String secondaryGroup = getSecondaryGroup(user); if (secondaryGroup != null) { return getPlacementContext(mapping, secondaryGroup); @@ -226,10 +151,10 @@ public class UserGroupMappingPlacementRule extends PlacementRule { return getPlacementContext(mapping); } } - if (user.equals(mapping.source)) { - if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) { + if (user.equals(mapping.getSource())) { + if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) { return getPlacementContext(mapping, groups.getGroups(user).get(0)); - } else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) { + } else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) { String secondaryGroup = getSecondaryGroup(user); if (secondaryGroup != null) { return getPlacementContext(mapping, secondaryGroup); @@ -241,10 +166,10 @@ public class UserGroupMappingPlacementRule extends PlacementRule { } } } - if (mapping.type == MappingType.GROUP) { + if (mapping.getType() == MappingType.GROUP) { for (String userGroups : groups.getGroups(user)) { - if (userGroups.equals(mapping.source)) { - if (mapping.queue.equals(CURRENT_USER_MAPPING)) { + if (userGroups.equals(mapping.getSource())) { + if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) { return getPlacementContext(mapping, user); } return getPlacementContext(mapping); @@ -292,7 +217,7 @@ public class UserGroupMappingPlacementRule extends PlacementRule { private ApplicationPlacementContext getPlacementContext(QueueMapping mapping, String leafQueueName) { - if (!StringUtils.isEmpty(mapping.parentQueue)) { + if (!StringUtils.isEmpty(mapping.getParentQueue())) { return new ApplicationPlacementContext(leafQueueName, mapping.getParentQueue()); } else{ @@ -426,16 +351,24 @@ public class UserGroupMappingPlacementRule extends PlacementRule { && (queuePath.getParentQueue().equals(PRIMARY_GROUP_MAPPING) || queuePath.getParentQueue().equals(SECONDARY_GROUP_MAPPING))) { // dynamic parent queue - return new QueueMapping(mapping.getType(), mapping.getSource(), - queuePath.getLeafQueue(), queuePath.getParentQueue()); + return QueueMappingBuilder.create() + .type(mapping.getType()) + .source(mapping.getSource()) + .queue(queuePath.getLeafQueue()) + .parentQueue(queuePath.getParentQueue()) + .build(); } else if (queuePath.hasParentQueue()) { //if parent queue is specified, // then it should exist and be an instance of ManagedParentQueue QueuePlacementRuleUtils.validateQueueMappingUnderParentQueue( queueManager.getQueue(queuePath.getParentQueue()), queuePath.getParentQueue(), queuePath.getLeafQueue()); - return new QueueMapping(mapping.getType(), mapping.getSource(), - queuePath.getLeafQueue(), queuePath.getParentQueue()); + return QueueMappingBuilder.create() + .type(mapping.getType()) + .source(mapping.getSource()) + .queue(queuePath.getLeafQueue()) + .parentQueue(queuePath.getParentQueue()) + .build(); } return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index c1ce6da2203..d57e3919cdd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -39,8 +39,9 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AppPriorityACLConfigurationParser.AppPriorityACLKeyType; @@ -1119,10 +1120,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur throw new IllegalArgumentException( "unknown mapping prefix " + mapping[0]); } - m = new QueueMapping( - mappingType, - mapping[1], - mapping[2]); + m = QueueMappingBuilder.create() + .type(mappingType) + .source(mapping[1]) + .queue(mapping[2]) + .build(); } catch (Throwable t) { throw new IllegalArgumentException( "Illegal queue mapping " + mappingValue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementManager.java index db5cd60e580..3b85fdad08d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementManager.java @@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -70,11 +72,13 @@ public class TestPlacementManager { .getQueuePlacementManager(); List queuePlacementRules = new ArrayList<>(); - UserGroupMappingPlacementRule.QueueMapping userQueueMapping = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - USER1, - getQueueMapping(PARENT_QUEUE, USER1)); + QueueMapping userQueueMapping = QueueMappingBuilder.create() + .type(MappingType.USER) + .source(USER1) + .queue( + getQueueMapping(PARENT_QUEUE, + USER1)) + .build(); UserGroupMappingPlacementRule ugRule = new UserGroupMappingPlacementRule( false, Arrays.asList(userQueueMapping), null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java index 5cd6ea16786..e663a14c462 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java @@ -31,8 +31,9 @@ import org.apache.hadoop.security.NullGroupsMapping; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; +import org.apache.hadoop.yarn.server.resourcemanager.placement.TestUserGroupMappingPlacementRule.QueueMappingTestData.QueueMappingTestDataBuilder; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; @@ -52,21 +53,16 @@ public class TestUserGroupMappingPlacementRule { SimpleGroupsMapping.class, GroupMappingServiceProvider.class); } - private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, - String expectedQueue) throws YarnException { - verifyQueueMapping(queueMapping, inputUser, - YarnConfiguration.DEFAULT_QUEUE_NAME, expectedQueue, false); - } + private void verifyQueueMapping(QueueMappingTestData queueMappingTestData) + throws YarnException { - private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, - String inputQueue, String expectedQueue, boolean overwrite) throws YarnException { - verifyQueueMapping(queueMapping, inputUser, inputQueue, expectedQueue, - overwrite, null); - } + QueueMapping queueMapping = queueMappingTestData.queueMapping; + String inputUser = queueMappingTestData.inputUser; + String inputQueue = queueMappingTestData.inputQueue; + String expectedQueue = queueMappingTestData.expectedQueue; + boolean overwrite = queueMappingTestData.overwrite; + String expectedParentQueue = queueMappingTestData.expectedParentQueue; - private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, - String inputQueue, String expectedQueue, boolean overwrite, - String expectedParentQueue) throws YarnException { Groups groups = new Groups(conf); UserGroupMappingPlacementRule rule = new UserGroupMappingPlacementRule( overwrite, Arrays.asList(queueMapping), groups); @@ -109,16 +105,28 @@ public class TestUserGroupMappingPlacementRule { @Test public void testSecondaryGroupMapping() throws YarnException { verifyQueueMapping( - new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a", - "asubgroup2"); + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%secondary_group").build()) + .inputUser("a") + .expectedQueue("asubgroup2") + .build()); // PrimaryGroupMapping.class returns only primary group, no secondary groups conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, PrimaryGroupMapping.class, GroupMappingServiceProvider.class); - verifyQueueMapping( - new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a", - "default"); + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%secondary_group") + .build()) + .inputUser("a") + .expectedQueue("default") + .build()); } @Test @@ -127,8 +135,15 @@ public class TestUserGroupMappingPlacementRule { NullGroupsMapping.class, GroupMappingServiceProvider.class); try { verifyQueueMapping( - new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a", - "default"); + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%secondary_group") + .build()) + .inputUser("a") + .expectedQueue("default") + .build()); fail("No Groups for user 'a'"); } catch (YarnException e) { // Exception is expected as there are no groups for given user @@ -137,59 +152,253 @@ public class TestUserGroupMappingPlacementRule { @Test public void testMapping() throws YarnException { - - verifyQueueMapping(new QueueMapping(MappingType.USER, "a", "q1"), "a", "q1"); - verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", "q1"), - "a", "q1"); - verifyQueueMapping(new QueueMapping(MappingType.USER, "%user", "q2"), "a", - "q2"); - verifyQueueMapping(new QueueMapping(MappingType.USER, "%user", "%user"), - "a", "a"); verifyQueueMapping( - new QueueMapping(MappingType.USER, "%user", "%primary_group"), "a", - "agroup"); - // Queue "bgroup" is not configured, hence "default" should be used + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("a") + .queue("q1") + .build()) + .inputUser("a") + .expectedQueue("q1") + .build()); verifyQueueMapping( - new QueueMapping(MappingType.USER, "%user", "%primary_group"), "b", - "default"); + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.GROUP) + .source("agroup") + .queue("q1") + .build()) + .inputUser("a") + .expectedQueue("q1") + .build()); verifyQueueMapping( - new QueueMapping(MappingType.USER, "%user", "%user", "%primary_group"), - "a", YarnConfiguration.DEFAULT_QUEUE_NAME, "a", false, "agroup"); + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("q2") + .build()) + .inputUser("a") + .expectedQueue("q2") + .build()); verifyQueueMapping( - new QueueMapping(MappingType.USER, "%user", "%user", - "%secondary_group"), - "b", YarnConfiguration.DEFAULT_QUEUE_NAME, "b", false, "bsubgroup2"); - verifyQueueMapping(new QueueMapping(MappingType.GROUP, "asubgroup1", "q1"), - "a", "q1"); + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%user") + .build()) + .inputUser("a") + .expectedQueue("a") + .build()); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%primary_group") + .build()) + .inputUser("a") + .expectedQueue("agroup") + .build()); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%user") + .parentQueue("%primary_group") + .build()) + .inputUser("a") + .expectedQueue("a") + .expectedParentQueue("agroup") + .build()); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%user") + .parentQueue("%secondary_group") + .build()) + .inputUser("b") + .expectedQueue("b") + .expectedParentQueue("bsubgroup2") + .build()); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.GROUP) + .source("asubgroup1") + .queue("q1") + .build()) + .inputUser("a") + .expectedQueue("q1") + .build()); // specify overwritten, and see if user specified a queue, and it will be // overridden - verifyQueueMapping(new QueueMapping(MappingType.USER, "user", "q1"), "user", - "q2", "q1", true); - - // if overwritten not specified, it should be which user specified - verifyQueueMapping(new QueueMapping(MappingType.USER, "user", "q1"), "user", - "q2", "q2", false); + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("user") + .queue("q1") + .build()) + .inputUser("user") + .inputQueue("q2") + .expectedQueue("q1") + .overwrite(true) + .build()); - // if overwritten not specified, it should be which user specified - verifyQueueMapping( - new QueueMapping(MappingType.GROUP, "usergroup", "%user", "usergroup"), - "user", "default", "user", false); + // if overwritten not specified, it should be which user specified + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("user") + .queue("q1") + .build()) + .inputUser("user") + .inputQueue("q2") + .expectedQueue("q2") + .build()); - // if overwritten not specified, it should be which user specified - verifyQueueMapping( - new QueueMapping(MappingType.GROUP, "usergroup", "%user", "usergroup"), - "user", "agroup", "user", true); + // if overwritten not specified, it should be which user specified + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.GROUP) + .source("usergroup") + .queue("%user") + .parentQueue("usergroup") + .build()) + .inputUser("user") + .inputQueue("default") + .expectedQueue("user") + .build()); - //If user specific queue is enabled for a specified group under a given - // parent queue - verifyQueueMapping( - new QueueMapping(MappingType.GROUP, "agroup", "%user", "parent1"), "a", - "a"); + // if overwritten not specified, it should be which user specified + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.GROUP) + .source("usergroup") + .queue("%user") + .parentQueue("usergroup") + .build()) + .inputUser("user") + .inputQueue("agroup") + .expectedQueue("user") + .overwrite(true) + .build()); - //If user specific queue is enabled for a specified group without parent - // queue - verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", "%user"), - "a", "a"); + // If user specific queue is enabled for a specified group under a given + // parent queue + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.GROUP) + .source("agroup") + .queue("%user") + .parentQueue("parent1") + .build()) + .inputUser("a") + .expectedQueue("a") + .build()); + + // If user specific queue is enabled for a specified group without parent + // queue + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.GROUP) + .source("agroup") + .queue("%user") + .build()) + .inputUser("a") + .expectedQueue("a") + .build()); + } + + /** + * Queue Mapping test class to prepare the test data. + * + */ + public static final class QueueMappingTestData { + + private QueueMapping queueMapping; + private String inputUser; + private String inputQueue; + private String expectedQueue; + private boolean overwrite; + private String expectedParentQueue; + + private QueueMappingTestData(QueueMappingTestDataBuilder builder) { + this.queueMapping = builder.queueMapping; + this.inputUser = builder.inputUser; + this.inputQueue = builder.inputQueue; + this.expectedQueue = builder.expectedQueue; + this.overwrite = builder.overwrite; + this.expectedParentQueue = builder.expectedParentQueue; + } + + /** + * Builder class to prepare the Queue Mapping test data. + * + */ + public static class QueueMappingTestDataBuilder { + + private QueueMapping queueMapping = null; + private String inputUser = null; + private String inputQueue = YarnConfiguration.DEFAULT_QUEUE_NAME; + private String expectedQueue = null; + private boolean overwrite = false; + private String expectedParentQueue = null; + + public QueueMappingTestDataBuilder() { + + } + + public static QueueMappingTestDataBuilder create() { + return new QueueMappingTestDataBuilder(); + } + + public QueueMappingTestDataBuilder queueMapping(QueueMapping mapping) { + this.queueMapping = mapping; + return this; + } + + public QueueMappingTestDataBuilder inputUser(String user) { + this.inputUser = user; + return this; + } + + public QueueMappingTestDataBuilder inputQueue(String queue) { + this.inputQueue = queue; + return this; + } + + public QueueMappingTestDataBuilder expectedQueue(String outputQueue) { + this.expectedQueue = outputQueue; + return this; + } + + public QueueMappingTestDataBuilder overwrite(boolean overwriteMappings) { + this.overwrite = overwriteMappings; + return this; + } + + public QueueMappingTestDataBuilder expectedParentQueue( + String outputParentQueue) { + this.expectedParentQueue = outputParentQueue; + return this; + } + + public QueueMappingTestData build() { + return new QueueMappingTestData(this); + } + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index 9527e80dfa2..ae30e4a723b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -46,8 +46,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.placement .ApplicationPlacementContext; -import org.apache.hadoop.yarn.server.resourcemanager.placement - .UserGroupMappingPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; @@ -268,19 +268,19 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE); conf.setQueuePlacementRules(queuePlacementRules); - List existingMappings = - conf.getQueueMappings(); + List existingMappings = conf.getQueueMappings(); //set queue mapping - List queueMappings = - new ArrayList<>(); + List queueMappings = new ArrayList<>(); for (int i = 0; i < userIds.length; i++) { //Set C as parent queue name for auto queue creation - UserGroupMappingPlacementRule.QueueMapping userQueueMapping = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - USER + userIds[i], - getQueueMapping(parentQueue, USER + userIds[i])); + QueueMapping userQueueMapping = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source(USER + userIds[i]) + .queue( + getQueueMapping(parentQueue, + USER + userIds[i])) + .build(); queueMappings.add(userQueueMapping); } @@ -295,12 +295,10 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { (String parentQueue, CapacitySchedulerConfiguration conf, String leafQueueName) { - List existingMappings = - conf.getQueueMappings(); + List existingMappings = conf.getQueueMappings(); //set queue mapping - List queueMappings = - new ArrayList<>(); + List queueMappings = new ArrayList<>(); //setup group mapping conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, @@ -309,11 +307,13 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { TEST_GROUPUSER +"=" + TEST_GROUP + ";invalid_user=invalid_group"); Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); - UserGroupMappingPlacementRule.QueueMapping userQueueMapping = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.GROUP, - TEST_GROUP, - getQueueMapping(parentQueue, leafQueueName)); + QueueMapping userQueueMapping = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.GROUP) + .source(TEST_GROUP) + .queue( + getQueueMapping(parentQueue, + leafQueueName)) + .build(); queueMappings.add(userQueueMapping); existingMappings.addAll(queueMappings); @@ -514,13 +514,14 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { return rmApp.getApplicationId(); } - protected List setupQueueMapping( + protected List setupQueueMapping( CapacityScheduler newCS, String user, String parentQueue, String queue) { - List queueMappings = - new ArrayList<>(); - queueMappings.add(new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, user, - getQueueMapping(parentQueue, queue))); + List queueMappings = new ArrayList<>(); + queueMappings.add(QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source(user) + .queue(getQueueMapping(parentQueue, queue)) + .build()); newCS.getConfiguration().setQueueMappings(queueMappings); return queueMappings; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java index 79b6f249524..c2fea87c621 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java @@ -26,6 +26,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMappingEntity; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -62,19 +65,19 @@ public class TestCapacitySchedulerQueueMappingFactory { conf.setQueuePlacementRules(queuePlacementRules); - List existingMappingsForUG = - conf.getQueueMappings(); + List existingMappingsForUG = conf.getQueueMappings(); //set queue mapping - List queueMappingsForUG = - new ArrayList<>(); + List queueMappingsForUG = new ArrayList<>(); for (int i = 0; i < sourceIds.length; i++) { //Set C as parent queue name for auto queue creation - UserGroupMappingPlacementRule.QueueMapping userQueueMapping = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - USER + sourceIds[i], - getQueueMapping(parentQueue, USER + sourceIds[i])); + QueueMapping userQueueMapping = QueueMappingBuilder.create() + .type(MappingType.USER) + .source(USER + sourceIds[i]) + .queue( + getQueueMapping(parentQueue, + USER + sourceIds[i])) + .build(); queueMappingsForUG.add(userQueueMapping); } @@ -152,23 +155,25 @@ public class TestCapacitySchedulerQueueMappingFactory { queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); conf.setQueuePlacementRules(queuePlacementRules); - List existingMappingsForUG = - conf.getQueueMappings(); + List existingMappingsForUG = conf.getQueueMappings(); // set queue mapping - List queueMappingsForUG = - new ArrayList<>(); + List queueMappingsForUG = new ArrayList<>(); // u:user1:b1 - UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "user1", - "b1"); + QueueMapping userQueueMapping1 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("user1") + .queue("b1") + .build(); + // u:%user:parentqueue.%user - UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "%user", - getQueueMapping("c", "%user")); + QueueMapping userQueueMapping2 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("%user") + .queue(getQueueMapping("c", "%user")) + .build(); + queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping2); @@ -221,26 +226,32 @@ public class TestCapacitySchedulerQueueMappingFactory { */ // set queue mapping - List queueMappingsForUG = - new ArrayList<>(); + List queueMappingsForUG = new ArrayList<>(); // u:%user:%primary_group.%user - UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - "%user", getQueueMapping("%primary_group", "%user")); + QueueMapping userQueueMapping1 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("%user") + .queue( + getQueueMapping("%primary_group", + "%user")) + .build(); // u:%user:%secondary_group.%user - UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - "%user", getQueueMapping("%secondary_group", "%user")); + QueueMapping userQueueMapping2 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("%user") + .queue( + getQueueMapping("%secondary_group", + "%user")) + .build(); // u:b4:%secondary_group - UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - "b4", "%secondary_group"); + QueueMapping userQueueMapping3 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("b4") + .queue("%secondary_group") + .build(); queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping2); queueMappingsForUG.add(userQueueMapping3); @@ -276,20 +287,26 @@ public class TestCapacitySchedulerQueueMappingFactory { */ // set queue mapping - List queueMappingsForUG = - new ArrayList<>(); + List queueMappingsForUG = new ArrayList<>(); // u:%user:%primary_group.%user - UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - "%user", getQueueMapping("%primary_group", "%user")); + QueueMapping userQueueMapping1 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("%user") + .queue( + getQueueMapping("%primary_group", + "%user")) + .build(); // u:%user:%secondary_group.%user - UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - "%user", getQueueMapping("%secondary_group", "%user")); + QueueMapping userQueueMapping2 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("%user") + .queue( + getQueueMapping( + "%secondary_group", "%user") + ) + .build(); queueMappingsForUG.add(userQueueMapping2); queueMappingsForUG.add(userQueueMapping1); @@ -298,8 +315,7 @@ public class TestCapacitySchedulerQueueMappingFactory { } private void testNestedUserQueueWithDynamicParentQueue( - List mapping, boolean primary, - String user) + List mapping, boolean primary, String user) throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); @@ -312,8 +328,7 @@ public class TestCapacitySchedulerQueueMappingFactory { queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); conf.setQueuePlacementRules(queuePlacementRules); - List existingMappingsForUG = - conf.getQueueMappings(); + List existingMappingsForUG = conf.getQueueMappings(); existingMappingsForUG.addAll(mapping); conf.setQueueMappings(existingMappingsForUG); @@ -367,24 +382,24 @@ public class TestCapacitySchedulerQueueMappingFactory { queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); conf.setQueuePlacementRules(queuePlacementRules); - List existingMappingsForUG = - conf.getQueueMappings(); + List existingMappingsForUG = conf.getQueueMappings(); // set queue mapping - List queueMappingsForUG = - new ArrayList<>(); + List queueMappingsForUG = new ArrayList<>(); // u:user1:b1 - UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - "user1", "b1"); + QueueMapping userQueueMapping1 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("user1") + .queue("b1") + .build(); // u:user2:%primary_group - UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - "user2", "%primary_group"); + QueueMapping userQueueMapping2 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("user2") + .queue("%primary_group") + .build(); queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping2); @@ -436,30 +451,31 @@ public class TestCapacitySchedulerQueueMappingFactory { queuePlacementRules.add(QUEUE_MAPPING_RULE_USER_GROUP); conf.setQueuePlacementRules(queuePlacementRules); - List existingMappingsForUG = - conf.getQueueMappings(); + List existingMappingsForUG = conf.getQueueMappings(); // set queue mapping - List queueMappingsForUG = - new ArrayList<>(); + List queueMappingsForUG = new ArrayList<>(); // u:user1:b1 - UserGroupMappingPlacementRule.QueueMapping userQueueMapping1 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - "user1", "b1"); + QueueMapping userQueueMapping1 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("user1") + .queue("b1") + .build(); // u:user2:%primary_group - UserGroupMappingPlacementRule.QueueMapping userQueueMapping2 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, - "user2", "%primary_group"); + QueueMapping userQueueMapping2 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("user2") + .queue("%primary_group") + .build(); // u:b4:%secondary_group - UserGroupMappingPlacementRule.QueueMapping userQueueMapping3 = - new UserGroupMappingPlacementRule.QueueMapping( - UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, "b4", - "%secondary_group"); + QueueMapping userQueueMapping3 = QueueMappingBuilder.create() + .type(QueueMapping.MappingType.USER) + .source("b4") + .queue("%secondary_group") + .build(); queueMappingsForUG.add(userQueueMapping1); queueMappingsForUG.add(userQueueMapping2); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java index fbf092f334c..2e7009eae65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java @@ -23,9 +23,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -91,7 +92,12 @@ public class TestQueueMappings { // space trimming conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, " u : a : " + Q1); cs.reinitialize(conf, null); - checkQMapping(new QueueMapping(MappingType.USER, "a", Q1)); + checkQMapping( + QueueMappingBuilder.create() + .type(MappingType.USER) + .source("a") + .queue(Q1) + .build()); } @Test (timeout = 60000)