From a8ef03e96183fad0ecc53b89f1f58aaa83a92ad8 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Tue, 15 Oct 2019 17:22:52 +0200 Subject: [PATCH] YARN-9840. Capacity scheduler: add support for Secondary Group rule mapping. Contributed by Manikandan R --- .../placement/QueuePlacementRuleUtils.java | 5 +- .../UserGroupMappingPlacementRule.java | 43 ++++++++++++++--- .../TestUserGroupMappingPlacementRule.java | 27 ++++++++++- .../scheduler/fair/PrimaryGroupMapping.java | 47 +++++++++++++++++++ .../src/site/markdown/CapacityScheduler.md | 4 +- 5 files changed, 116 insertions(+), 10 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java index adee5d7e79a..6f2ee33f86b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java @@ -36,6 +36,8 @@ public final class QueuePlacementRuleUtils { public static final String PRIMARY_GROUP_MAPPING = "%primary_group"; + public static final String SECONDARY_GROUP_MAPPING = "%secondary_group"; + private QueuePlacementRuleUtils() { } @@ -100,7 +102,8 @@ public static QueueMappingEntity validateAndGetQueueMapping( public static boolean isStaticQueueMapping(QueueMappingEntity mapping) { return !mapping.getQueue().contains(CURRENT_USER_MAPPING) && !mapping - .getQueue().contains(PRIMARY_GROUP_MAPPING); + .getQueue().contains(PRIMARY_GROUP_MAPPING) + && !mapping.getQueue().contains(SECONDARY_GROUP_MAPPING); } public static QueuePath extractQueuePath(String queueName) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java index 43a3043b4d0..71541e35385 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java @@ -54,9 +54,12 @@ public class UserGroupMappingPlacementRule extends PlacementRule { public static final String PRIMARY_GROUP_MAPPING = "%primary_group"; + public static final String SECONDARY_GROUP_MAPPING = "%secondary_group"; + private boolean overrideWithQueueMappings = false; private List mappings = null; private Groups groups; + private CapacitySchedulerQueueManager queueManager; @Private public static class QueueMapping { @@ -163,6 +166,27 @@ private ApplicationPlacementContext getPlacementForUser(String user) return getPlacementContext(mapping, user); } else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) { return getPlacementContext(mapping, groups.getGroups(user).get(0)); + } else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) { + List groupsList = groups.getGroups(user); + String secondaryGroup = null; + // Traverse all secondary groups (as there could be more than one + // and position is not guaranteed) and ensure there is queue with + // the same name + for (int i = 1; i < groupsList.size(); i++) { + if (this.queueManager.getQueue(groupsList.get(i)) != null) { + secondaryGroup = groupsList.get(i); + break; + } + } + if (secondaryGroup != null) { + return getPlacementContext(mapping, secondaryGroup); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("User {} is not associated with any Secondary " + + "Group. Hence it may use the 'default' queue", user); + } + return null; + } } else { return getPlacementContext(mapping); } @@ -251,8 +275,7 @@ public boolean initialize(ResourceScheduler scheduler) // Get new user/group mappings List newMappings = new ArrayList<>(); - CapacitySchedulerQueueManager queueManager = - schedulerContext.getCapacitySchedulerQueueManager(); + queueManager = schedulerContext.getCapacitySchedulerQueueManager(); // check if mappings refer to valid queues for (QueueMapping mapping : queueMappings) { @@ -365,10 +388,12 @@ private static QueueMapping validateAndGetAutoCreatedQueueMapping( } private static boolean isStaticQueueMapping(QueueMapping mapping) { - return !mapping.getQueue().contains( - UserGroupMappingPlacementRule.CURRENT_USER_MAPPING) && !mapping - .getQueue().contains( - UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING); + return !mapping.getQueue() + .contains(UserGroupMappingPlacementRule.CURRENT_USER_MAPPING) + && !mapping.getQueue() + .contains(UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING) + && !mapping.getQueue() + .contains(UserGroupMappingPlacementRule.SECONDARY_GROUP_MAPPING); } private static class QueuePath { @@ -443,4 +468,10 @@ private static void validateParentQueue(CSQueue parentQueue, public List getQueueMappings() { return mappings; } + + @VisibleForTesting + @Private + public void setQueueManager(CapacitySchedulerQueueManager queueManager) { + this.queueManager = queueManager; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java index a6143ba6cbb..589e180ec32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.placement; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.util.Arrays; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -28,6 +31,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.PrimaryGroupMapping; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SimpleGroupsMapping; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; @@ -54,6 +60,10 @@ private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, Groups groups = new Groups(conf); UserGroupMappingPlacementRule rule = new UserGroupMappingPlacementRule( overwrite, Arrays.asList(queueMapping), groups); + CapacitySchedulerQueueManager queueManager = + mock(CapacitySchedulerQueueManager.class); + when(queueManager.getQueue("asubgroup2")).thenReturn(mock(CSQueue.class)); + rule.setQueueManager(queueManager); ApplicationSubmissionContext asc = Records.newRecord( ApplicationSubmissionContext.class); asc.setQueue(inputQueue); @@ -62,9 +72,24 @@ private void verifyQueueMapping(QueueMapping queueMapping, String inputUser, ctx != null ? ctx.getQueue() : inputQueue); } + @Test + public void testSecondaryGroupMapping() throws YarnException { + verifyQueueMapping( + new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a", + "asubgroup2"); + + // PrimaryGroupMapping.class returns only primary group, no secondary groups + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + PrimaryGroupMapping.class, GroupMappingServiceProvider.class); + + verifyQueueMapping( + new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a", + "default"); + } + @Test public void testMapping() throws YarnException { - // simple base case for mapping user to queue + verifyQueueMapping(new QueueMapping(MappingType.USER, "a", "q1"), "a", "q1"); verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", "q1"), "a", "q1"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java new file mode 100644 index 00000000000..11415b0f757 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import org.apache.hadoop.security.GroupMappingServiceProvider; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +/** + * Group Mapping class used for test cases. Returns only primary group of the + * given user + */ +public class PrimaryGroupMapping implements GroupMappingServiceProvider { + + @Override + public List getGroups(String user) { + return Arrays.asList(user + "group"); + } + + @Override + public void cacheGroupsRefresh() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void cacheGroupsAdd(List groups) throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 33d2b13e06f..511a4857986 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -161,7 +161,7 @@ Configuration | Property | Description | |:---- |:---- | -| `yarn.scheduler.capacity.queue-mappings` | This configuration specifies the mapping of user or group to a specific queue. You can map a single user or a list of users to queues. Syntax: `[u or g]:[name]:[queue_name][,next_mapping]*`. Here, *u or g* indicates whether the mapping is for a user or group. The value is *u* for user and *g* for group. *name* indicates the user name or group name. To specify the user who has submitted the application, %user can be used. *queue_name* indicates the queue name for which the application has to be mapped. To specify queue name same as user name, *%user* can be used. To specify queue name same as the name of the primary group for which the user belongs to, *%primary_group* can be used.| +| `yarn.scheduler.capacity.queue-mappings` | This configuration specifies the mapping of user or group to a specific queue. You can map a single user or a list of users to queues. Syntax: `[u or g]:[name]:[queue_name][,next_mapping]*`. Here, *u or g* indicates whether the mapping is for a user or group. The value is *u* for user and *g* for group. *name* indicates the user name or group name. To specify the user who has submitted the application, %user can be used. *queue_name* indicates the queue name for which the application has to be mapped. To specify queue name same as user name, *%user* can be used. To specify queue name same as the name of the primary group for which the user belongs to, *%primary_group* can be used. Secondary group can be referenced as *%secondary_group* | | `yarn.scheduler.queue-placement-rules.app-name` | This configuration specifies the mapping of application_name to a specific queue. You can map a single application or a list of applications to queues. Syntax: `[app_name]:[queue_name][,next_mapping]*`. Here, *app_name* indicates the application name you want to do the mapping. *queue_name* indicates the queue name for which the application has to be mapped. To specify the current application's name as the app_name, %application can be used.| | `yarn.scheduler.capacity.queue-mappings-override.enable` | This function is used to specify whether the user specified queues can be overridden. This is a Boolean value and the default value is *false*. | @@ -170,7 +170,7 @@ Example: ``` yarn.scheduler.capacity.queue-mappings - u:user1:queue1,g:group1:queue2,u:%user:%user,u:user2:%primary_group + u:user1:queue1,g:group1:queue2,u:%user:%user,u:user2:%primary_group,u:user3:%secondary_group Here, is mapped to , is mapped to , maps users to queues with the same name as user, is mapped