From 807d7aa3aff24032fe001b1ecceb07ea2d4a3b18 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Thu, 2 Jan 2020 14:40:55 +0530 Subject: [PATCH] YARN-10053. Use Shared Group Mapping Service in Placement Rules. Contributed by Wilfred Spiegelenburg. (cherry picked from commit 217b56ffdd5fa254f06734bc8cb6f04a02066f1a) --- .../placement/UserGroupMappingPlacementRule.java | 4 ++-- .../scheduler/fair/QueuePlacementPolicy.java | 2 +- .../scheduler/capacity/TestCapacityScheduler.java | 5 ++--- ...TestCapacitySchedulerAutoCreatedQueueBase.java | 5 ++--- .../scheduler/fair/TestFairScheduler.java | 15 +++++++++++++++ .../scheduler/fair/TestQueuePlacementPolicy.java | 7 +++++-- 6 files changed, 27 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java index c9bf1029371..022172876d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java @@ -313,9 +313,9 @@ public class UserGroupMappingPlacementRule extends PlacementRule { // initialize groups if mappings are present if (newMappings.size() > 0) { - Groups groups = new Groups(conf); this.mappings = newMappings; - this.groups = groups; + this.groups = Groups.getUserToGroupsMappingService( + ((CapacityScheduler)scheduler).getConfig()); this.overrideWithQueueMappings = overrideWithQueueMappings; return true; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java index 30ea213529d..ecbd170925e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java @@ -72,7 +72,7 @@ public class QueuePlacementPolicy { } this.rules = rules; this.configuredQueues = configuredQueues; - groups = new Groups(conf); + this.groups = Groups.getUserToGroupsMappingService(conf); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 4b727cea870..6e234f11026 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -47,6 +47,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.TestGroupsCaching; import org.apache.hadoop.security.UserGroupInformation; @@ -107,9 +108,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMW import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; - -import org.apache.hadoop.yarn.server.resourcemanager.placement - .UserGroupMappingPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; @@ -4821,6 +4819,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { "a1" +"=" + "agroup" + ""); config.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, "g:agroup:%user"); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(config); MockRM rm = new MockRM(config); rm.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index addec662862..6e22608b9ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -23,7 +23,7 @@ import org.apache.commons.lang.math.RandomUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.TestGroupsCaching; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -63,8 +63,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event .AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event .SchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .SimpleGroupsMapping; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.YarnVersionInfo; @@ -296,6 +294,7 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { TestGroupsCaching.FakeunPrivilegedGroupMapping.class, ShellBasedUnixGroupsMapping.class); conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES, TEST_GROUPUSER +"=" + TEST_GROUP + ";invalid_user=invalid_group"); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); UserGroupMappingPlacementRule.QueueMapping userQueueMapping = new UserGroupMappingPlacementRule.QueueMapping( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index d8da2da5e5d..60b7344fb62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -51,6 +51,8 @@ import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -1806,6 +1808,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { public void testQueuePlacementWithPolicy() throws Exception { conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -1851,6 +1854,12 @@ public class TestFairScheduler extends FairSchedulerTestBase { assertEquals("root.somequeue", scheduler.getSchedulerApp(appId).getQueueName()); appId = createSchedulingRequest(1024, "default", "otheruser"); assertEquals("root.default", scheduler.getSchedulerApp(appId).getQueueName()); + + // undo the group change + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + JniBasedUnixGroupsMappingWithFallback.class, + GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); } @Test @@ -1903,6 +1912,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println(""); out.println(""); @@ -1928,6 +1938,11 @@ public class TestFairScheduler extends FairSchedulerTestBase { "user1"); assertEquals("root.user1group.user1", user1Leaf.getName()); + // undo the group change + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + JniBasedUnixGroupsMappingWithFallback.class, + GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java index 3fe9ce31021..47d58b3da7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java @@ -20,10 +20,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import static org.junit.Assert.*; import java.io.IOException; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; @@ -34,6 +32,7 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.apache.hadoop.security.Groups; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -389,8 +388,10 @@ public class TestQueuePlacementPolicy { sb.append(" "); sb.append(""); + // change the group resolution conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, PeriodGroupsMapping.class, GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); // User queue would be created under primary group queue, and the period // in the group name should be converted into _dot_ QueuePlacementPolicy policy = parse(sb.toString()); @@ -399,6 +400,7 @@ public class TestQueuePlacementPolicy { conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); } @Test(expected=IOException.class) @@ -412,6 +414,7 @@ public class TestQueuePlacementPolicy { // Add a static mapping that returns empty groups for users conf.setStrings(CommonConfigurationKeys .HADOOP_USER_GROUP_STATIC_OVERRIDES, "emptygroupuser="); + Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf); QueuePlacementPolicy policy = parse(sb.toString()); policy.assignAppToQueue(null, "emptygroupuser"); }