From f276f1af80b4edb74b98790b899264213e785107 Mon Sep 17 00:00:00 2001 From: Wilfred Spiegelenburg Date: Wed, 17 Mar 2021 10:47:49 +1100 Subject: [PATCH] YARN-10652. Capacity Scheduler fails to handle user weights for a user that has a "." (dot) in it This only fixes the user name resolution for weights in the queues. It does not add generic support for user names with dots in all use cases in the capacity scheduler. Contributed by: Siddharth Ahuja --- .../capacity/CapacitySchedulerConfiguration.java | 2 +- .../scheduler/capacity/TestLeafQueue.java | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index ce3c0cb3e0c..efd56a24919 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -1963,7 +1963,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur getQueuePrefix(queuePath).replaceAll("\\.", "\\\\.") + USER_SETTINGS + "\\."; String weightKeyRegex = - qPathPlusPrefix + "\\w+\\." + USER_WEIGHT; + qPathPlusPrefix + "\\S+\\." + USER_WEIGHT; Map props = getValByRegex(weightKeyRegex); for (Entry e : props.entrySet()) { String userName = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 4b023107b39..75ab16f3a84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -1660,17 +1660,25 @@ public class TestLeafQueue { LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); // Set minimum-user-limit-percent for queue "a" in the configs. csConf.setUserLimit(a.getQueuePath(), 50); - // Set weight for "user_0" to be 1.5 for the a queue in the configs. + // Set weight for "user_0" to be 1.5f for the a queue in the configs. csConf.setFloat("yarn.scheduler.capacity." + a.getQueuePath() + ".user-settings.user_0." + CapacitySchedulerConfiguration.USER_WEIGHT, 1.5f); + // Set weight for "firstname.lastname" to be 0.7f for the a queue + // in the configs. Notice the user contains a dot. This is to test + // that weights are accepted for a username that contains dots. + csConf.setFloat("yarn.scheduler.capacity." + a.getQueuePath() + + ".user-settings.firstname.lastname." + + CapacitySchedulerConfiguration.USER_WEIGHT, + 0.7f); when(csContext.getClusterResource()) .thenReturn(Resources.createResource(16 * GB, 32)); // Verify that configs were updated and parsed correctly. Assert.assertNull(a.getUserWeights().get("user_0")); a.reinitialize(a, csContext.getClusterResource()); - assertEquals(1.5, a.getUserWeights().get("user_0").floatValue(), 0.0); + assertEquals(1.5f, a.getUserWeights().get("user_0"), 0.0f); + assertEquals(0.7f, a.getUserWeights().get("firstname.lastname"), 0.0f); // set maxCapacity a.setMaxCapacity(1.0f);