From 288dab75633b92f3104e61d3163e51d1f1263041 Mon Sep 17 00:00:00 2001 From: Peter Bacsko Date: Thu, 3 Sep 2020 15:02:48 +0200 Subject: [PATCH] YARN-10372. Create MappingRule class to represent each CS mapping rule. Contributed by Gergely Pollak --- .../placement/MappingRule.java | 136 +++++++++++++++++ .../placement/TestMappingRule.java | 143 ++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRule.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java new file mode 100644 index 00000000000..e03be2a46bc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + * Mapping rule represents a single mapping setting defined by the user. All + * rules have matchers and actions. Matcher determine if a mapping rule applies + * to a given applicationSubmission, while action represent the course of action + * we need to take when a rule applies. + * + * MappingRules also support fallback actions, which will be evaluated when the + * main action fails due to any reason (Eg. trying to place to a queue which + * does not exist) + */ +public class MappingRule { + public static final String USER_MAPPING = "u"; + public static final String GROUP_MAPPING = "g"; + public static final String APPLICATION_MAPPING = "a"; + private final MappingRuleMatcher matcher; + private final MappingRuleAction action; + + public MappingRule(MappingRuleMatcher matcher, MappingRuleAction action) { + this.matcher = matcher; + this.action = action; + } + + /** + * This method evaluates the rule, and returns the MappingRuleResult, if + * the rule matches, skip action otherwise. + * @param variables The variable context, which contains all the variables + * @return The rule's result or skip action if the rule doesn't apply + */ + public MappingRuleResult evaluate(VariableContext variables) { + if (matcher.match(variables)) { + return action.execute(variables); + } + + return MappingRuleResult.createSkipResult(); + } + + /** + * Returns the associated action's fallback. + * @return The fallback of the action + */ + public MappingRuleResult getFallback() { + return action.getFallback(); + } + + /** + * Creates a MappingRule object from the legacy style configuration. The + * configuration is a [TYPE]:SOURCE:PATH (eg. u:bob:root.users.%user). + * Using the source and path parts of the legacy rule, this method will + * create an application MappingRule which behaves as the legacy rule defined. + * This version of the method does not require type, since legacy application + * mappings omitted the 'a', so this method is to be used for those rules, + * which in all case are application mappings. + * @param source This part of the rule determines which applications the rule + * will be applied + * @param path The path where the application is to be placed + * @return MappingRule based on the provided settings + */ + public static MappingRule createLegacyRule(String source, String path) { + return createLegacyRule(APPLICATION_MAPPING, source, path); + } + + /** + * Creates a MappingRule object from the legacy style configuration. The + * configuration is a [TYPE]:SOURCE:PATH (eg. u:bob:root.users.%user). + * Using the type, source and path parts of the legacy rule, this method will + * create a MappingRule which behaves as the legacy rule defined. + * @param type The type of the rule, can be + * 'u' for user mapping, 'g' for group mapping or + * 'a' for application mapping + * @param source This part of the rule determines which submissions this rule + * should apply to (eg. if type is 'u', source will match + * against the user name) + * @param path The path where the application is to be placed + * @return MappingRule based on the provided settings + */ + public static MappingRule createLegacyRule( + String type, String source, String path) { + MappingRuleMatcher matcher; + MappingRuleAction action = new MappingRuleActions.PlaceToQueueAction(path); + //While legacy rule fallback handling is a bit inconsistent, the most cases + //it fall back to default queue placement, so this is the best approximation + action.setFallbackDefaultPlacement(); + + switch (type) { + case USER_MAPPING: + matcher = MappingRuleMatchers.createUserMatcher(source); + break; + case GROUP_MAPPING: + matcher = MappingRuleMatchers.createGroupMatcher(source); + break; + case APPLICATION_MAPPING: + matcher = MappingRuleMatchers.createApplicationNameMatcher(source); + break; + default: + throw new IllegalArgumentException("Invalid mapping rule type '" + + type + "'"); + } + + return new MappingRule(matcher, action); + } + + public void validate(MappingRuleValidationContext ctx) + throws YarnException { + this.action.validate(ctx); + } + + @Override + public String toString() { + return "MappingRule{" + + "matcher=" + matcher + + ", action=" + action + + '}'; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRule.java new file mode 100644 index 00000000000..2fa3140c797 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRule.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.util.StringUtils; +import org.junit.Test; + +public class TestMappingRule { + VariableContext setupVariables( + String user, String group, String secGroup, String appName) { + VariableContext variables = new VariableContext(); + variables.put("%default", "root.default"); + variables.put("%user", user); + variables.put("%primary_group", group); + variables.put("%secondary_group", secGroup); + variables.put("%application", appName); + variables.put("%sub", "xxx"); + variables.put("%empty", ""); + variables.put("%null", null); + variables.setImmutables("%user", "%primary_group", "%secondary_group", + "%application"); + + return variables; + } + + void assertSkipResult(MappingRuleResult result) { + assertTrue( + MappingRuleResultType.SKIP == result.getResult()); + } + + void assertPlaceResult(MappingRuleResult result, String queue) { + assertTrue( + MappingRuleResultType.PLACE == result.getResult()); + assertEquals(queue, result.getQueue()); + } + + @Test + public void testMappingRuleEvaluation() { + VariableContext matching = setupVariables( + "bob", "developer", "users", "MR"); + VariableContext mismatching = setupVariables( + "joe", "tester", "admins", "Spark"); + + MappingRule rule = new MappingRule( + MappingRuleMatchers.createUserMatcher("bob"), + (new MappingRuleActions.PlaceToQueueAction("%default.%default")) + .setFallbackSkip() + ); + + assertSkipResult(rule.getFallback()); + + MappingRuleResult matchingResult = rule.evaluate(matching); + MappingRuleResult mismatchingResult = rule.evaluate(mismatching); + + assertSkipResult(mismatchingResult); + assertPlaceResult(matchingResult, "root.default.root.default"); + } + + MappingRule createMappingRuleFromLegacyString(String legacyMapping) { + String[] mapping = + StringUtils + .getTrimmedStringCollection(legacyMapping, ":") + .toArray(new String[] {}); + + if (mapping.length == 2) { + return MappingRule.createLegacyRule(mapping[0], mapping[1]); + } + + return MappingRule.createLegacyRule(mapping[0], mapping[1], mapping[2]); + } + + void evaluateLegacyStringTestcase( + String legacyString, VariableContext variables, String expectedQueue) { + MappingRule rule = createMappingRuleFromLegacyString(legacyString); + MappingRuleResult result = rule.evaluate(variables); + assertEquals( + rule.getFallback().getResult(), MappingRuleResultType.PLACE_TO_DEFAULT); + + if (expectedQueue == null) { + assertSkipResult(result); + return; + } + + assertPlaceResult(result, expectedQueue); + } + + @Test + public void testLegacyEvaluation() { + VariableContext matching = setupVariables( + "bob", "developer", "users", "MR"); + VariableContext mismatching = setupVariables( + "joe", "tester", "admins", "Spark"); + + evaluateLegacyStringTestcase( + "u:bob:root.%primary_group", matching, "root.developer"); + evaluateLegacyStringTestcase( + "u:bob:root.%primary_group", mismatching, null); + evaluateLegacyStringTestcase( + "g:developer:%secondary_group.%user", matching, "users.bob"); + evaluateLegacyStringTestcase( + "g:developer:%secondary_group.%user", mismatching, null); + evaluateLegacyStringTestcase( + "MR:root.static", matching, "root.static"); + evaluateLegacyStringTestcase( + "MR:root.static", mismatching, null); + + //catch all tests + evaluateLegacyStringTestcase( + "u:%user:root.%primary_group", matching, "root.developer"); + evaluateLegacyStringTestcase( + "u:%user:root.%primary_group", mismatching, "root.tester"); + } + + @Test + public void testToStrings() { + MappingRuleAction action = new MappingRuleActions.PlaceToQueueAction( + "queue"); + MappingRuleMatcher matcher = MappingRuleMatchers.createUserMatcher("bob"); + MappingRule rule = new MappingRule(matcher, action); + + assertEquals("MappingRule{matcher=" + matcher.toString() + + ", action=" + action.toString() + "}", rule.toString()); + } +} \ No newline at end of file