From 56a5c360a101470b56e40a021891b73a26e6db73 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Sat, 29 Aug 2020 21:34:55 +0200 Subject: [PATCH] YARN-10373. Create Matchers for CS mapping rules. Contributed by Gergely Pollak --- .../placement/MappingRuleMatcher.java | 28 ++ .../placement/MappingRuleMatchers.java | 239 ++++++++++++++++ .../placement/TestMappingRuleMatchers.java | 255 ++++++++++++++++++ 3 files changed, 522 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatcher.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatchers.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleMatchers.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatcher.java new file mode 100644 index 00000000000..d2650e9b478 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatcher.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +public interface MappingRuleMatcher { + /** + * Returns true if the matcher matches the current context. + * @param variables The variable context, which contains all the variables + * @return true if this matcher matches to the provided variable set + */ + boolean match(VariableContext variables); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatchers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatchers.java new file mode 100644 index 00000000000..fdc239f1f01 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRuleMatchers.java @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import java.util.Arrays; + +/** + * This class contains all the matcher and some helper methods to generate them. + */ +public class MappingRuleMatchers { + /** + * Utility class, hiding constructor. + */ + private MappingRuleMatchers() {} + + /** + * MatchAllMatcher is a matcher which matches everything. + */ + public static class MatchAllMatcher implements MappingRuleMatcher { + /** + * The match will return true in all cases, to match all submissions. + * @param variables The variable context, which contains all the variables + * @return true + */ + @Override + public boolean match(VariableContext variables) { + return true; + } + + @Override + public String toString() { + return "MatchAllMatcher"; + } + } + + /** + * VariableMatcher will check if a provided variable's value matches the + * provided value. The provided value might contain variables as well, which + * will get evaluated before the comparison. + */ + public static class VariableMatcher implements MappingRuleMatcher { + /** + * Name of the variable to be checked. + */ + private String variable; + /** + * The value which should match the variable's value. + */ + private String value; + + VariableMatcher(String variable, String value) { + this.variable = variable; + this.value = value == null ? "" : value; + } + + /** + * The method will replace all variables in the value, then compares this + * substituted value against the variable's value, if they match we return + * true. + * If the variable is null we always return false. + * @param variables The variable context, which contains all the variables + * @return true if the value matches the variable's value, false otherwise + */ + @Override + public boolean match(VariableContext variables) { + if (variable == null) { + return false; + } + + String substituted = variables.replaceVariables(value); + return substituted.equals(variables.get(variable)); + } + + @Override + public String toString() { + return "VariableMatcher{" + + "variable='" + variable + '\'' + + ", value='" + value + '\'' + + '}'; + } + } + + /** + * AndMatcher is a basic boolean matcher which takes multiple other + * matcher as it's arguments, and on match it checks if all of them are true. + */ + public static class AndMatcher implements MappingRuleMatcher { + /** + * The list of matchers to be checked during evaluation. + */ + private MappingRuleMatcher[] matchers; + + /** + * Constructor. + * @param matchers List of matchers to be checked during evaluation + */ + AndMatcher(MappingRuleMatcher...matchers) { + this.matchers = matchers; + } + + /** + * This match method will go through all the provided matchers and call + * their match method, if all match we return true. + * @param variables The variable context, which contains all the variables + * @return true if all matchers match + */ + @Override + public boolean match(VariableContext variables) { + for (MappingRuleMatcher matcher : matchers) { + if (!matcher.match(variables)) { + return false; + } + } + + return true; + } + + @Override + public String toString() { + return "AndMatcher{" + + "matchers=" + Arrays.toString(matchers) + + '}'; + } + } + + /** + * OrMatcher is a basic boolean matcher which takes multiple other + * matcher as its arguments, and on match it checks if any of them are true. + */ + public static class OrMatcher implements MappingRuleMatcher { + /** + * The list of matchers to be checked during evaluation. + */ + private MappingRuleMatcher[] matchers; + + /** + * Constructor. + * @param matchers List of matchers to be checked during evaluation + */ + OrMatcher(MappingRuleMatcher...matchers) { + this.matchers = matchers; + } + + /** + * This match method will go through all the provided matchers and call + * their match method, if any of them match we return true. + * @param variables The variable context, which contains all the variables + * @return true if any of the matchers match + */ + @Override + public boolean match(VariableContext variables) { + for (MappingRuleMatcher matcher : matchers) { + if (matcher.match(variables)) { + return true; + } + } + + return false; + } + + @Override + public String toString() { + return "OrMatcher{" + + "matchers=" + Arrays.toString(matchers) + + '}'; + } + } + + /** + * Convenience method to create a variable matcher which matches against the + * username. + * @param userName The username to be matched + * @return VariableMatcher with %user as the variable + */ + public static MappingRuleMatcher createUserMatcher(String userName) { + return new VariableMatcher("%user", userName); + } + + /** + * Convenience method to create a variable matcher which matches against the + * user's primary group. + * @param groupName The groupName to be matched + * @return VariableMatcher with %primary_group as the variable + */ + public static MappingRuleMatcher createGroupMatcher(String groupName) { + return new VariableMatcher("%primary_group", groupName); + } + + /** + * Convenience method to create a composite matcher which matches against the + * user's user name and the user's primary group. Only matches if both + * matches. + * @param userName The username to be matched + * @param groupName The groupName to be matched + * @return AndMatcher with two matchers one for userName and one for + * primaryGroup + */ + public static MappingRuleMatcher createUserGroupMatcher( + String userName, String groupName) { + return new AndMatcher( + createUserMatcher(userName), + createGroupMatcher(groupName)); + } + + /** + * Convenience method to create a variable matcher which matches against the + * submitted application's name. + * @param name The name to be matched + * @return VariableMatcher with %application as the variable + */ + public static MappingRuleMatcher createApplicationNameMatcher(String name) { + return new VariableMatcher("%application", name); + } + + + /** + * Convenience method to create a matcher that matches all + * @return MatchAllMatcher. + */ + public static MappingRuleMatcher createAllMatcher() { + return new MatchAllMatcher(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleMatchers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleMatchers.java new file mode 100644 index 00000000000..d384f93884a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestMappingRuleMatchers.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.placement; + +import junit.framework.TestCase; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +public class TestMappingRuleMatchers extends TestCase { + + @Test + public void testCatchAll() { + VariableContext variables = new VariableContext(); + MappingRuleMatcher matcher = new MappingRuleMatchers.MatchAllMatcher(); + + assertTrue(matcher.match(variables)); + } + + @Test + public void testVariableMatcher() { + VariableContext matchingContext = new VariableContext(); + matchingContext.put("%user", "bob"); + matchingContext.put("%primary_group", "developers"); + matchingContext.put("%application", "TurboMR"); + matchingContext.put("%custom", "Matching string"); + + VariableContext mismatchingContext = new VariableContext(); + mismatchingContext.put("%user", "dave"); + mismatchingContext.put("%primary_group", "testers"); + mismatchingContext.put("%application", "Tester APP"); + mismatchingContext.put("%custom", "Not matching string"); + + VariableContext emptyContext = new VariableContext(); + + Map matchers = new HashMap<>(); + matchers.put("User matcher", MappingRuleMatchers.createUserMatcher("bob")); + matchers.put("Group matcher", + MappingRuleMatchers.createGroupMatcher("developers")); + matchers.put("Application name matcher", + MappingRuleMatchers.createApplicationNameMatcher("TurboMR")); + matchers.put("Custom matcher", + new MappingRuleMatchers.VariableMatcher("%custom", "Matching string")); + + matchers.forEach((matcherName, matcher) -> { + assertTrue(matcherName + " with matchingContext should match", + matcher.match(matchingContext)); + assertFalse(matcherName + " with mismatchingContext shouldn't match", + matcher.match(mismatchingContext)); + assertFalse(matcherName + " with emptyContext shouldn't match", + matcher.match(emptyContext)); + }); + } + + @Test + public void testVariableMatcherSubstitutions() { + VariableContext matchingContext = new VariableContext(); + matchingContext.put("%user", "bob"); + matchingContext.put("%primary_group", "developers"); + matchingContext.put("%application", "TurboMR"); + matchingContext.put("%custom", "bob"); + matchingContext.put("%cus", "b"); + matchingContext.put("%tom", "ob"); + + VariableContext mismatchingContext = new VariableContext(); + mismatchingContext.put("%user", "dave"); + mismatchingContext.put("%primary_group", "testers"); + mismatchingContext.put("%application", "Tester APP"); + mismatchingContext.put("%custom", "bob"); + mismatchingContext.put("%cus", "b"); + mismatchingContext.put("%tom", "ob"); + + MappingRuleMatcher customUser = + new MappingRuleMatchers.VariableMatcher("%custom", "%user"); + + MappingRuleMatcher userCusTom = + new MappingRuleMatchers.VariableMatcher("%user", "%cus%tom"); + + MappingRuleMatcher userCustom = + new MappingRuleMatchers.VariableMatcher("%user", "%custom"); + + MappingRuleMatcher userUser = + new MappingRuleMatchers.VariableMatcher("%user", "%user"); + + MappingRuleMatcher userStatic = + new MappingRuleMatchers.VariableMatcher("%user", "bob"); + + assertTrue("%custom should match %user in matching context", + customUser.match(matchingContext)); + assertTrue("%user should match %custom in matching context", + userCustom.match(matchingContext)); + assertTrue("%user (bob) should match %cus%tom (b + ob) in matching context", + userCusTom.match(matchingContext)); + assertTrue("%user should match %user in any context", + userUser.match(matchingContext)); + assertTrue("%user (bob) should match bob in in matching context", + userStatic.match(matchingContext)); + + assertFalse( + "%custom (bob) should NOT match %user (dave) in mismatching context", + customUser.match(mismatchingContext)); + assertFalse( + "%user (dave) should NOT match %custom (bob) in mismatching context", + userCustom.match(mismatchingContext)); + assertFalse( + "%user (dave) should NOT match %cus%tom (b+ob) in mismatching context", + userCusTom.match(mismatchingContext)); + assertTrue("%user should match %user in any context", + userUser.match(mismatchingContext)); + assertFalse( + "%user (dave) should NOT match match bob in in matching context", + userStatic.match(mismatchingContext)); + } + + @Test + public void testVariableMatcherWithEmpties() { + VariableContext emptyContext = new VariableContext(); + VariableContext allNull = new VariableContext(); + allNull.put("%null", null); + allNull.put("%empty", null); + + VariableContext allEmpty = new VariableContext(); + allEmpty.put("%null", ""); + allEmpty.put("%empty", ""); + + VariableContext allMakesSense = new VariableContext(); + allMakesSense.put("%null", null); + allMakesSense.put("%empty", ""); + + VariableContext allFull = new VariableContext(); + allFull.put("%null", "full"); + allFull.put("%empty", "full"); + + MappingRuleMatcher nullMatcher = + new MappingRuleMatchers.VariableMatcher("%null", null); + + MappingRuleMatcher emptyMatcher = + new MappingRuleMatchers.VariableMatcher("%empty", ""); + + //nulls should be handled as empty strings, so all should match + assertTrue(nullMatcher.match(emptyContext)); + assertTrue(emptyMatcher.match(emptyContext)); + assertTrue(nullMatcher.match(allEmpty)); + assertTrue(emptyMatcher.match(allEmpty)); + assertTrue(nullMatcher.match(allNull)); + assertTrue(emptyMatcher.match(allNull)); + assertTrue(nullMatcher.match(allMakesSense)); + assertTrue(emptyMatcher.match(allMakesSense)); + + //neither null nor "" should match an actual string + assertFalse(nullMatcher.match(allFull)); + assertFalse(emptyMatcher.match(allFull)); + + MappingRuleMatcher nullVarMatcher = + new MappingRuleMatchers.VariableMatcher(null, ""); + + //null variable should never match + assertFalse(nullVarMatcher.match(emptyContext)); + assertFalse(nullVarMatcher.match(allNull)); + assertFalse(nullVarMatcher.match(allEmpty)); + assertFalse(nullVarMatcher.match(allMakesSense)); + assertFalse(nullVarMatcher.match(allFull)); + } + + @Test + public void testBoolOperatorMatchers() { + VariableContext developerBob = new VariableContext(); + developerBob.put("%user", "bob"); + developerBob.put("%primary_group", "developers"); + + + VariableContext testerBob = new VariableContext(); + testerBob.put("%user", "bob"); + testerBob.put("%primary_group", "testers"); + + VariableContext testerDave = new VariableContext(); + testerDave.put("%user", "dave"); + testerDave.put("%primary_group", "testers"); + + + VariableContext accountantDave = new VariableContext(); + accountantDave.put("%user", "dave"); + accountantDave.put("%primary_group", "accountants"); + + MappingRuleMatcher userBob = + new MappingRuleMatchers.VariableMatcher("%user", "bob"); + + MappingRuleMatcher groupDevelopers = + new MappingRuleMatchers.VariableMatcher( + "%primary_group", "developers"); + + MappingRuleMatcher groupAccountants = + new MappingRuleMatchers.VariableMatcher( + "%primary_group", "accountants"); + + MappingRuleMatcher developerBobMatcher = new MappingRuleMatchers.AndMatcher( + userBob, groupDevelopers); + + MappingRuleMatcher testerDaveMatcher = + MappingRuleMatchers.createUserGroupMatcher("dave", "testers"); + + MappingRuleMatcher accountantOrBobMatcher = + new MappingRuleMatchers.OrMatcher(groupAccountants, userBob); + + assertTrue(developerBobMatcher.match(developerBob)); + assertFalse(developerBobMatcher.match(testerBob)); + assertFalse(developerBobMatcher.match(testerDave)); + assertFalse(developerBobMatcher.match(accountantDave)); + + assertFalse(testerDaveMatcher.match(developerBob)); + assertFalse(testerDaveMatcher.match(testerBob)); + assertTrue(testerDaveMatcher.match(testerDave)); + assertFalse(testerDaveMatcher.match(accountantDave)); + + assertTrue(accountantOrBobMatcher.match(developerBob)); + assertTrue(accountantOrBobMatcher.match(testerBob)); + assertFalse(accountantOrBobMatcher.match(testerDave)); + assertTrue(accountantOrBobMatcher.match(accountantDave)); + } + + @Test + public void testToStrings() { + MappingRuleMatcher var = new MappingRuleMatchers.VariableMatcher("%a", "b"); + MappingRuleMatcher all = MappingRuleMatchers.createAllMatcher(); + MappingRuleMatcher and = new MappingRuleMatchers.AndMatcher(var, all, var); + MappingRuleMatcher or = new MappingRuleMatchers.OrMatcher(var, all, var); + + assertEquals("VariableMatcher{variable='%a', value='b'}", var.toString()); + assertEquals("MatchAllMatcher", all.toString()); + assertEquals("AndMatcher{matchers=[" + var.toString() + + ", " + all.toString() + + ", " + var.toString() + "]}", and.toString()); + assertEquals("OrMatcher{matchers=[" + var.toString() + + ", " + all.toString() + + ", " + var.toString() + "]}", or.toString()); + } + +} \ No newline at end of file