YARN-10372. Create MappingRule class to represent each CS mapping rule. Contributed by Gergely Pollak
This commit is contained in:
Peter Bacsko
parent
5e12dc51f0
commit
288dab7563
@ -0,0 +1,136 @@
|
|||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||||
|
|
||||||
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mapping rule represents a single mapping setting defined by the user. All
|
||||||
|
* rules have matchers and actions. Matcher determine if a mapping rule applies
|
||||||
|
* to a given applicationSubmission, while action represent the course of action
|
||||||
|
* we need to take when a rule applies.
|
||||||
|
*
|
||||||
|
* MappingRules also support fallback actions, which will be evaluated when the
|
||||||
|
* main action fails due to any reason (Eg. trying to place to a queue which
|
||||||
|
* does not exist)
|
||||||
|
*/
|
||||||
|
public class MappingRule {
|
||||||
|
public static final String USER_MAPPING = "u";
|
||||||
|
public static final String GROUP_MAPPING = "g";
|
||||||
|
public static final String APPLICATION_MAPPING = "a";
|
||||||
|
private final MappingRuleMatcher matcher;
|
||||||
|
private final MappingRuleAction action;
|
||||||
|
|
||||||
|
public MappingRule(MappingRuleMatcher matcher, MappingRuleAction action) {
|
||||||
|
this.matcher = matcher;
|
||||||
|
this.action = action;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method evaluates the rule, and returns the MappingRuleResult, if
|
||||||
|
* the rule matches, skip action otherwise.
|
||||||
|
* @param variables The variable context, which contains all the variables
|
||||||
|
* @return The rule's result or skip action if the rule doesn't apply
|
||||||
|
*/
|
||||||
|
public MappingRuleResult evaluate(VariableContext variables) {
|
||||||
|
if (matcher.match(variables)) {
|
||||||
|
return action.execute(variables);
|
||||||
|
}
|
||||||
|
|
||||||
|
return MappingRuleResult.createSkipResult();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the associated action's fallback.
|
||||||
|
* @return The fallback of the action
|
||||||
|
*/
|
||||||
|
public MappingRuleResult getFallback() {
|
||||||
|
return action.getFallback();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a MappingRule object from the legacy style configuration. The
|
||||||
|
* configuration is a [TYPE]:SOURCE:PATH (eg. u:bob:root.users.%user).
|
||||||
|
* Using the source and path parts of the legacy rule, this method will
|
||||||
|
* create an application MappingRule which behaves as the legacy rule defined.
|
||||||
|
* This version of the method does not require type, since legacy application
|
||||||
|
* mappings omitted the 'a', so this method is to be used for those rules,
|
||||||
|
* which in all case are application mappings.
|
||||||
|
* @param source This part of the rule determines which applications the rule
|
||||||
|
* will be applied
|
||||||
|
* @param path The path where the application is to be placed
|
||||||
|
* @return MappingRule based on the provided settings
|
||||||
|
*/
|
||||||
|
public static MappingRule createLegacyRule(String source, String path) {
|
||||||
|
return createLegacyRule(APPLICATION_MAPPING, source, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a MappingRule object from the legacy style configuration. The
|
||||||
|
* configuration is a [TYPE]:SOURCE:PATH (eg. u:bob:root.users.%user).
|
||||||
|
* Using the type, source and path parts of the legacy rule, this method will
|
||||||
|
* create a MappingRule which behaves as the legacy rule defined.
|
||||||
|
* @param type The type of the rule, can be
|
||||||
|
* 'u' for user mapping, 'g' for group mapping or
|
||||||
|
* 'a' for application mapping
|
||||||
|
* @param source This part of the rule determines which submissions this rule
|
||||||
|
* should apply to (eg. if type is 'u', source will match
|
||||||
|
* against the user name)
|
||||||
|
* @param path The path where the application is to be placed
|
||||||
|
* @return MappingRule based on the provided settings
|
||||||
|
*/
|
||||||
|
public static MappingRule createLegacyRule(
|
||||||
|
String type, String source, String path) {
|
||||||
|
MappingRuleMatcher matcher;
|
||||||
|
MappingRuleAction action = new MappingRuleActions.PlaceToQueueAction(path);
|
||||||
|
//While legacy rule fallback handling is a bit inconsistent, the most cases
|
||||||
|
//it fall back to default queue placement, so this is the best approximation
|
||||||
|
action.setFallbackDefaultPlacement();
|
||||||
|
|
||||||
|
switch (type) {
|
||||||
|
case USER_MAPPING:
|
||||||
|
matcher = MappingRuleMatchers.createUserMatcher(source);
|
||||||
|
break;
|
||||||
|
case GROUP_MAPPING:
|
||||||
|
matcher = MappingRuleMatchers.createGroupMatcher(source);
|
||||||
|
break;
|
||||||
|
case APPLICATION_MAPPING:
|
||||||
|
matcher = MappingRuleMatchers.createApplicationNameMatcher(source);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException("Invalid mapping rule type '" +
|
||||||
|
type + "'");
|
||||||
|
}
|
||||||
|
|
||||||
|
return new MappingRule(matcher, action);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void validate(MappingRuleValidationContext ctx)
|
||||||
|
throws YarnException {
|
||||||
|
this.action.validate(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "MappingRule{" +
|
||||||
|
"matcher=" + matcher +
|
||||||
|
", action=" + action +
|
||||||
|
'}';
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,143 @@
|
|||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
import org.apache.hadoop.util.StringUtils;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class TestMappingRule {
|
||||||
|
VariableContext setupVariables(
|
||||||
|
String user, String group, String secGroup, String appName) {
|
||||||
|
VariableContext variables = new VariableContext();
|
||||||
|
variables.put("%default", "root.default");
|
||||||
|
variables.put("%user", user);
|
||||||
|
variables.put("%primary_group", group);
|
||||||
|
variables.put("%secondary_group", secGroup);
|
||||||
|
variables.put("%application", appName);
|
||||||
|
variables.put("%sub", "xxx");
|
||||||
|
variables.put("%empty", "");
|
||||||
|
variables.put("%null", null);
|
||||||
|
variables.setImmutables("%user", "%primary_group", "%secondary_group",
|
||||||
|
"%application");
|
||||||
|
|
||||||
|
return variables;
|
||||||
|
}
|
||||||
|
|
||||||
|
void assertSkipResult(MappingRuleResult result) {
|
||||||
|
assertTrue(
|
||||||
|
MappingRuleResultType.SKIP == result.getResult());
|
||||||
|
}
|
||||||
|
|
||||||
|
void assertPlaceResult(MappingRuleResult result, String queue) {
|
||||||
|
assertTrue(
|
||||||
|
MappingRuleResultType.PLACE == result.getResult());
|
||||||
|
assertEquals(queue, result.getQueue());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMappingRuleEvaluation() {
|
||||||
|
VariableContext matching = setupVariables(
|
||||||
|
"bob", "developer", "users", "MR");
|
||||||
|
VariableContext mismatching = setupVariables(
|
||||||
|
"joe", "tester", "admins", "Spark");
|
||||||
|
|
||||||
|
MappingRule rule = new MappingRule(
|
||||||
|
MappingRuleMatchers.createUserMatcher("bob"),
|
||||||
|
(new MappingRuleActions.PlaceToQueueAction("%default.%default"))
|
||||||
|
.setFallbackSkip()
|
||||||
|
);
|
||||||
|
|
||||||
|
assertSkipResult(rule.getFallback());
|
||||||
|
|
||||||
|
MappingRuleResult matchingResult = rule.evaluate(matching);
|
||||||
|
MappingRuleResult mismatchingResult = rule.evaluate(mismatching);
|
||||||
|
|
||||||
|
assertSkipResult(mismatchingResult);
|
||||||
|
assertPlaceResult(matchingResult, "root.default.root.default");
|
||||||
|
}
|
||||||
|
|
||||||
|
MappingRule createMappingRuleFromLegacyString(String legacyMapping) {
|
||||||
|
String[] mapping =
|
||||||
|
StringUtils
|
||||||
|
.getTrimmedStringCollection(legacyMapping, ":")
|
||||||
|
.toArray(new String[] {});
|
||||||
|
|
||||||
|
if (mapping.length == 2) {
|
||||||
|
return MappingRule.createLegacyRule(mapping[0], mapping[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return MappingRule.createLegacyRule(mapping[0], mapping[1], mapping[2]);
|
||||||
|
}
|
||||||
|
|
||||||
|
void evaluateLegacyStringTestcase(
|
||||||
|
String legacyString, VariableContext variables, String expectedQueue) {
|
||||||
|
MappingRule rule = createMappingRuleFromLegacyString(legacyString);
|
||||||
|
MappingRuleResult result = rule.evaluate(variables);
|
||||||
|
assertEquals(
|
||||||
|
rule.getFallback().getResult(), MappingRuleResultType.PLACE_TO_DEFAULT);
|
||||||
|
|
||||||
|
if (expectedQueue == null) {
|
||||||
|
assertSkipResult(result);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPlaceResult(result, expectedQueue);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testLegacyEvaluation() {
|
||||||
|
VariableContext matching = setupVariables(
|
||||||
|
"bob", "developer", "users", "MR");
|
||||||
|
VariableContext mismatching = setupVariables(
|
||||||
|
"joe", "tester", "admins", "Spark");
|
||||||
|
|
||||||
|
evaluateLegacyStringTestcase(
|
||||||
|
"u:bob:root.%primary_group", matching, "root.developer");
|
||||||
|
evaluateLegacyStringTestcase(
|
||||||
|
"u:bob:root.%primary_group", mismatching, null);
|
||||||
|
evaluateLegacyStringTestcase(
|
||||||
|
"g:developer:%secondary_group.%user", matching, "users.bob");
|
||||||
|
evaluateLegacyStringTestcase(
|
||||||
|
"g:developer:%secondary_group.%user", mismatching, null);
|
||||||
|
evaluateLegacyStringTestcase(
|
||||||
|
"MR:root.static", matching, "root.static");
|
||||||
|
evaluateLegacyStringTestcase(
|
||||||
|
"MR:root.static", mismatching, null);
|
||||||
|
|
||||||
|
//catch all tests
|
||||||
|
evaluateLegacyStringTestcase(
|
||||||
|
"u:%user:root.%primary_group", matching, "root.developer");
|
||||||
|
evaluateLegacyStringTestcase(
|
||||||
|
"u:%user:root.%primary_group", mismatching, "root.tester");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testToStrings() {
|
||||||
|
MappingRuleAction action = new MappingRuleActions.PlaceToQueueAction(
|
||||||
|
"queue");
|
||||||
|
MappingRuleMatcher matcher = MappingRuleMatchers.createUserMatcher("bob");
|
||||||
|
MappingRule rule = new MappingRule(matcher, action);
|
||||||
|
|
||||||
|
assertEquals("MappingRule{matcher=" + matcher.toString() +
|
||||||
|
", action=" + action.toString() + "}", rule.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user