Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-10619. CS Mapping Rule %specified rule catches default submissions. Contributed by Gergely Pollak

This commit is contained in:
Szilard Nemeth 2021-02-09 18:16:42 +01:00
parent 9434c1eccc
commit fc5f241535
2 changed files with 56 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
View File

@ -217,7 +217,13 @@ public class CSMappingPlacementRule extends PlacementRule {
VariableContext vctx = new VariableContext();
vctx.put("%user", user);
vctx.put("%specified", asc.getQueue());
//If the specified matches the default it means NO queue have been specified
//as per ClientRMService#submitApplication which sets the queue to default
//when no queue is provided.
//To place queues specifically to default, users must use root.default
if (!asc.getQueue().equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) {
vctx.put("%specified", asc.getQueue());
}
vctx.put("%application", asc.getApplicationName());
vctx.put("%default", "root.default");
try {
@ -379,7 +385,6 @@ public class CSMappingPlacementRule extends PlacementRule {
asc.getApplicationName(), appQueue, overrideWithQueueMappings);
if (appQueue != null &&
!appQueue.equals(YarnConfiguration.DEFAULT_QUEUE_NAME) &&
!appQueue.equals(YarnConfiguration.DEFAULT_QUEUE_FULL_NAME) &&
!overrideWithQueueMappings &&
!recovery) {
LOG.info("Have no jurisdiction over application submission '{}', " +

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestCSMappingPlacementRule.java
View File

@ -425,6 +425,55 @@ public class TestCSMappingPlacementRule {
}
@Test
public void testSpecified() throws IOException {
ArrayList<MappingRule> rules = new ArrayList<>();
rules.add(
new MappingRule(
MappingRuleMatchers.createAllMatcher(),
(new MappingRuleActions.PlaceToQueueAction("%specified", true))
.setFallbackSkip()));
rules.add(
new MappingRule(
MappingRuleMatchers.createAllMatcher(),
(new MappingRuleActions.PlaceToQueueAction(
"root.ambiguous.group.tester", true))
.setFallbackSkip()));
rules.add(
new MappingRule(
MappingRuleMatchers.createAllMatcher(),
(new MappingRuleActions.RejectAction())
.setFallbackReject()));
CSMappingPlacementRule engine = setupEngine(true, rules);
ApplicationSubmissionContext appNoQueue = createApp("app");
ApplicationSubmissionContext appDefault = createApp("app", "default");
ApplicationSubmissionContext appRootDefault =
createApp("app", "root.default");
ApplicationSubmissionContext appBob =
createApp("app", "root.user.bob");
assertPlace("App with non specified queue should end up in " +
"'root.ambiguous.group.tester' because no queue was specified and " +
"this is the only rule matching the submission",
engine, appNoQueue, "alice", "root.ambiguous.group.tester");
assertPlace("App with specified 'default' should end up in " +
"'root.ambiguous.group.tester' because 'default' is the same as " +
"no queue being specified and this is the only rule matching the " +
"submission ",
engine, appDefault, "alice", "root.ambiguous.group.tester");
assertPlace("App with specified root.default should end up in " +
"'root.default' because root.default is specifically provided",
engine, appRootDefault, "alice", "root.default");
assertPlace("App with specified queue should end up in the specified " +
"queue 'root.user.bob'", engine, appBob, "alice", "root.user.bob");
}
private MappingRule createGroupMapping(String group, String queue) {
MappingRuleMatcher matcher = MappingRuleMatchers.createUserGroupMatcher(group);
MappingRuleAction action =