Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-10070. Fix NPE if no queue mapping defined for proxy user when using application tag based placement. 

Contributed by Kinga Marton. Reviewed by Adam Antal.
This commit is contained in:
Prabhu Joseph 2020-01-16 12:55:19 +05:30 committed by Prabhu Joseph
parent b8518e6dab
commit a0ff42d761
2 changed files with 71 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -929,7 +929,6 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
boolean applicationTagBasedPlacementEnabled = conf boolean applicationTagBasedPlacementEnabled = conf
.getBoolean(YarnConfiguration.APPLICATION_TAG_BASED_PLACEMENT_ENABLED, .getBoolean(YarnConfiguration.APPLICATION_TAG_BASED_PLACEMENT_ENABLED,
YarnConfiguration.DEFAULT_APPLICATION_TAG_BASED_PLACEMENT_ENABLED); YarnConfiguration.DEFAULT_APPLICATION_TAG_BASED_PLACEMENT_ENABLED);
String usernameUsedForPlacement = user; String usernameUsedForPlacement = user;
if (!applicationTagBasedPlacementEnabled) { if (!applicationTagBasedPlacementEnabled) {
return usernameUsedForPlacement; return usernameUsedForPlacement;
@ -946,11 +945,16 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
if (userNameFromAppTag != null) { if (userNameFromAppTag != null) {
LOG.debug("Found 'userid' '{}' in application tag", userNameFromAppTag); LOG.debug("Found 'userid' '{}' in application tag", userNameFromAppTag);
UserGroupInformation callerUGI = UserGroupInformation UserGroupInformation callerUGI = UserGroupInformation
.createRemoteUser(userNameFromAppTag); .createRemoteUser(user);
// check if the actual user has rights to submit application to the // check if the actual user has rights to submit application to the
// user's queue from the application tag // user's queue from the application tag
String queue = placementManager ApplicationPlacementContext appPlacementContext = placementManager
.placeApplication(context, usernameUsedForPlacement).getQueue(); .placeApplication(context, userNameFromAppTag);
if (appPlacementContext == null) {
LOG.warn("No rule was found for user '{}'", userNameFromAppTag);
return usernameUsedForPlacement;
}
String queue = appPlacementContext.getQueue();
if (callerUGI != null && scheduler if (callerUGI != null && scheduler
.checkAccess(callerUGI, QueueACL.SUBMIT_APPLICATIONS, queue)) { .checkAccess(callerUGI, QueueACL.SUBMIT_APPLICATIONS, queue)) {
usernameUsedForPlacement = userNameFromAppTag; usernameUsedForPlacement = userNameFromAppTag;

91
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -1207,13 +1207,14 @@ public class TestAppManager extends AppManagerTestBase{
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String userIdTag = USER_ID_PREFIX + "user2"; String userNameFromAppTag = "user2";
String userIdTag = USER_ID_PREFIX + userNameFromAppTag;
setApplicationTags("tag1", userIdTag, "tag2"); setApplicationTags("tag1", userIdTag, "tag2");
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, userNameFromAppTag, user);
} }
/** /**
* Test case for when the application tag based placement is enabled and * Test for the case when the application tag based placement is enabled and
* the submitting user 'user1' is whitelisted and the user from the * the submitting user 'user1' is whitelisted and the user from the
* application tag has access to queue. * application tag has access to queue.
* Expected behaviour: the placement is done for user from the tag 'user2' * Expected behaviour: the placement is done for user from the tag 'user2'
@ -1222,16 +1223,18 @@ public class TestAppManager extends AppManagerTestBase{
public void testGetUserNameForPlacementTagBasedPlacementEnabled() public void testGetUserNameForPlacementTagBasedPlacementEnabled()
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String usernameFromAppTag = "user2";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String expectedUser = "user2"; String expectedUser = usernameFromAppTag;
String userIdTag = USER_ID_PREFIX + expectedUser; String userIdTag = USER_ID_PREFIX + usernameFromAppTag;
setApplicationTags("tag1", userIdTag, "tag2"); setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(true, user); enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, expectedUser); verifyPlacementUsername(expectedQueue, user, usernameFromAppTag,
expectedUser);
} }
/** /**
* Test case for when the application tag based placement is enabled. * Test for the case when the application tag based placement is enabled.
* And submitting user 'user1' is whitelisted and there are multiple valid * And submitting user 'user1' is whitelisted and there are multiple valid
* username tags passed * username tags passed
* Expected behaviour: the placement is done for the first valid username * Expected behaviour: the placement is done for the first valid username
@ -1242,16 +1245,18 @@ public class TestAppManager extends AppManagerTestBase{
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String expectedUser = "user2"; String userNameFromAppTag = "user2";
String expectedUser = userNameFromAppTag;
String userIdTag = USER_ID_PREFIX + expectedUser; String userIdTag = USER_ID_PREFIX + expectedUser;
String userIdTag2 = USER_ID_PREFIX + "user3"; String userIdTag2 = USER_ID_PREFIX + "user3";
setApplicationTags("tag1", userIdTag, "tag2", userIdTag2); setApplicationTags("tag1", userIdTag, "tag2", userIdTag2);
enableApplicationTagPlacement(true, user); enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, expectedUser); verifyPlacementUsername(expectedQueue, user, userNameFromAppTag,
expectedUser);
} }
/** /**
* Test case for when the application tag based placement is enabled. * Test for the case when the application tag based placement is enabled.
* And no username is set in the application tag * And no username is set in the application tag
* Expected behaviour: the placement is done for the submitting user 'user1' * Expected behaviour: the placement is done for the submitting user 'user1'
*/ */
@ -1260,13 +1265,14 @@ public class TestAppManager extends AppManagerTestBase{
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String userNameFromAppTag = null;
setApplicationTags("tag1", "tag2"); setApplicationTags("tag1", "tag2");
enableApplicationTagPlacement(true, user); enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, userNameFromAppTag, user);
} }
/** /**
* Test case for when the application tag based placement is enabled but * Test for the case when the application tag based placement is enabled but
* the user from the application tag 'user2' does not have access to the * the user from the application tag 'user2' does not have access to the
* queue. * queue.
* Expected behaviour: the placement is done for the submitting user 'user1' * Expected behaviour: the placement is done for the submitting user 'user1'
@ -1276,14 +1282,15 @@ public class TestAppManager extends AppManagerTestBase{
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String userIdTag = USER_ID_PREFIX + "user2"; String userNameFromAppTag = "user2";
String userIdTag = USER_ID_PREFIX + userNameFromAppTag;
setApplicationTags("tag1", userIdTag, "tag2"); setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(false, user); enableApplicationTagPlacement(false, user);
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, userNameFromAppTag, user);
} }
/** /**
* Test case for when the application tag based placement is enabled but * Test for the case when the application tag based placement is enabled but
* the submitting user 'user1' is not whitelisted and there is a valid * the submitting user 'user1' is not whitelisted and there is a valid
* username tag passed. * username tag passed.
* Expected behaviour: the placement is done for the submitting user 'user1' * Expected behaviour: the placement is done for the submitting user 'user1'
@ -1293,14 +1300,15 @@ public class TestAppManager extends AppManagerTestBase{
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String userIdTag = USER_ID_PREFIX + "user2"; String userNameFromAppTag = "user2";
String userIdTag = USER_ID_PREFIX + userNameFromAppTag;
setApplicationTags("tag1", userIdTag, "tag2"); setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(true, "someUser"); enableApplicationTagPlacement(true, "someUser");
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, userNameFromAppTag, user);
} }
/** /**
* Test case for when the application tag based placement is enabled but * Test for the case when the application tag based placement is enabled but
* there is no whitelisted user. * there is no whitelisted user.
* Expected behaviour: the placement is done for the submitting user 'user1' * Expected behaviour: the placement is done for the submitting user 'user1'
*/ */
@ -1309,15 +1317,16 @@ public class TestAppManager extends AppManagerTestBase{
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String userIdTag = USER_ID_PREFIX + "user2"; String userNameFromAppTag = "user2";
String userIdTag = USER_ID_PREFIX + userNameFromAppTag;
setApplicationTags("tag1", userIdTag, "tag2"); setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(false); enableApplicationTagPlacement(false);
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, userNameFromAppTag, user);
} }
/** /**
* Test case for when the application tag based placement is enabled and * Test for the case when the application tag based placement is enabled and
* there is one wrongly qualified user * there is one wrongly qualified user
* 'userid=' and a valid user 'userid=user2' passed * 'userid=' and a valid user 'userid=user2' passed
* with application tag. * with application tag.
@ -1329,16 +1338,18 @@ public class TestAppManager extends AppManagerTestBase{
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String expectedUser = "user2"; String userNameFromAppTag = "user2";
String userIdTag = USER_ID_PREFIX + expectedUser; String expectedUser = userNameFromAppTag;
String userIdTag = USER_ID_PREFIX + userNameFromAppTag;
String wrongUserIdTag = USER_ID_PREFIX; String wrongUserIdTag = USER_ID_PREFIX;
setApplicationTags("tag1", wrongUserIdTag, userIdTag, "tag2"); setApplicationTags("tag1", wrongUserIdTag, userIdTag, "tag2");
enableApplicationTagPlacement(true, user); enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, expectedUser); verifyPlacementUsername(expectedQueue, user, userNameFromAppTag,
expectedUser);
} }
/** /**
* Test case for when the application tag based placement is enabled and * Test for the case when the application tag based placement is enabled and
* there is only one wrongly qualified user 'userid=' passed * there is only one wrongly qualified user 'userid=' passed
* with application tag. * with application tag.
* Expected behaviour: the placement is done for the submitting user 'user1' * Expected behaviour: the placement is done for the submitting user 'user1'
@ -1348,10 +1359,33 @@ public class TestAppManager extends AppManagerTestBase{
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String userNameFromAppTag = "";
String wrongUserIdTag = USER_ID_PREFIX; String wrongUserIdTag = USER_ID_PREFIX;
setApplicationTags("tag1", wrongUserIdTag, "tag2"); setApplicationTags("tag1", wrongUserIdTag, "tag2");
enableApplicationTagPlacement(true, user); enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, userNameFromAppTag, user);
}
/**
* Test for the case when the application tag based placement is enabled.
* And there is no placement rule defined for the user from the application tag
* Expected behaviour: the placement is done for the submitting user 'user1'
*/
@Test
public void testGetUserNameForPlacementNoRuleDefined()
throws YarnException {
String user = "user1";
String expectedUser = user;
String userNameFromAppTag = "user2";
String wrongUserIdTag = USER_ID_PREFIX + userNameFromAppTag;
setApplicationTags("tag1", wrongUserIdTag, "tag2");
enableApplicationTagPlacement(true, user);
PlacementManager placementMgr = mock(PlacementManager.class);
when(placementMgr.placeApplication(asContext, userNameFromAppTag))
.thenReturn(null);
String userNameForPlacement = appMonitor
.getUserNameForPlacement(user, asContext, placementMgr);
Assert.assertEquals(expectedUser, userNameForPlacement);
} }
private void enableApplicationTagPlacement(boolean userHasAccessToQueue, private void enableApplicationTagPlacement(boolean userHasAccessToQueue,
@ -1375,12 +1409,13 @@ public class TestAppManager extends AppManagerTestBase{
} }
private void verifyPlacementUsername(final String queue, private void verifyPlacementUsername(final String queue,
final String submittingUser, final String expectedUser) final String submittingUser, final String userNameFRomAppTag,
final String expectedUser)
throws YarnException { throws YarnException {
PlacementManager placementMgr = mock(PlacementManager.class); PlacementManager placementMgr = mock(PlacementManager.class);
ApplicationPlacementContext appContext ApplicationPlacementContext appContext
= new ApplicationPlacementContext(queue); = new ApplicationPlacementContext(queue);
when(placementMgr.placeApplication(asContext, submittingUser)) when(placementMgr.placeApplication(asContext, userNameFRomAppTag))
.thenReturn(appContext); .thenReturn(appContext);
String userNameForPlacement = appMonitor String userNameForPlacement = appMonitor
.getUserNameForPlacement(submittingUser, asContext, placementMgr); .getUserNameForPlacement(submittingUser, asContext, placementMgr);