Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9991. Fix Application Tag prefix to userid. Contributed by Szilard Nemeth.

This commit is contained in:
prabhujoseph 2019-11-27 01:43:24 +05:30
parent 8c9018d5c7
commit aa7ab2719f
4 changed files with 42 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1860,7 +1860,7 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED = public static final boolean DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED =
false; false;
private static final String APPLICATION_TAG_BASED_PLACEMENT_PREFIX = private static final String APPLICATION_TAG_BASED_PLACEMENT_PREFIX =
"application-tag-based-placement"; RM_PREFIX + "application-tag-based-placement";
public static final String APPLICATION_TAG_BASED_PLACEMENT_ENABLED = public static final String APPLICATION_TAG_BASED_PLACEMENT_ENABLED =
APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".enable"; APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".enable";
public static final boolean DEFAULT_APPLICATION_TAG_BASED_PLACEMENT_ENABLED = public static final boolean DEFAULT_APPLICATION_TAG_BASED_PLACEMENT_ENABLED =

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1291,11 +1291,12 @@
<property> <property>
<description> <description>
Whether to enable application placement based on user ID passed via Whether to enable application placement based on user ID passed via
application tags. When it is enabled, u=&lt;userId&gt; pattern will be checked application tags. When it is enabled, userid=&lt;userId&gt;
and if found, the application will be placed onto the found user's queue, pattern will be checked and if found, the application will be placed
onto the found user's queue,
if the original user has enough rights on the passed user's queue. if the original user has enough rights on the passed user's queue.
</description> </description>
<name>application-tag-based-placement.enable</name> <name>yarn.resourcemanager.application-tag-based-placement.enable</name>
<value>false</value> <value>false</value>
</property> </property>
@ -1304,7 +1305,7 @@
Comma separated list of users who can use the application tag based Comma separated list of users who can use the application tag based
placement, if it is enabled. placement, if it is enabled.
</description> </description>
<name>application-tag-based-placement.username.whitelist</name> <name>yarn.resourcemanager.application-tag-based-placement.username.whitelist</name>
<value></value> <value></value>
</property> </property>

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -105,6 +105,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private boolean nodeLabelsEnabled; private boolean nodeLabelsEnabled;
private Set<String> exclusiveEnforcedPartitions; private Set<String> exclusiveEnforcedPartitions;
private static final String USER_ID_PREFIX = "userid=";
public RMAppManager(RMContext context, public RMAppManager(RMContext context,
YarnScheduler scheduler, ApplicationMasterService masterService, YarnScheduler scheduler, ApplicationMasterService masterService,
ApplicationACLsManager applicationACLsManager, Configuration conf) { ApplicationACLsManager applicationACLsManager, Configuration conf) {
@ -938,11 +940,11 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
return usernameUsedForPlacement; return usernameUsedForPlacement;
} }
LOG.debug("Application tag based placement is enabled, checking for " + LOG.debug("Application tag based placement is enabled, checking for " +
"userId in the application tag"); "'userid' among the application tags");
Set<String> applicationTags = context.getApplicationTags(); Set<String> applicationTags = context.getApplicationTags();
String userNameFromAppTag = getUserNameFromApplicationTag(applicationTags); String userNameFromAppTag = getUserNameFromApplicationTag(applicationTags);
if (userNameFromAppTag != null) { if (userNameFromAppTag != null) {
LOG.debug("Found userId '{}' in application tag", userNameFromAppTag); LOG.debug("Found 'userid' '{}' in application tag", userNameFromAppTag);
UserGroupInformation callerUGI = UserGroupInformation UserGroupInformation callerUGI = UserGroupInformation
.createRemoteUser(userNameFromAppTag); .createRemoteUser(userNameFromAppTag);
// check if the actual user has rights to submit application to the // check if the actual user has rights to submit application to the
@ -958,7 +960,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
userNameFromAppTag, queue, user); userNameFromAppTag, queue, user);
} }
} else { } else {
LOG.warn("userId was not found in application tags"); LOG.warn("'userid' was not found in application tags");
} }
return usernameUsedForPlacement; return usernameUsedForPlacement;
} }
@ -979,9 +981,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
} }
private String getUserNameFromApplicationTag(Set<String> applicationTags) { private String getUserNameFromApplicationTag(Set<String> applicationTags) {
String userIdPrefix = "u=";
for (String tag: applicationTags) { for (String tag: applicationTags) {
if (tag.startsWith(userIdPrefix)) { if (tag.startsWith(USER_ID_PREFIX)) {
String[] userIdTag = tag.split("="); String[] userIdTag = tag.split("=");
if (userIdTag.length == 2) { if (userIdTag.length == 2) {
return userIdTag[1]; return userIdTag[1];

55
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -125,6 +125,8 @@ public class TestAppManager extends AppManagerTestBase{
private static String USER = "user_"; private static String USER = "user_";
private static String USER0 = USER + 0; private static String USER0 = USER + 0;
private static final String USER_ID_PREFIX = "userid=";
public synchronized RMAppEventType getAppEventType() { public synchronized RMAppEventType getAppEventType() {
return appEventType; return appEventType;
} }
@ -1205,55 +1207,55 @@ public class TestAppManager extends AppManagerTestBase{
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String userIdTag = "u=user2"; String userIdTag = USER_ID_PREFIX + "user2";
setApplicationTags("tag1", userIdTag, "tag2"); setApplicationTags("tag1", userIdTag, "tag2");
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, user);
} }
@Test
/** /**
* Test case for when the application tag based placement is enabled and * Test case for when the application tag based placement is enabled and
* the submitting user 'user1' is whitelisted and the user from the * the submitting user 'user1' is whitelisted and the user from the
* application tag has access to queue. * application tag has access to queue.
* Expected behaviour: the placement is done for user from the tag 'user2' * Expected behaviour: the placement is done for user from the tag 'user2'
*/ */
@Test
public void testGetUserNameForPlacementTagBasedPlacementEnabled() public void testGetUserNameForPlacementTagBasedPlacementEnabled()
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String expectedUser = "user2"; String expectedUser = "user2";
String userIdTag = "u=" + expectedUser; String userIdTag = USER_ID_PREFIX + expectedUser;
setApplicationTags("tag1", userIdTag, "tag2"); setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(true, user); enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, expectedUser); verifyPlacementUsername(expectedQueue, user, expectedUser);
} }
@Test
/** /**
* Test case for when the application tag based placement is enabled and * Test case for when the application tag based placement is enabled.
* the submitting user 'user1' is whitelisted and there are multiple valid * And submitting user 'user1' is whitelisted and there are multiple valid
* username tags passed * username tags passed
* Expected behaviour: the placement is done for the first valid username * Expected behaviour: the placement is done for the first valid username
* from the tag 'user2' * from the tag 'user2'
*/ */
@Test
public void testGetUserNameForPlacementTagBasedPlacementMultipleUserIds() public void testGetUserNameForPlacementTagBasedPlacementMultipleUserIds()
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String expectedUser = "user2"; String expectedUser = "user2";
String userIdTag = "u=" + expectedUser; String userIdTag = USER_ID_PREFIX + expectedUser;
String userIdTag2 = "u=user3"; String userIdTag2 = USER_ID_PREFIX + "user3";
setApplicationTags("tag1", userIdTag, "tag2", userIdTag2); setApplicationTags("tag1", userIdTag, "tag2", userIdTag2);
enableApplicationTagPlacement(true, user); enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, expectedUser); verifyPlacementUsername(expectedQueue, user, expectedUser);
} }
@Test
/** /**
* Test case for when the application tag based placement is enabled but * Test case for when the application tag based placement is enabled.
* no username is set in the application tag * And no username is set in the application tag
* Expected behaviour: the placement is done for the submitting user 'user1' * Expected behaviour: the placement is done for the submitting user 'user1'
*/ */
@Test
public void testGetUserNameForPlacementTagBasedPlacementNoUserId() public void testGetUserNameForPlacementTagBasedPlacementNoUserId()
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
@ -1263,87 +1265,90 @@ public class TestAppManager extends AppManagerTestBase{
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, user);
} }
@Test
/** /**
* Test case for when the application tag based placement is enabled but * Test case for when the application tag based placement is enabled but
* the user from the application tag 'user2' does not have access to the * the user from the application tag 'user2' does not have access to the
* queue. * queue.
* Expected behaviour: the placement is done for the submitting user 'user1' * Expected behaviour: the placement is done for the submitting user 'user1'
*/ */
@Test
public void testGetUserNameForPlacementUserWithoutAccessToQueue() public void testGetUserNameForPlacementUserWithoutAccessToQueue()
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String userIdTag = "u=user2"; String userIdTag = USER_ID_PREFIX + "user2";
setApplicationTags("tag1", userIdTag, "tag2"); setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(false, user); enableApplicationTagPlacement(false, user);
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, user);
} }
@Test
/** /**
* Test case for when the application tag based placement is enabled but * Test case for when the application tag based placement is enabled but
* the submitting user 'user1' is not whitelisted and there is a valid * the submitting user 'user1' is not whitelisted and there is a valid
* username tag passed. * username tag passed.
* Expected behaviour: the placement is done for the submitting user 'user1' * Expected behaviour: the placement is done for the submitting user 'user1'
*/ */
@Test
public void testGetUserNameForPlacementNotWhitelistedUser() public void testGetUserNameForPlacementNotWhitelistedUser()
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String userIdTag = "u=user2"; String userIdTag = USER_ID_PREFIX + "user2";
setApplicationTags("tag1", userIdTag, "tag2"); setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(true, "someUser"); enableApplicationTagPlacement(true, "someUser");
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, user);
} }
@Test
/** /**
* Test case for when the application tag based placement is enabled but * Test case for when the application tag based placement is enabled but
* there is no whitelisted user. * there is no whitelisted user.
* Expected behaviour: the placement is done for the submitting user 'user1' * Expected behaviour: the placement is done for the submitting user 'user1'
*/ */
@Test
public void testGetUserNameForPlacementEmptyWhiteList() public void testGetUserNameForPlacementEmptyWhiteList()
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String userIdTag = "u=user2"; String userIdTag = USER_ID_PREFIX + "user2";
setApplicationTags("tag1", userIdTag, "tag2"); setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(false); enableApplicationTagPlacement(false);
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, user);
} }
@Test
/** /**
* Test case for when the application tag based placement is enabled and * Test case for when the application tag based placement is enabled and
* there is one wrongly qualified user 'u=' and a valid user 'u=user2' passed * there is one wrongly qualified user
* via application tag. * 'userid=' and a valid user 'userid=user2' passed
* with application tag.
* Expected behaviour: the placement is done for the first valid username * Expected behaviour: the placement is done for the first valid username
* from the tag 'user2' * from the tag 'user2'
*/ */
@Test
public void testGetUserNameForPlacementWronglyQualifiedFirstUserNameInTag() public void testGetUserNameForPlacementWronglyQualifiedFirstUserNameInTag()
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String expectedUser = "user2"; String expectedUser = "user2";
String userIdTag = "u=" + expectedUser; String userIdTag = USER_ID_PREFIX + expectedUser;
String wrongUserIdTag = "u="; String wrongUserIdTag = USER_ID_PREFIX;
setApplicationTags("tag1", wrongUserIdTag, userIdTag, "tag2"); setApplicationTags("tag1", wrongUserIdTag, userIdTag, "tag2");
enableApplicationTagPlacement(true, user); enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, expectedUser); verifyPlacementUsername(expectedQueue, user, expectedUser);
} }
@Test
/** /**
* Test case for when the application tag based placement is enabled and * Test case for when the application tag based placement is enabled and
* there is only one wrongly qualified user 'u=' passed via application tag. * there is only one wrongly qualified user 'userid=' passed
* with application tag.
* Expected behaviour: the placement is done for the submitting user 'user1' * Expected behaviour: the placement is done for the submitting user 'user1'
*/ */
@Test
public void testGetUserNameForPlacementWronglyQualifiedUserNameInTag() public void testGetUserNameForPlacementWronglyQualifiedUserNameInTag()
throws YarnException { throws YarnException {
String user = "user1"; String user = "user1";
String expectedQueue = "user1Queue"; String expectedQueue = "user1Queue";
String wrongUserIdTag = "u="; String wrongUserIdTag = USER_ID_PREFIX;
setApplicationTags("tag1", wrongUserIdTag, "tag2"); setApplicationTags("tag1", wrongUserIdTag, "tag2");
enableApplicationTagPlacement(true, user); enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, user); verifyPlacementUsername(expectedQueue, user, user);