From c570ddaa665d907cc7503fb32daa05de13f8ec9c Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Wed, 27 Sep 2017 11:49:59 +0530 Subject: [PATCH] YARN-7157. Add admin configuration to filter per-user's apps in secure cluster. Contributed by Sunil G. --- .../hadoop/yarn/conf/YarnConfiguration.java | 7 ++ .../src/main/resources/yarn-default.xml | 9 +++ .../resourcemanager/ClientRMService.java | 19 ++++- .../resourcemanager/TestClientRMService.java | 73 +++++++++++++++++++ 4 files changed, 107 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0f05f1c760b..117c8ce3332 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2936,6 +2936,13 @@ public class YarnConfiguration extends Configuration { public static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS = NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts"; + /* + * Support to view apps for given user in secure cluster. + */ + public static final String DISPLAY_APPS_FOR_LOGGED_IN_USER = + RM_PREFIX + "display.per-user-apps"; + public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER = false; + // RM and NM CSRF props public static final String REST_CSRF = "webapp.rest-csrf."; public static final String RM_CSRF_PREFIX = RM_PREFIX + REST_CSRF; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 122b824e9ba..2840a634895 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3183,4 +3183,13 @@ 0.0.0.0:8091 + + yarn.resourcemanager.display.per-user-apps + false + + Flag to enable display of applications per user as an admin + configuration. + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index b7084442674..6ba6b91ba96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -205,6 +205,8 @@ public class ClientRMService extends AbstractService implements private ReservationSystem reservationSystem; private ReservationInputValidator rValidator; + private boolean displayPerUserApps = false; + private static final EnumSet ACTIVE_APP_STATES = EnumSet.of( RMAppState.ACCEPTED, RMAppState.RUNNING); @@ -262,7 +264,11 @@ public class ClientRMService extends AbstractService implements } refreshServiceAcls(conf, RMPolicyProvider.getInstance()); } - + + this.displayPerUserApps = conf.getBoolean( + YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER, + YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER); + this.server.start(); clientBindAddress = conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST, YarnConfiguration.RM_ADDRESS, @@ -881,6 +887,12 @@ public class ClientRMService extends AbstractService implements continue; } + // Given RM is configured to display apps per user, skip apps to which + // this caller doesn't have access to view. + if (displayPerUserApps && !allowAccess) { + continue; + } + reports.add(application.createAndGetApplicationReport( callerUGI.getUserName(), allowAccess)); } @@ -1729,4 +1741,9 @@ public class ClientRMService extends AbstractService implements return application; } + @VisibleForTesting + public void setDisplayPerUserApps(boolean displayPerUserApps) { + this.displayPerUserApps = displayPerUserApps; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index e7ecbd339c1..34166f6e23f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyListOf; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; @@ -853,6 +854,12 @@ public class TestClientRMService { assertEquals("Incorrect number of applications for user", 3, rmService.getApplications(request).getApplicationList().size()); + rmService.setDisplayPerUserApps(true); + userSet.clear(); + assertEquals("Incorrect number of applications for user", 6, + rmService.getApplications(request).getApplicationList().size()); + rmService.setDisplayPerUserApps(false); + // Check tags request = GetApplicationsRequest.newInstance( ApplicationsRequestScope.ALL, null, null, null, null, null, null, @@ -1793,4 +1800,70 @@ public class TestClientRMService { rpc.stopProxy(client, conf); new File(excludeFile).delete(); } + + @Test + public void testGetApplicationsWithPerUserApps() + throws IOException, YarnException { + /* + * Submit 3 applications alternately in two queues + */ + // Basic setup + YarnScheduler yarnScheduler = mockYarnScheduler(); + RMContext rmContext = mock(RMContext.class); + mockRMContext(yarnScheduler, rmContext); + RMStateStore stateStore = mock(RMStateStore.class); + when(rmContext.getStateStore()).thenReturn(stateStore); + + RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, + mock(ApplicationACLsManager.class), new Configuration()); + when(rmContext.getDispatcher().getEventHandler()) + .thenReturn(new EventHandler() { + public void handle(Event event) { + } + }); + + // Simulate Queue ACL manager which returns false always + QueueACLsManager queueAclsManager = mock(QueueACLsManager.class); + when(queueAclsManager.checkAccess(any(UserGroupInformation.class), + any(QueueACL.class), any(RMApp.class), any(String.class), + anyListOf(String.class))).thenReturn(false); + + // Simulate app ACL manager which returns false always + ApplicationACLsManager appAclsManager = mock(ApplicationACLsManager.class); + when(appAclsManager.checkAccess(eq(UserGroupInformation.getCurrentUser()), + any(ApplicationAccessType.class), any(String.class), + any(ApplicationId.class))).thenReturn(false); + ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, + appManager, appAclsManager, queueAclsManager, null); + rmService.init(new Configuration()); + + // Initialize appnames and queues + String[] queues = {QUEUE_1, QUEUE_2}; + String[] appNames = {MockApps.newAppName(), MockApps.newAppName(), + MockApps.newAppName()}; + ApplicationId[] appIds = {getApplicationId(101), getApplicationId(102), + getApplicationId(103)}; + List tags = Arrays.asList("Tag1", "Tag2", "Tag3"); + + long[] submitTimeMillis = new long[3]; + // Submit applications + for (int i = 0; i < appIds.length; i++) { + ApplicationId appId = appIds[i]; + SubmitApplicationRequest submitRequest = mockSubmitAppRequest(appId, + appNames[i], queues[i % queues.length], + new HashSet(tags.subList(0, i + 1))); + rmService.submitApplication(submitRequest); + submitTimeMillis[i] = System.currentTimeMillis(); + } + + // Test different cases of ClientRMService#getApplications() + GetApplicationsRequest request = GetApplicationsRequest.newInstance(); + assertEquals("Incorrect total number of apps", 6, + rmService.getApplications(request).getApplicationList().size()); + + rmService.setDisplayPerUserApps(true); + assertEquals("Incorrect number of applications for user", 0, + rmService.getApplications(request).getApplicationList().size()); + rmService.setDisplayPerUserApps(false); + } }