From 51dc407f3016c9ae3bd97f14559520f2ce1655e5 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 11 Apr 2013 22:00:10 +0000 Subject: [PATCH] YARN-319. Submitting a job to a fair scheduler queue for which the user does not have permission causes the client to wait forever. Contributed by Shenhong. svn merge --ignore-ancestry -c 1437336 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1467110 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 6 +- .../scheduler/fair/FairScheduler.java | 8 ++- .../scheduler/fair/TestFairScheduler.java | 59 +++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 874977d4820..5976b53bf62 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -162,6 +162,10 @@ Release 2.0.5-beta - UNRELEASED YARN-539. Addressed memory leak of LocalResource objects NM when a resource localization fails. (Omkar Vinit Joshi via vinodkv) + YARN-319. Submitting a job to a fair scheduler queue for which the user + does not have permission causes the client to wait forever. + (shenhong via tomwhite) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES @@ -377,7 +381,7 @@ Release 2.0.3-alpha - 2013-02-06 unregistered on App-finish. (vinodkv via sseth) YARN-302. Fair scheduler assignmultiple should default to false. (sandyr via tucu) - + YARN-372. Move InlineDispatcher from hadoop-yarn-server-resourcemanager to hadoop-yarn-common (sseth via hitesh) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 5a2b0b6cca4..238432e941c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMSta import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRejectedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; @@ -502,8 +503,11 @@ public class FairScheduler implements ResourceScheduler { // Enforce ACLs UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user); if (!queue.hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi)) { - LOG.info("User " + userUgi.getUserName() + - " cannot submit applications to queue " + queue.getName()); + String msg = "User " + userUgi.getUserName() + + " cannot submit applications to queue " + queue.getName(); + LOG.info(msg); + rmContext.getDispatcher().getEventHandler().handle( + new RMAppAttemptRejectedEvent(applicationAttemptId, msg)); return; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 6a7a86769e7..29cccb97ea3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -40,19 +40,28 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.Clock; +import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -1369,4 +1378,54 @@ public class TestFairScheduler { assertEquals(2, app1.getLiveContainers().size()); assertEquals(1, app2.getLiveContainers().size()); } + + + @SuppressWarnings("unchecked") + @Test + public void testNotAllowSubmitApplication() throws Exception { + // Set acl's + Configuration conf = createConfiguration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("userallow"); + out.println(""); + out.println(""); + out.close(); + QueueManager queueManager = scheduler.getQueueManager(); + queueManager.initialize(); + + int appId = this.APP_ID++; + String user = "usernotallow"; + String queue = "queue1"; + ApplicationId applicationId = MockApps.newAppID(appId); + String name = MockApps.newAppName(); + ApplicationMasterService masterService = + new ApplicationMasterService(resourceManager.getRMContext(), scheduler); + ApplicationSubmissionContext submissionContext = new ApplicationSubmissionContextPBImpl(); + RMApp application = + new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user, + queue, submissionContext, scheduler, masterService, + System.currentTimeMillis()); + resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application); + application.handle(new RMAppEvent(applicationId, RMAppEventType.START)); + + ApplicationAttemptId attId = recordFactory.newRecordInstance(ApplicationAttemptId.class); + attId.setAttemptId(this.ATTEMPT_ID++); + attId.setApplicationId(applicationId); + scheduler.addApplication(attId, queue, user); + + final int MAX_TRIES=20; + int numTries = 0; + while (application.getFinishTime() == 0 && numTries < MAX_TRIES) { + try { + Thread.sleep(100); + } catch (InterruptedException ex) {ex.printStackTrace();} + numTries++; + } + assertEquals(FinalApplicationStatus.FAILED, application.getFinalApplicationStatus()); + } }