From 2d068f179390da79b28a3f2aa28cf8f1838b7085 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Wed, 23 Nov 2016 23:29:39 +0530 Subject: [PATCH] YARN-5865. Retrospect updateApplicationPriority api to handle state store exception in align with YARN-5611. Contributed by Sunil G. --- .../ApplicationMasterService.java | 2 +- .../resourcemanager/ClientRMService.java | 10 ++-- .../server/resourcemanager/RMAppManager.java | 49 +++++++++++++++++-- .../metrics/SystemMetricsPublisher.java | 3 +- .../server/resourcemanager/rmapp/RMApp.java | 14 ++++++ .../resourcemanager/rmapp/RMAppImpl.java | 37 +++++++++++--- .../scheduler/AbstractYarnScheduler.java | 6 ++- .../scheduler/YarnScheduler.java | 11 ++++- .../scheduler/capacity/CapacityScheduler.java | 28 +++++------ .../event/AppAddedSchedulerEvent.java | 5 +- .../resourcemanager/webapp/RMWebServices.java | 6 +-- .../resourcemanager/webapp/dao/AppInfo.java | 9 ++-- .../resourcemanager/TestAppManager.java | 1 + .../TestApplicationMasterService.java | 6 +-- .../resourcemanager/TestClientRMService.java | 13 +++-- .../applicationsmanager/MockAsm.java | 11 +++++ .../metrics/TestSystemMetricsPublisher.java | 2 + .../resourcemanager/rmapp/MockRMApp.java | 10 ++++ .../rmapp/TestRMAppTransitions.java | 2 + .../capacity/TestApplicationPriority.java | 13 +++-- 20 files changed, 177 insertions(+), 61 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 888bea93da1..108c327033e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -604,7 +604,7 @@ public class ApplicationMasterService extends AbstractService implements // Set application priority allocateResponse.setApplicationPriority(app - .getApplicationSubmissionContext().getPriority()); + .getApplicationPriority()); // update AMRMToken if the token is rolled-up MasterKeyData nextMasterKey = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 6690cd88fcc..21a9d13f154 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -1584,14 +1584,14 @@ public class ClientRMService extends AbstractService implements .newRecordInstance(UpdateApplicationPriorityResponse.class); // Update priority only when app is tracked by the scheduler if (!ACTIVE_APP_STATES.contains(application.getState())) { - if (COMPLETED_APP_STATES.contains(application.getState())) { + if (application.isAppInCompletedStates()) { // If Application is in any of the final states, change priority // can be skipped rather throwing exception. RMAuditLogger.logSuccess(callerUGI.getShortUserName(), AuditConstants.UPDATE_APP_PRIORITY, "ClientRMService", applicationId); response.setApplicationPriority(application - .getApplicationSubmissionContext().getPriority()); + .getApplicationPriority()); return response; } String msg = "Application in " + application.getState() @@ -1604,8 +1604,7 @@ public class ClientRMService extends AbstractService implements } try { - rmContext.getScheduler().updateApplicationPriority(newAppPriority, - applicationId); + rmAppManager.updateApplicationPriority(applicationId, newAppPriority); } catch (YarnException ex) { RMAuditLogger.logFailure(callerUGI.getShortUserName(), AuditConstants.UPDATE_APP_PRIORITY, "UNKNOWN", "ClientRMService", @@ -1615,8 +1614,7 @@ public class ClientRMService extends AbstractService implements RMAuditLogger.logSuccess(callerUGI.getShortUserName(), AuditConstants.UPDATE_APP_PRIORITY, "ClientRMService", applicationId); - response.setApplicationPriority(application - .getApplicationSubmissionContext().getPriority()); + response.setApplicationPriority(application.getApplicationPriority()); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 136fc5df529..c4e9e14d64d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -355,9 +355,9 @@ public class RMAppManager implements EventHandler, // Verify and get the update application priority and set back to // submissionContext - Priority appPriority = rmContext.getScheduler() - .checkAndGetApplicationPriority(submissionContext.getPriority(), user, - submissionContext.getQueue(), applicationId); + Priority appPriority = scheduler.checkAndGetApplicationPriority( + submissionContext.getPriority(), user, submissionContext.getQueue(), + applicationId); submissionContext.setPriority(appPriority); UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user); @@ -513,6 +513,10 @@ public class RMAppManager implements EventHandler, throws YarnException { ApplicationId applicationId = app.getApplicationId(); synchronized (applicationId) { + if (app.isAppInCompletedStates()) { + return; + } + Map newExpireTime = RMServerUtils .validateISO8601AndConvertToLocalTimeEpoch(newTimeoutInISO8601Format); @@ -540,4 +544,43 @@ public class RMAppManager implements EventHandler, ((RMAppImpl) app).updateApplicationTimeout(newExpireTime); } } + + /** + * updateApplicationPriority will invoke scheduler api to update the + * new priority to RM and StateStore. + * @param applicationId Application Id + * @param newAppPriority proposed new application priority + * @throws YarnException Handle exceptions + */ + public void updateApplicationPriority(ApplicationId applicationId, + Priority newAppPriority) throws YarnException { + RMApp app = this.rmContext.getRMApps().get(applicationId); + + synchronized (applicationId) { + if (app.isAppInCompletedStates()) { + return; + } + + // Create a future object to capture exceptions from StateStore. + SettableFuture future = SettableFuture.create(); + + // Invoke scheduler api to update priority in scheduler and to + // State Store. + Priority appPriority = rmContext.getScheduler() + .updateApplicationPriority(newAppPriority, applicationId, future); + + if (app.getApplicationPriority().equals(appPriority)) { + return; + } + + Futures.get(future, YarnException.class); + + // update in-memory + ((RMAppImpl) app).setApplicationPriority(appPriority); + } + + // Update the changed application state to timeline server + rmContext.getSystemMetricsPublisher().appUpdated(app, + System.currentTimeMillis()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index 5d9c6ad292a..775e6e3966b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -129,8 +129,7 @@ public class SystemMetricsPublisher extends CompositeService { if (publishSystemMetrics) { dispatcher.getEventHandler() .handle(new ApplicationUpdatedEvent(app.getApplicationId(), - app.getQueue(), updatedTime, - app.getApplicationSubmissionContext().getPriority())); + app.getQueue(), updatedTime, app.getApplicationPriority())); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index cf77e723596..e55649d60a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -260,4 +261,17 @@ public interface RMApp extends EventHandler { CallerContext getCallerContext(); Map getApplicationTimeouts(); + + /** + * Get priority of the application. + * @return priority + */ + Priority getApplicationPriority(); + + /** + * To verify whether app has reached in its completing/completed states. + * + * @return True/False to confirm whether app is in final states + */ + boolean isAppInCompletedStates(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 6975ceb824b..cd9465fb8dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -195,6 +196,8 @@ public class RMAppImpl implements RMApp, Recoverable { Object transitionTodo; + private Priority applicationPriority; + private static final StateMachineFactory future) + throws YarnException { // Dummy Implementation till Application Priority changes are done in // specific scheduler. + return Priority.newInstance(0); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index c4f575f0410..62ab7f4848b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -51,6 +51,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import com.google.common.util.concurrent.SettableFuture; + /** * This interface is used by the components to talk to the * scheduler for allocating of resources, cleaning up resources. @@ -318,9 +320,14 @@ public interface YarnScheduler extends EventHandler { * @param newPriority Submitted Application priority. * * @param applicationId Application ID + * + * @param future Sets any type of exception happened from StateStore + * + * @return updated priority */ - public void updateApplicationPriority(Priority newPriority, - ApplicationId applicationId) throws YarnException; + public Priority updateApplicationPriority(Priority newPriority, + ApplicationId applicationId, SettableFuture future) + throws YarnException; /** * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index af3c9981c6c..86896cf690c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -146,6 +146,7 @@ import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.SettableFuture; @LimitedPrivate("yarn") @Evolving @@ -2406,8 +2407,9 @@ public class CapacityScheduler extends } @Override - public void updateApplicationPriority(Priority newPriority, - ApplicationId applicationId) throws YarnException { + public Priority updateApplicationPriority(Priority newPriority, + ApplicationId applicationId, SettableFuture future) + throws YarnException { Priority appPriority = null; SchedulerApplication application = applications .get(applicationId); @@ -2418,38 +2420,36 @@ public class CapacityScheduler extends } RMApp rmApp = rmContext.getRMApps().get(applicationId); + appPriority = checkAndGetApplicationPriority(newPriority, rmApp.getUser(), rmApp.getQueue(), applicationId); if (application.getPriority().equals(appPriority)) { - return; + future.set(null); + return appPriority; } - // Update new priority in Submission Context to keep track in HA + // Update new priority in Submission Context to update to StateStore. rmApp.getApplicationSubmissionContext().setPriority(appPriority); // Update to state store - ApplicationStateData appState = - ApplicationStateData.newInstance(rmApp.getSubmitTime(), - rmApp.getStartTime(), rmApp.getApplicationSubmissionContext(), - rmApp.getUser(), rmApp.getCallerContext()); + ApplicationStateData appState = ApplicationStateData.newInstance( + rmApp.getSubmitTime(), rmApp.getStartTime(), + rmApp.getApplicationSubmissionContext(), rmApp.getUser(), + rmApp.getCallerContext()); appState.setApplicationTimeouts(rmApp.getApplicationTimeouts()); rmContext.getStateStore().updateApplicationStateSynchronously(appState, - false, null); + false, future); // As we use iterator over a TreeSet for OrderingPolicy, once we change // priority then reinsert back to make order correct. LeafQueue queue = (LeafQueue) getQueue(rmApp.getQueue()); - queue.updateApplicationPriority(application, appPriority); - // Update the changed application state to timeline server - rmContext.getSystemMetricsPublisher().appUpdated(rmApp, - System.currentTimeMillis()); - LOG.info("Priority '" + appPriority + "' is updated in queue :" + rmApp.getQueue() + " for application: " + applicationId + " for the user: " + rmApp.getUser()); + return appPriority; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java index 89d2f66b094..0a8d6fe529b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java @@ -43,10 +43,11 @@ public class AppAddedSchedulerEvent extends SchedulerEvent { } public AppAddedSchedulerEvent(String user, - ApplicationSubmissionContext submissionContext, boolean isAppRecovering) { + ApplicationSubmissionContext submissionContext, boolean isAppRecovering, + Priority appPriority) { this(submissionContext.getApplicationId(), submissionContext.getQueue(), user, isAppRecovering, submissionContext.getReservationID(), - submissionContext.getPriority()); + appPriority); } public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 963f7b0e5e0..4c692fe2376 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -1221,7 +1221,7 @@ public class RMWebServices extends WebServices { AppPriority ret = new AppPriority(); ret.setPriority( - app.getApplicationSubmissionContext().getPriority().getPriority()); + app.getApplicationPriority().getPriority()); return ret; } @@ -1261,7 +1261,7 @@ public class RMWebServices extends WebServices { "Trying to update priority an absent application " + appId); throw e; } - Priority priority = app.getApplicationSubmissionContext().getPriority(); + Priority priority = app.getApplicationPriority(); if (priority == null || priority.getPriority() != targetPriority.getPriority()) { return modifyApplicationPriority(app, callerUGI, @@ -1308,7 +1308,7 @@ public class RMWebServices extends WebServices { } } AppPriority ret = new AppPriority( - app.getApplicationSubmissionContext().getPriority().getPriority()); + app.getApplicationPriority().getPriority()); return Response.status(Status.OK).entity(ret).build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 42449b08d36..d3ddf80394e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -148,10 +148,9 @@ public class AppInfo { this.name = app.getName().toString(); this.queue = app.getQueue().toString(); this.priority = 0; - ApplicationSubmissionContext appSubmissionContext = - app.getApplicationSubmissionContext(); - if (appSubmissionContext.getPriority() != null) { - this.priority = appSubmissionContext.getPriority() + + if (app.getApplicationPriority() != null) { + this.priority = app.getApplicationPriority() .getPriority(); } this.progress = app.getProgress() * 100; @@ -220,6 +219,8 @@ public class AppInfo { vcoreSeconds = appMetrics.getVcoreSeconds(); preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds(); preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds(); + ApplicationSubmissionContext appSubmissionContext = + app.getApplicationSubmissionContext(); unmanagedApplication = appSubmissionContext.getUnmanagedAM(); appNodeLabelExpression = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index d8693364f25..39e07e08cf2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -246,6 +246,7 @@ public class TestAppManager{ asContext.setApplicationId(appId); asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory)); asContext.setResource(mockResource()); + asContext.setPriority(Priority.newInstance(0)); setupDispatcher(rmContext, conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java index 93befcbdba3..00466ae8c48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java @@ -461,12 +461,10 @@ public class TestApplicationMasterService { AllocateResponse response1 = am1.allocate(allocateRequest); Assert.assertEquals(appPriority1, response1.getApplicationPriority()); - // get scheduler - CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); - // Change the priority of App1 to 8 Priority appPriority2 = Priority.newInstance(8); - cs.updateApplicationPriority(appPriority2, app1.getApplicationId()); + rm.getRMAppManager().updateApplicationPriority(app1.getApplicationId(), + appPriority2); AllocateResponse response2 = am1.allocate(allocateRequest); Assert.assertEquals(appPriority2, response2.getApplicationPriority()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 553d7cc0af0..99904532928 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -455,7 +455,7 @@ public class TestClientRMService { } } - public ClientRMService createRMService() throws IOException { + public ClientRMService createRMService() throws IOException, YarnException { YarnScheduler yarnScheduler = mockYarnScheduler(); RMContext rmContext = mock(RMContext.class); mockRMContext(yarnScheduler, rmContext); @@ -956,6 +956,7 @@ public class TestClientRMService { submissionContext.setApplicationType(appType); submissionContext.setApplicationTags(tags); submissionContext.setUnmanagedAM(unmanaged); + submissionContext.setPriority(Priority.newInstance(0)); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); @@ -1030,6 +1031,7 @@ public class TestClientRMService { ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class); when(asContext.getMaxAppAttempts()).thenReturn(1); when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression); + when(asContext.getPriority()).thenReturn(Priority.newInstance(0)); RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null, @@ -1064,6 +1066,7 @@ public class TestClientRMService { attempts.put(attemptId, rmAppAttemptImpl); when(app.getCurrentAppAttempt()).thenReturn(rmAppAttemptImpl); when(app.getAppAttempts()).thenReturn(attempts); + when(app.getApplicationPriority()).thenReturn(Priority.newInstance(0)); when(rmAppAttemptImpl.getMasterContainer()).thenReturn(container); ResourceScheduler rs = mock(ResourceScheduler.class); when(rmContext.getScheduler()).thenReturn(rs); @@ -1086,7 +1089,7 @@ public class TestClientRMService { return app; } - private static YarnScheduler mockYarnScheduler() { + private static YarnScheduler mockYarnScheduler() throws YarnException { YarnScheduler yarnScheduler = mock(YarnScheduler.class); when(yarnScheduler.getMinimumResourceCapability()).thenReturn( Resources.createResource( @@ -1104,6 +1107,9 @@ public class TestClientRMService { ResourceCalculator rs = mock(ResourceCalculator.class); when(yarnScheduler.getResourceCalculator()).thenReturn(rs); + when(yarnScheduler.checkAndGetApplicationPriority(any(Priority.class), + anyString(), anyString(), any(ApplicationId.class))) + .thenReturn(Priority.newInstance(0)); return yarnScheduler; } @@ -1663,8 +1669,7 @@ public class TestClientRMService { RMApp app1 = rm.submitApp(1024, Priority.newInstance(appPriority)); Assert.assertEquals("Incorrect priority has been set to application", - appPriority, app1.getApplicationSubmissionContext().getPriority() - .getPriority()); + appPriority, app1.getApplicationPriority().getPriority()); appPriority = 11; ClientRMService rmService = rm.getClientRMService(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 2f33df887ee..3d3aeaa41b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -223,6 +224,16 @@ public abstract class MockAsm extends MockApps { public Map getApplicationTimeouts() { throw new UnsupportedOperationException("Not supported yet."); } + + @Override + public Priority getApplicationPriority() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public boolean isAppInCompletedStates() { + throw new UnsupportedOperationException("Not supported yet."); + } } public static RMApp newApplication(int i) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index c945681418e..f6290ebda4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -126,6 +126,7 @@ public class TestSystemMetricsPublisher { .thenReturn(Collections.singletonList("java -Xmx1024m")); when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext); when(app.getApplicationSubmissionContext()).thenReturn(asc); + when(app.getApplicationPriority()).thenReturn(Priority.newInstance(1)); metricsPublisher.appUpdated(app, 4L); } else { metricsPublisher.appUpdated(app, 4L); @@ -526,6 +527,7 @@ public class TestSystemMetricsPublisher { when(amReq.getNodeLabelExpression()).thenReturn("high-mem"); when(app.getAMResourceRequest()).thenReturn(amReq); when(app.getAmNodeLabelExpression()).thenCallRealMethod(); + when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10)); when(app.getCallerContext()) .thenReturn(new CallerContext.Builder("context").build()); return app; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index f215a4b877f..5dbe8582882 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -307,4 +307,14 @@ public class MockRMApp implements RMApp { public Map getApplicationTimeouts() { throw new UnsupportedOperationException("Not supported yet."); } + + @Override + public Priority getApplicationPriority() { + return null; + } + + @Override + public boolean isAppInCompletedStates() { + return false; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 17628384e99..1d74d8175dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -262,6 +263,7 @@ public class TestRMAppTransitions { // applicationId will not be used because RMStateStore is mocked, // but applicationId is still set for safety submissionContext.setApplicationId(applicationId); + submissionContext.setPriority(Priority.newInstance(0)); RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java index 2a346f828c6..164ca20c1ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java @@ -344,7 +344,7 @@ public class TestApplicationPriority { // Change the priority of App1 to 8 Priority appPriority2 = Priority.newInstance(8); - cs.updateApplicationPriority(appPriority2, app1.getApplicationId()); + cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null); // get scheduler app FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications() @@ -378,7 +378,7 @@ public class TestApplicationPriority { // Change the priority of App1 to 15 Priority appPriority2 = Priority.newInstance(15); - cs.updateApplicationPriority(appPriority2, app1.getApplicationId()); + cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null); // get scheduler app FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications() @@ -428,7 +428,7 @@ public class TestApplicationPriority { // Change the priority of App1 to 8 Priority appPriority2 = Priority.newInstance(8); - cs.updateApplicationPriority(appPriority2, app1.getApplicationId()); + cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null); // let things settle down Thread.sleep(1000); @@ -449,8 +449,7 @@ public class TestApplicationPriority { .get(app1.getApplicationId()); // Verify whether priority 15 is reset to 10 - Assert.assertEquals(appPriority2, loadedApp.getCurrentAppAttempt() - .getSubmissionContext().getPriority()); + Assert.assertEquals(appPriority2, loadedApp.getApplicationPriority()); rm2.stop(); rm1.stop(); @@ -558,7 +557,7 @@ public class TestApplicationPriority { // Change the priority of App1 to 3 (lowest) Priority appPriority3 = Priority.newInstance(3); - cs.updateApplicationPriority(appPriority3, app2.getApplicationId()); + cs.updateApplicationPriority(appPriority3, app2.getApplicationId(), null); // add request for containers App2 am2.allocate("127.0.0.1", 2 * GB, 3, new ArrayList()); @@ -790,7 +789,7 @@ public class TestApplicationPriority { throws YarnException { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); cs.updateApplicationPriority(Priority.newInstance(2), - app.getApplicationId()); + app.getApplicationId(), null); SchedulerEvent removeAttempt; removeAttempt = new AppAttemptRemovedSchedulerEvent( app.getCurrentAppAttempt().getAppAttemptId(), RMAppAttemptState.KILLED,