From 8ef46595da6aefe4458aa7181670c3d9b13e7ec6 Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Tue, 27 Aug 2019 15:35:17 -0700 Subject: [PATCH] YARN-9438. launchTime not written to state store for running applications --- .../server/resourcemanager/RMAppManager.java | 2 ++ .../resourcemanager/rmapp/RMAppImpl.java | 6 +++++ .../scheduler/capacity/CapacityScheduler.java | 1 + .../server/resourcemanager/TestRMRestart.java | 19 +++++++++------- .../rmapp/TestRMAppTransitions.java | 22 +++++++++++++++++++ 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 3cf3dd1d969..977a5bb65c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -722,6 +722,7 @@ public Map updateApplicationTimeout(RMApp app, app.getStartTime(), app.getApplicationSubmissionContext(), app.getUser(), app.getCallerContext()); appState.setApplicationTimeouts(currentExpireTimeouts); + appState.setLaunchTime(app.getLaunchTime()); // update to state store. Though it synchronous call, update via future to // know any exception has been set. It is required because in non-HA mode, @@ -847,6 +848,7 @@ private void updateAppDataToStateStore(String queue, RMApp app, app.getApplicationSubmissionContext(), app.getUser(), app.getCallerContext()); appState.setApplicationTimeouts(app.getApplicationTimeouts()); + appState.setLaunchTime(app.getLaunchTime()); rmContext.getStateStore().updateApplicationStateSynchronously(appState, false, future); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index d25dddc1fa6..4ff7aeaf869 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -1032,6 +1032,12 @@ public void transition(RMAppImpl app, RMAppEvent event) { app.getApplicationId()+", attemptId: "+ app.getCurrentAppAttempt().getAppAttemptId()+ "launchTime: "+event.getTimestamp()); + ApplicationStateData appState = ApplicationStateData.newInstance( + app.submitTime, app.startTime, app.submissionContext, app.user, + app.callerContext); + appState.setApplicationTimeouts(app.getApplicationTimeouts()); + appState.setLaunchTime(event.getTimestamp()); + app.rmContext.getStateStore().updateApplicationState(appState); app.launchTime = event.getTimestamp(); app.rmContext.getSystemMetricsPublisher().appLaunched( app, app.launchTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index d6f20d7fa29..21cdbf9113f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2675,6 +2675,7 @@ public Priority updateApplicationPriority(Priority newPriority, rmApp.getApplicationSubmissionContext(), rmApp.getUser(), rmApp.getCallerContext()); appState.setApplicationTimeouts(rmApp.getApplicationTimeouts()); + appState.setLaunchTime(rmApp.getLaunchTime()); rmContext.getStateStore().updateApplicationStateSynchronously(appState, false, future); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 87c05921e8c..6127214fab0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -754,13 +754,14 @@ public void testRMRestartWaitForPreviousSucceededAttempt() throws Exception { @Override public void updateApplicationStateInternal(ApplicationId appId, ApplicationStateData appStateData) throws Exception { - if (count == 0) { - // do nothing; simulate app final state is not saved. + if (count == 1) { + // Application state is updated on attempt launch. + // After that, do nothing; simulate app final state is not saved. LOG.info(appId + " final state is not saved."); - count++; } else { super.updateApplicationStateInternal(appId, appStateData); } + count++; } }; memStore.init(conf); @@ -774,7 +775,6 @@ public void updateApplicationStateInternal(ApplicationId appId, MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 15120); RMApp app0 = rm1.submitApp(200); MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1); - FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance( FinalApplicationStatus.SUCCEEDED, "", ""); @@ -1798,8 +1798,11 @@ public void testClientRetryOnKillingApplication() throws Exception { rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.KILLED); rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED); - Assert.assertEquals(1, ((TestMemoryRMStateStore) memStore).updateAttempt); - Assert.assertEquals(2, ((TestMemoryRMStateStore) memStore).updateApp); + // count = 1 on storing RMApp launchTime + // count = 2 on storing attempt state on kill + // count = 3 on storing app state on kill + Assert.assertEquals(2, ((TestMemoryRMStateStore) memStore).updateAttempt); + Assert.assertEquals(3, ((TestMemoryRMStateStore) memStore).updateApp); } // Test Application that fails on submission is saved in state store. @@ -2548,8 +2551,6 @@ public void testRMRestartOnMissingAttempts() throws Exception { // create an app and finish the app. RMApp app0 = rm1.submitApp(200); - ApplicationStateData app0State = memStore.getState().getApplicationState() - .get(app0.getApplicationId()); MockAM am0 = launchAndFailAM(app0, rm1, nm1); MockAM am1 = launchAndFailAM(app0, rm1, nm1); @@ -2558,6 +2559,8 @@ public void testRMRestartOnMissingAttempts() throws Exception { // am1 is missed from MemoryRMStateStore memStore.removeApplicationAttemptInternal(am1.getApplicationAttemptId()); + ApplicationStateData app0State = memStore.getState().getApplicationState() + .get(app0.getApplicationId()); ApplicationAttemptStateData am2State = app0State.getAttempt( am2.getApplicationAttemptId()); // am2's state is not consistent: MemoryRMStateStore just saved its initial diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 2e176cc9763..122aa30a6b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -483,6 +483,13 @@ private void assertAppFinalStateNotSaved(RMApp application){ any(ApplicationStateData.class)); } + private void assertAppStateLaunchTimeSaved(long expectedLaunchTime) { + ArgumentCaptor state = + ArgumentCaptor.forClass(ApplicationStateData.class); + verify(store, times(1)).updateApplicationState(state.capture()); + assertEquals(expectedLaunchTime, state.getValue().getLaunchTime()); + } + private void assertKilled(RMApp application) { assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); @@ -932,6 +939,21 @@ public void testAppAcceptedKill() throws IOException, InterruptedException { verifyRMAppFieldsForFinalTransitions(application); } + @Test + public void testAppAcceptedAccepted() throws IOException { + LOG.info("--- START: testAppAcceptedAccepted ---"); + + RMApp application = testCreateAppAccepted(null); + // ACCEPTED => ACCEPTED event RMAppEventType.ATTEMPT_LAUNCHED + RMAppEvent appAttemptLaunched = + new RMAppEvent(application.getApplicationId(), + RMAppEventType.ATTEMPT_LAUNCHED, 1234L); + application.handle(appAttemptLaunched); + rmDispatcher.await(); + assertAppState(RMAppState.ACCEPTED, application); + assertAppStateLaunchTimeSaved(1234L); + } + @Test public void testAppAcceptedAttemptKilled() throws IOException, InterruptedException {