From 7db50ffcebf5413bbd1a80ee2cd8e2ffc11befe1 Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Fri, 22 Feb 2019 20:05:29 +0530 Subject: [PATCH] YARN-8132. Final Status of applications shown as UNDEFINED in ATS app queries. Contributed by Prabhu Joseph --- .../rmapp/attempt/RMAppAttemptImpl.java | 12 +++--- .../rmapp/TestRMAppTransitions.java | 43 +++++++++++-------- .../attempt/TestRMAppAttemptTransitions.java | 3 +- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 03039daae96..7b04ae7e72c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1372,7 +1372,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { setTrackingUrlToRMAppPage(stateToBeStored); } String finalTrackingUrl = getOriginalTrackingUrl(); - FinalApplicationStatus finalStatus = null; + FinalApplicationStatus status = null; int exitStatus = ContainerExitStatus.INVALID; switch (event.getType()) { case LAUNCH_FAILED: @@ -1387,7 +1387,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { diags.append(unregisterEvent.getDiagnosticMsg()); // reset finalTrackingUrl to url sent by am finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl()); - finalStatus = unregisterEvent.getFinalApplicationStatus(); + status = unregisterEvent.getFinalApplicationStatus(); break; case CONTAINER_FINISHED: RMAppAttemptContainerFinishedEvent finishEvent = @@ -1396,8 +1396,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { exitStatus = finishEvent.getContainerStatus().getExitStatus(); break; case KILL: + status = FinalApplicationStatus.KILLED; break; case FAIL: + status = FinalApplicationStatus.FAILED; diags.append(event.getDiagnosticMsg()); break; case EXPIRE: @@ -1414,13 +1416,14 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { ApplicationAttemptStateData attemptState = ApplicationAttemptStateData .newInstance(applicationAttemptId, getMasterContainer(), rmStore.getCredentialsFromAppAttempt(this), startTime, - stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus, - getFinishTime(), resUsage.getResourceUsageSecondsMap(), + stateToBeStored, finalTrackingUrl, diags.toString(), status, + exitStatus, getFinishTime(), resUsage.getResourceUsageSecondsMap(), this.attemptMetrics.getPreemptedResourceSecondsMap()); LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus); rmStore.updateApplicationAttemptState(attemptState); + finalStatus = status; } private static class FinalSavingTransition extends BaseTransition { @@ -1916,7 +1919,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { (RMAppAttemptUnregistrationEvent) event; this.diagnostics.append(unregisterEvent.getDiagnosticMsg()); originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl()); - finalStatus = unregisterEvent.getFinalApplicationStatus(); } private static final class ContainerFinishedTransition diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 70887e0f4bf..1cd22e8b36d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -694,7 +694,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateNotSaved(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -713,7 +713,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -731,7 +731,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); verifyRMAppFieldsForFinalTransitions(application); rmContext.getStateStore().removeApplication(application); } @@ -753,7 +753,7 @@ public class TestRMAppTransitions { rmDispatcher.await(); sendAppUpdateSavedEvent(application); assertKilled(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -772,7 +772,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -789,7 +789,7 @@ public class TestRMAppTransitions { assertFailed(application, rejectedText); verify(store, times(0)).updateApplicationState( any(ApplicationStateData.class)); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); assertTimesAtFinish(application); } @@ -807,7 +807,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -829,7 +829,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -866,7 +866,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertFailed(application, ".*" + message + ".*Failing the application.*"); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); } @Test @@ -893,7 +893,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -917,7 +917,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); } @@ -941,7 +941,7 @@ public class TestRMAppTransitions { sendAttemptUpdateSavedEvent(application); sendAppUpdateSavedEvent(application); assertKilled(application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1000,7 +1000,7 @@ public class TestRMAppTransitions { rmDispatcher.await(); assertFailed(application, ".*Failing the application.*"); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1062,7 +1062,9 @@ public class TestRMAppTransitions { StringBuilder diag = application.getDiagnostics(); Assert.assertEquals("application diagnostics is not correct", "", diag.toString()); - verifyApplicationFinished(RMAppState.FINISHED); + // finished without a proper final state is the same as failed + verifyApplicationFinished(RMAppState.FINISHED, + FinalApplicationStatus.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1089,7 +1091,7 @@ public class TestRMAppTransitions { rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); - verifyApplicationFinished(RMAppState.FAILED); + verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); @@ -1144,7 +1146,7 @@ public class TestRMAppTransitions { rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); - verifyApplicationFinished(RMAppState.KILLED); + verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); @@ -1257,15 +1259,20 @@ public class TestRMAppTransitions { + "/")); } - private void verifyApplicationFinished(RMAppState state) { + private void verifyApplicationFinished(RMAppState state, + FinalApplicationStatus finalAppStatus) { ArgumentCaptor finalState = ArgumentCaptor.forClass(RMAppState.class); verify(writer).applicationFinished(any(RMApp.class), finalState.capture()); Assert.assertEquals(state, finalState.getValue()); finalState = ArgumentCaptor.forClass(RMAppState.class); - verify(publisher).appFinished(any(RMApp.class), finalState.capture(), + ArgumentCaptor app = + ArgumentCaptor.forClass(RMApp.class); + verify(publisher).appFinished(app.capture(), finalState.capture(), anyLong()); Assert.assertEquals(state, finalState.getValue()); + Assert.assertEquals(finalAppStatus, + app.getValue().getFinalApplicationStatus()); } private void verifyAppRemovedSchedulerEvent(RMAppState finalState) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index f5f8b766ae8..1d5eb543e2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -442,7 +442,8 @@ public class TestRMAppAttemptTransitions { assertEquals(amContainer, applicationAttempt.getMasterContainer()); assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001); assertEquals(0, application.getRanNodes().size()); - assertNull(applicationAttempt.getFinalApplicationStatus()); + assertEquals(FinalApplicationStatus.KILLED, + applicationAttempt.getFinalApplicationStatus()); verifyTokenCount(applicationAttempt.getAppAttemptId(), 1); verifyAttemptFinalStateSaved(); assertFalse(transferStateFromPreviousAttempt);