From e663a6af89f9bd970032d1e1cc41d203a7bcf7a5 Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Mon, 4 Mar 2019 17:03:45 +0530 Subject: [PATCH] Revert "YARN-8132. Final Status of applications shown as UNDEFINED in ATS app queries. Contributed by Prabhu Joseph" This reverts commit 7db50ffcebf5413bbd1a80ee2cd8e2ffc11befe1. --- .../rmapp/attempt/RMAppAttemptImpl.java | 12 +++--- .../rmapp/TestRMAppTransitions.java | 43 ++++++++----------- .../attempt/TestRMAppAttemptTransitions.java | 3 +- 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 7b04ae7e72c..03039daae96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1372,7 +1372,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { setTrackingUrlToRMAppPage(stateToBeStored); } String finalTrackingUrl = getOriginalTrackingUrl(); - FinalApplicationStatus status = null; + FinalApplicationStatus finalStatus = null; int exitStatus = ContainerExitStatus.INVALID; switch (event.getType()) { case LAUNCH_FAILED: @@ -1387,7 +1387,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { diags.append(unregisterEvent.getDiagnosticMsg()); // reset finalTrackingUrl to url sent by am finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl()); - status = unregisterEvent.getFinalApplicationStatus(); + finalStatus = unregisterEvent.getFinalApplicationStatus(); break; case CONTAINER_FINISHED: RMAppAttemptContainerFinishedEvent finishEvent = @@ -1396,10 +1396,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { exitStatus = finishEvent.getContainerStatus().getExitStatus(); break; case KILL: - status = FinalApplicationStatus.KILLED; break; case FAIL: - status = FinalApplicationStatus.FAILED; diags.append(event.getDiagnosticMsg()); break; case EXPIRE: @@ -1416,14 +1414,13 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { ApplicationAttemptStateData attemptState = ApplicationAttemptStateData .newInstance(applicationAttemptId, getMasterContainer(), rmStore.getCredentialsFromAppAttempt(this), startTime, - stateToBeStored, finalTrackingUrl, diags.toString(), status, - exitStatus, getFinishTime(), resUsage.getResourceUsageSecondsMap(), + stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus, + getFinishTime(), resUsage.getResourceUsageSecondsMap(), this.attemptMetrics.getPreemptedResourceSecondsMap()); LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus); rmStore.updateApplicationAttemptState(attemptState); - finalStatus = status; } private static class FinalSavingTransition extends BaseTransition { @@ -1919,6 +1916,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { (RMAppAttemptUnregistrationEvent) event; this.diagnostics.append(unregisterEvent.getDiagnosticMsg()); originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl()); + finalStatus = unregisterEvent.getFinalApplicationStatus(); } private static final class ContainerFinishedTransition diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 1cd22e8b36d..70887e0f4bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -694,7 +694,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateNotSaved(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -713,7 +713,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -731,7 +731,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); verifyRMAppFieldsForFinalTransitions(application); rmContext.getStateStore().removeApplication(application); } @@ -753,7 +753,7 @@ public class TestRMAppTransitions { rmDispatcher.await(); sendAppUpdateSavedEvent(application); assertKilled(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -772,7 +772,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -789,7 +789,7 @@ public class TestRMAppTransitions { assertFailed(application, rejectedText); verify(store, times(0)).updateApplicationState( any(ApplicationStateData.class)); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); assertTimesAtFinish(application); } @@ -807,7 +807,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -829,7 +829,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -866,7 +866,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertFailed(application, ".*" + message + ".*Failing the application.*"); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); } @Test @@ -893,7 +893,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -917,7 +917,7 @@ public class TestRMAppTransitions { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); } @@ -941,7 +941,7 @@ public class TestRMAppTransitions { sendAttemptUpdateSavedEvent(application); sendAppUpdateSavedEvent(application); assertKilled(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1000,7 +1000,7 @@ public class TestRMAppTransitions { rmDispatcher.await(); assertFailed(application, ".*Failing the application.*"); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1062,9 +1062,7 @@ public class TestRMAppTransitions { StringBuilder diag = application.getDiagnostics(); Assert.assertEquals("application diagnostics is not correct", "", diag.toString()); - // finished without a proper final state is the same as failed - verifyApplicationFinished(RMAppState.FINISHED, - FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FINISHED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1091,7 +1089,7 @@ public class TestRMAppTransitions { rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); @@ -1146,7 +1144,7 @@ public class TestRMAppTransitions { rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); @@ -1259,20 +1257,15 @@ public class TestRMAppTransitions { + "/")); } - private void verifyApplicationFinished(RMAppState state, - FinalApplicationStatus finalAppStatus) { + private void verifyApplicationFinished(RMAppState state) { ArgumentCaptor finalState = ArgumentCaptor.forClass(RMAppState.class); verify(writer).applicationFinished(any(RMApp.class), finalState.capture()); Assert.assertEquals(state, finalState.getValue()); finalState = ArgumentCaptor.forClass(RMAppState.class); - ArgumentCaptor app = - ArgumentCaptor.forClass(RMApp.class); - verify(publisher).appFinished(app.capture(), finalState.capture(), + verify(publisher).appFinished(any(RMApp.class), finalState.capture(), anyLong()); Assert.assertEquals(state, finalState.getValue()); - Assert.assertEquals(finalAppStatus, - app.getValue().getFinalApplicationStatus()); } private void verifyAppRemovedSchedulerEvent(RMAppState finalState) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 1d5eb543e2b..f5f8b766ae8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -442,8 +442,7 @@ public class TestRMAppAttemptTransitions { assertEquals(amContainer, applicationAttempt.getMasterContainer()); assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001); assertEquals(0, application.getRanNodes().size()); - assertEquals(FinalApplicationStatus.KILLED, - applicationAttempt.getFinalApplicationStatus()); + assertNull(applicationAttempt.getFinalApplicationStatus()); verifyTokenCount(applicationAttempt.getAppAttemptId(), 1); verifyAttemptFinalStateSaved(); assertFalse(transferStateFromPreviousAttempt);