From 9fae455e26e0230107e1c6db58a49a5b6b296cf4 Mon Sep 17 00:00:00 2001 From: Xuan Date: Mon, 23 Mar 2015 20:33:16 -0700 Subject: [PATCH] YARN-3393. Getting application(s) goes wrong when app finishes before starting the attempt. Contributed by Zhijie Shen --- hadoop-yarn-project/CHANGES.txt | 3 ++ ...licationHistoryManagerOnTimelineStore.java | 13 +++---- ...licationHistoryManagerOnTimelineStore.java | 39 ++++++++++++++++--- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e7d4f590d29..3d9f2710ca3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -828,6 +828,9 @@ Release 2.7.0 - UNRELEASED YARN-3336. FileSystem memory leak in DelegationTokenRenewer. (Zhihai Xu via cnauroth) + YARN-3393. Getting application(s) goes wrong when app finishes before + starting the attempt. (Zhijie Shen via xgong) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 1010f62835e..49041c75cf5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -517,15 +517,14 @@ private ApplicationReportExt generateApplicationReport(TimelineEntity entity, if (app.appReport.getCurrentApplicationAttemptId() != null) { ApplicationAttemptReport appAttempt = getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId()); - if (appAttempt != null) { - app.appReport.setHost(appAttempt.getHost()); - app.appReport.setRpcPort(appAttempt.getRpcPort()); - app.appReport.setTrackingUrl(appAttempt.getTrackingUrl()); - app.appReport.setOriginalTrackingUrl(appAttempt.getOriginalTrackingUrl()); - } + app.appReport.setHost(appAttempt.getHost()); + app.appReport.setRpcPort(appAttempt.getRpcPort()); + app.appReport.setTrackingUrl(appAttempt.getTrackingUrl()); + app.appReport.setOriginalTrackingUrl(appAttempt.getOriginalTrackingUrl()); } - } catch (AuthorizationException e) { + } catch (AuthorizationException | ApplicationAttemptNotFoundException e) { // AuthorizationException is thrown because the user doesn't have access + // It's possible that the app is finished before the first attempt is created. app.appReport.setDiagnostics(null); app.appReport.setCurrentApplicationAttemptId(null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index 50a15f1bfd3..8cf12403405 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -76,6 +76,10 @@ public class TestApplicationHistoryManagerOnTimelineStore { @BeforeClass public static void prepareStore() throws Exception { store = createStore(SCALE); + TimelineEntities entities = new TimelineEntities(); + entities.addEntity(createApplicationTimelineEntity( + ApplicationId.newInstance(0, SCALE + 1), true, false)); + store.put(entities); } public static TimelineStore createStore(int scale) throws Exception { @@ -129,9 +133,9 @@ private static void prepareTimelineStore(TimelineStore store, int scale) TimelineEntities entities = new TimelineEntities(); ApplicationId appId = ApplicationId.newInstance(0, i); if (i == 2) { - entities.addEntity(createApplicationTimelineEntity(appId, true)); + entities.addEntity(createApplicationTimelineEntity(appId, true, true)); } else { - entities.addEntity(createApplicationTimelineEntity(appId, false)); + entities.addEntity(createApplicationTimelineEntity(appId, false, true)); } store.put(entities); for (int j = 1; j <= scale; ++j) { @@ -215,6 +219,27 @@ public ApplicationReport run() throws Exception { } } + @Test + public void testGetApplicationReportWithNotAttempt() throws Exception { + final ApplicationId appId = ApplicationId.newInstance(0, SCALE + 1); + ApplicationReport app; + if (callerUGI == null) { + app = historyManager.getApplication(appId); + } else { + app = + callerUGI.doAs(new PrivilegedExceptionAction () { + @Override + public ApplicationReport run() throws Exception { + return historyManager.getApplication(appId); + } + }); + } + Assert.assertNotNull(app); + Assert.assertEquals(appId, app.getApplicationId()); + Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1), + app.getCurrentApplicationAttemptId()); + } + @Test public void testGetApplicationAttemptReport() throws Exception { final ApplicationAttemptId appAttemptId = @@ -308,7 +333,7 @@ public void testGetApplications() throws Exception { Collection apps = historyManager.getAllApplications().values(); Assert.assertNotNull(apps); - Assert.assertEquals(SCALE, apps.size()); + Assert.assertEquals(SCALE + 1, apps.size()); } @Test @@ -408,7 +433,7 @@ public ContainerReport run() throws Exception { } private static TimelineEntity createApplicationTimelineEntity( - ApplicationId appId, boolean emptyACLs) { + ApplicationId appId, boolean emptyACLs, boolean noAttempt) { TimelineEntity entity = new TimelineEntity(); entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE); entity.setEntityId(appId.toString()); @@ -447,8 +472,10 @@ private static TimelineEntity createApplicationTimelineEntity( FinalApplicationStatus.UNDEFINED.toString()); eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, YarnApplicationState.FINISHED.toString()); - eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, - ApplicationAttemptId.newInstance(appId, 1)); + if (noAttempt) { + eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, + ApplicationAttemptId.newInstance(appId, 1)); + } tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); return entity;