From eb4d1ed61283f497361e7cc340ac820a56ecd4a9 Mon Sep 17 00:00:00 2001 From: Xuan Date: Fri, 22 May 2015 14:34:39 -0700 Subject: [PATCH] YARN-3701. Isolating the error of generating a single app report when getting all apps from generic history service. Contributed by Zhijie Shen (cherry picked from commit 455b3acf0e82b214e06bd7b538968252945cd3c4) (cherry picked from commit 33be070a5e944bdc04693e7be6f39fd3d145d8d1) --- hadoop-yarn-project/CHANGES.txt | 3 ++ ...licationHistoryManagerOnTimelineStore.java | 37 ++++++++++++++----- ...licationHistoryManagerOnTimelineStore.java | 25 ++++++++++--- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 92f8c3fa8ec..15e6ce15e71 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -122,6 +122,9 @@ Release 2.7.1 - UNRELEASED YARN-3675. FairScheduler: RM quits when node removal races with continuous-scheduling on the same node. (Anubhav Dhoot via kasha) + YARN-3701. Isolating the error of generating a single app report when + getting all apps from generic history service. (Zhijie Shen via xgong) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 49041c75cf5..5f4510bd60a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -25,6 +25,8 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; @@ -67,6 +69,8 @@ import com.google.common.annotations.VisibleForTesting; public class ApplicationHistoryManagerOnTimelineStore extends AbstractService implements ApplicationHistoryManager { + private static final Log LOG = LogFactory + .getLog(ApplicationHistoryManagerOnTimelineStore.class); @VisibleForTesting static final String UNAVAILABLE = "N/A"; @@ -107,9 +111,14 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService new LinkedHashMap(); if (entities != null && entities.getEntities() != null) { for (TimelineEntity entity : entities.getEntities()) { - ApplicationReportExt app = - generateApplicationReport(entity, ApplicationReportField.ALL); - apps.put(app.appReport.getApplicationId(), app.appReport); + try { + ApplicationReportExt app = + generateApplicationReport(entity, ApplicationReportField.ALL); + apps.put(app.appReport.getApplicationId(), app.appReport); + } catch (Exception e) { + LOG.error("Error on generating application report for " + + entity.getEntityId(), e); + } } } return apps; @@ -142,9 +151,18 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService @Override public ApplicationAttemptReport getApplicationAttempt( ApplicationAttemptId appAttemptId) throws YarnException, IOException { - ApplicationReportExt app = getApplication( - appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS); - checkAccess(app); + return getApplicationAttempt(appAttemptId, true); + } + + private ApplicationAttemptReport getApplicationAttempt( + ApplicationAttemptId appAttemptId, boolean checkACLs) + throws YarnException, IOException { + if (checkACLs) { + ApplicationReportExt app = getApplication( + appAttemptId.getApplicationId(), + ApplicationReportField.USER_AND_ACLS); + checkAccess(app); + } TimelineEntity entity = timelineDataManager.getEntity( AppAttemptMetricsConstants.ENTITY_TYPE, appAttemptId.toString(), EnumSet.allOf(Field.class), @@ -182,7 +200,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService @Override public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) throws YarnException, IOException { - ApplicationAttemptReport appAttempt = getApplicationAttempt(appAttemptId); + ApplicationAttemptReport appAttempt = + getApplicationAttempt(appAttemptId, false); return getContainer(appAttempt.getAMContainerId()); } @@ -515,8 +534,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService try { checkAccess(app); if (app.appReport.getCurrentApplicationAttemptId() != null) { - ApplicationAttemptReport appAttempt = - getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId()); + ApplicationAttemptReport appAttempt = getApplicationAttempt( + app.appReport.getCurrentApplicationAttemptId(), false); app.appReport.setHost(appAttempt.getHost()); app.appReport.setRpcPort(appAttempt.getRpcPort()); app.appReport.setTrackingUrl(appAttempt.getTrackingUrl()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index 8cf12403405..fcdafc29442 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -78,7 +78,9 @@ public class TestApplicationHistoryManagerOnTimelineStore { store = createStore(SCALE); TimelineEntities entities = new TimelineEntities(); entities.addEntity(createApplicationTimelineEntity( - ApplicationId.newInstance(0, SCALE + 1), true, false)); + ApplicationId.newInstance(0, SCALE + 1), true, true, false)); + entities.addEntity(createApplicationTimelineEntity( + ApplicationId.newInstance(0, SCALE + 2), true, false, true)); store.put(entities); } @@ -133,9 +135,11 @@ public class TestApplicationHistoryManagerOnTimelineStore { TimelineEntities entities = new TimelineEntities(); ApplicationId appId = ApplicationId.newInstance(0, i); if (i == 2) { - entities.addEntity(createApplicationTimelineEntity(appId, true, true)); + entities.addEntity(createApplicationTimelineEntity( + appId, true, false, false)); } else { - entities.addEntity(createApplicationTimelineEntity(appId, false, true)); + entities.addEntity(createApplicationTimelineEntity( + appId, false, false, false)); } store.put(entities); for (int j = 1; j <= scale; ++j) { @@ -334,6 +338,10 @@ public class TestApplicationHistoryManagerOnTimelineStore { historyManager.getAllApplications().values(); Assert.assertNotNull(apps); Assert.assertEquals(SCALE + 1, apps.size()); + ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2); + for (ApplicationReport app : apps) { + Assert.assertNotEquals(ignoredAppId, app.getApplicationId()); + } } @Test @@ -433,10 +441,15 @@ public class TestApplicationHistoryManagerOnTimelineStore { } private static TimelineEntity createApplicationTimelineEntity( - ApplicationId appId, boolean emptyACLs, boolean noAttempt) { + ApplicationId appId, boolean emptyACLs, boolean noAttemptId, + boolean wrongAppId) { TimelineEntity entity = new TimelineEntity(); entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE); - entity.setEntityId(appId.toString()); + if (wrongAppId) { + entity.setEntityId("wrong_app_id"); + } else { + entity.setEntityId(appId.toString()); + } entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID); entity.addPrimaryFilter( TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn"); @@ -472,7 +485,7 @@ public class TestApplicationHistoryManagerOnTimelineStore { FinalApplicationStatus.UNDEFINED.toString()); eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, YarnApplicationState.FINISHED.toString()); - if (noAttempt) { + if (!noAttemptId) { eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, ApplicationAttemptId.newInstance(appId, 1)); }