YARN-3701. Isolating the error of generating a single app report when

getting all apps from generic history service. Contributed by Zhijie
Shen

(cherry picked from commit 455b3acf0e)
(cherry picked from commit 33be070a5e)

hadoop-yarn-project/CHANGES.txt

@@ -122,6 +122,9 @@ Release 2.7.1 - UNRELEASED
     YARN-3675. FairScheduler: RM quits when node removal races with 
     continuous-scheduling on the same node. (Anubhav Dhoot via kasha)
 
+    YARN-3701. Isolating the error of generating a single app report when 
+    getting all apps from generic history service. (Zhijie Shen via xgong)
+
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java

@@ -25,6 +25,8 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AuthorizationException;
@@ -67,6 +69,8 @@ import com.google.common.annotations.VisibleForTesting;
 public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
     implements
     ApplicationHistoryManager {
+  private static final Log LOG = LogFactory
+      .getLog(ApplicationHistoryManagerOnTimelineStore.class);
 
   @VisibleForTesting
   static final String UNAVAILABLE = "N/A";
@@ -107,9 +111,14 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
         new LinkedHashMap<ApplicationId, ApplicationReport>();
     if (entities != null && entities.getEntities() != null) {
       for (TimelineEntity entity : entities.getEntities()) {
-        ApplicationReportExt app =
+        try {
-            generateApplicationReport(entity, ApplicationReportField.ALL);
+          ApplicationReportExt app =
-        apps.put(app.appReport.getApplicationId(), app.appReport);
+              generateApplicationReport(entity, ApplicationReportField.ALL);
+          apps.put(app.appReport.getApplicationId(), app.appReport);
+        } catch (Exception e) {
+          LOG.error("Error on generating application report for " +
+              entity.getEntityId(), e);
+        }
       }
     }
     return apps;
@@ -142,9 +151,18 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
   @Override
   public ApplicationAttemptReport getApplicationAttempt(
       ApplicationAttemptId appAttemptId) throws YarnException, IOException {
-    ApplicationReportExt app = getApplication(
+    return getApplicationAttempt(appAttemptId, true);
-        appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
+  }
-    checkAccess(app);
+
+  private ApplicationAttemptReport getApplicationAttempt(
+      ApplicationAttemptId appAttemptId, boolean checkACLs)
+      throws YarnException, IOException {
+    if (checkACLs) {
+      ApplicationReportExt app = getApplication(
+          appAttemptId.getApplicationId(),
+          ApplicationReportField.USER_AND_ACLS);
+      checkAccess(app);
+    }
     TimelineEntity entity = timelineDataManager.getEntity(
         AppAttemptMetricsConstants.ENTITY_TYPE,
         appAttemptId.toString(), EnumSet.allOf(Field.class),
@@ -182,7 +200,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
   @Override
   public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
       throws YarnException, IOException {
-    ApplicationAttemptReport appAttempt = getApplicationAttempt(appAttemptId);
+    ApplicationAttemptReport appAttempt =
+        getApplicationAttempt(appAttemptId, false);
     return getContainer(appAttempt.getAMContainerId());
   }
 
@@ -515,8 +534,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
     try {
       checkAccess(app);
       if (app.appReport.getCurrentApplicationAttemptId() != null) {
-        ApplicationAttemptReport appAttempt =
+        ApplicationAttemptReport appAttempt = getApplicationAttempt(
-            getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId());
+            app.appReport.getCurrentApplicationAttemptId(), false);
         app.appReport.setHost(appAttempt.getHost());
         app.appReport.setRpcPort(appAttempt.getRpcPort());
         app.appReport.setTrackingUrl(appAttempt.getTrackingUrl());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java

@@ -78,7 +78,9 @@ public class TestApplicationHistoryManagerOnTimelineStore {
     store = createStore(SCALE);
     TimelineEntities entities = new TimelineEntities();
     entities.addEntity(createApplicationTimelineEntity(
-        ApplicationId.newInstance(0, SCALE + 1), true, false));
+        ApplicationId.newInstance(0, SCALE + 1), true, true, false));
+    entities.addEntity(createApplicationTimelineEntity(
+        ApplicationId.newInstance(0, SCALE + 2), true, false, true));
     store.put(entities);
   }
 
@@ -133,9 +135,11 @@ public class TestApplicationHistoryManagerOnTimelineStore {
       TimelineEntities entities = new TimelineEntities();
       ApplicationId appId = ApplicationId.newInstance(0, i);
       if (i == 2) {
-        entities.addEntity(createApplicationTimelineEntity(appId, true, true));
+        entities.addEntity(createApplicationTimelineEntity(
+            appId, true, false, false));
       } else {
-        entities.addEntity(createApplicationTimelineEntity(appId, false, true));
+        entities.addEntity(createApplicationTimelineEntity(
+            appId, false, false, false));
       }
       store.put(entities);
       for (int j = 1; j <= scale; ++j) {
@@ -334,6 +338,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
         historyManager.getAllApplications().values();
     Assert.assertNotNull(apps);
     Assert.assertEquals(SCALE + 1, apps.size());
+    ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
+    for (ApplicationReport app : apps) {
+      Assert.assertNotEquals(ignoredAppId, app.getApplicationId());
+    }
   }
 
   @Test
@@ -433,10 +441,15 @@ public class TestApplicationHistoryManagerOnTimelineStore {
   }
 
   private static TimelineEntity createApplicationTimelineEntity(
-      ApplicationId appId, boolean emptyACLs, boolean noAttempt) {
+      ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
+      boolean wrongAppId) {
     TimelineEntity entity = new TimelineEntity();
     entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
-    entity.setEntityId(appId.toString());
+    if (wrongAppId) {
+      entity.setEntityId("wrong_app_id");
+    } else {
+      entity.setEntityId(appId.toString());
+    }
     entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
     entity.addPrimaryFilter(
         TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
@@ -472,7 +485,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
         FinalApplicationStatus.UNDEFINED.toString());
     eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
         YarnApplicationState.FINISHED.toString());
-    if (noAttempt) {
+    if (!noAttemptId) {
       eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
           ApplicationAttemptId.newInstance(appId, 1));
     }