YARN-3701. Isolating the error of generating a single app report when
getting all apps from generic history service. Contributed by Zhijie
Shen
(cherry picked from commit 455b3acf0e
)
This commit is contained in:
parent
97ebe1b2cf
commit
33be070a5e
|
@ -509,6 +509,9 @@ Release 2.7.1 - UNRELEASED
|
|||
YARN-3675. FairScheduler: RM quits when node removal races with
|
||||
continuous-scheduling on the same node. (Anubhav Dhoot via kasha)
|
||||
|
||||
YARN-3701. Isolating the error of generating a single app report when
|
||||
getting all apps from generic history service. (Zhijie Shen via xgong)
|
||||
|
||||
Release 2.7.0 - 2015-04-20
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
@ -25,6 +25,8 @@ import java.util.LinkedHashMap;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.authorize.AuthorizationException;
|
||||
|
@ -67,6 +69,8 @@ import com.google.common.annotations.VisibleForTesting;
|
|||
public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
||||
implements
|
||||
ApplicationHistoryManager {
|
||||
private static final Log LOG = LogFactory
|
||||
.getLog(ApplicationHistoryManagerOnTimelineStore.class);
|
||||
|
||||
@VisibleForTesting
|
||||
static final String UNAVAILABLE = "N/A";
|
||||
|
@ -107,9 +111,14 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
|||
new LinkedHashMap<ApplicationId, ApplicationReport>();
|
||||
if (entities != null && entities.getEntities() != null) {
|
||||
for (TimelineEntity entity : entities.getEntities()) {
|
||||
ApplicationReportExt app =
|
||||
generateApplicationReport(entity, ApplicationReportField.ALL);
|
||||
apps.put(app.appReport.getApplicationId(), app.appReport);
|
||||
try {
|
||||
ApplicationReportExt app =
|
||||
generateApplicationReport(entity, ApplicationReportField.ALL);
|
||||
apps.put(app.appReport.getApplicationId(), app.appReport);
|
||||
} catch (Exception e) {
|
||||
LOG.error("Error on generating application report for " +
|
||||
entity.getEntityId(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
return apps;
|
||||
|
@ -142,9 +151,18 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
|||
@Override
|
||||
public ApplicationAttemptReport getApplicationAttempt(
|
||||
ApplicationAttemptId appAttemptId) throws YarnException, IOException {
|
||||
ApplicationReportExt app = getApplication(
|
||||
appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
|
||||
checkAccess(app);
|
||||
return getApplicationAttempt(appAttemptId, true);
|
||||
}
|
||||
|
||||
private ApplicationAttemptReport getApplicationAttempt(
|
||||
ApplicationAttemptId appAttemptId, boolean checkACLs)
|
||||
throws YarnException, IOException {
|
||||
if (checkACLs) {
|
||||
ApplicationReportExt app = getApplication(
|
||||
appAttemptId.getApplicationId(),
|
||||
ApplicationReportField.USER_AND_ACLS);
|
||||
checkAccess(app);
|
||||
}
|
||||
TimelineEntity entity = timelineDataManager.getEntity(
|
||||
AppAttemptMetricsConstants.ENTITY_TYPE,
|
||||
appAttemptId.toString(), EnumSet.allOf(Field.class),
|
||||
|
@ -182,7 +200,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
|||
@Override
|
||||
public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
|
||||
throws YarnException, IOException {
|
||||
ApplicationAttemptReport appAttempt = getApplicationAttempt(appAttemptId);
|
||||
ApplicationAttemptReport appAttempt =
|
||||
getApplicationAttempt(appAttemptId, false);
|
||||
return getContainer(appAttempt.getAMContainerId());
|
||||
}
|
||||
|
||||
|
@ -517,8 +536,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
|||
try {
|
||||
checkAccess(app);
|
||||
if (app.appReport.getCurrentApplicationAttemptId() != null) {
|
||||
ApplicationAttemptReport appAttempt =
|
||||
getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId());
|
||||
ApplicationAttemptReport appAttempt = getApplicationAttempt(
|
||||
app.appReport.getCurrentApplicationAttemptId(), false);
|
||||
app.appReport.setHost(appAttempt.getHost());
|
||||
app.appReport.setRpcPort(appAttempt.getRpcPort());
|
||||
app.appReport.setTrackingUrl(appAttempt.getTrackingUrl());
|
||||
|
|
|
@ -78,7 +78,9 @@ public class TestApplicationHistoryManagerOnTimelineStore {
|
|||
store = createStore(SCALE);
|
||||
TimelineEntities entities = new TimelineEntities();
|
||||
entities.addEntity(createApplicationTimelineEntity(
|
||||
ApplicationId.newInstance(0, SCALE + 1), true, false));
|
||||
ApplicationId.newInstance(0, SCALE + 1), true, true, false));
|
||||
entities.addEntity(createApplicationTimelineEntity(
|
||||
ApplicationId.newInstance(0, SCALE + 2), true, false, true));
|
||||
store.put(entities);
|
||||
}
|
||||
|
||||
|
@ -133,9 +135,11 @@ public class TestApplicationHistoryManagerOnTimelineStore {
|
|||
TimelineEntities entities = new TimelineEntities();
|
||||
ApplicationId appId = ApplicationId.newInstance(0, i);
|
||||
if (i == 2) {
|
||||
entities.addEntity(createApplicationTimelineEntity(appId, true, true));
|
||||
entities.addEntity(createApplicationTimelineEntity(
|
||||
appId, true, false, false));
|
||||
} else {
|
||||
entities.addEntity(createApplicationTimelineEntity(appId, false, true));
|
||||
entities.addEntity(createApplicationTimelineEntity(
|
||||
appId, false, false, false));
|
||||
}
|
||||
store.put(entities);
|
||||
for (int j = 1; j <= scale; ++j) {
|
||||
|
@ -334,6 +338,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
|
|||
historyManager.getAllApplications().values();
|
||||
Assert.assertNotNull(apps);
|
||||
Assert.assertEquals(SCALE + 1, apps.size());
|
||||
ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
|
||||
for (ApplicationReport app : apps) {
|
||||
Assert.assertNotEquals(ignoredAppId, app.getApplicationId());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
|
@ -433,10 +441,15 @@ public class TestApplicationHistoryManagerOnTimelineStore {
|
|||
}
|
||||
|
||||
private static TimelineEntity createApplicationTimelineEntity(
|
||||
ApplicationId appId, boolean emptyACLs, boolean noAttempt) {
|
||||
ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
|
||||
boolean wrongAppId) {
|
||||
TimelineEntity entity = new TimelineEntity();
|
||||
entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
|
||||
entity.setEntityId(appId.toString());
|
||||
if (wrongAppId) {
|
||||
entity.setEntityId("wrong_app_id");
|
||||
} else {
|
||||
entity.setEntityId(appId.toString());
|
||||
}
|
||||
entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
|
||||
entity.addPrimaryFilter(
|
||||
TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
|
||||
|
@ -472,7 +485,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
|
|||
FinalApplicationStatus.UNDEFINED.toString());
|
||||
eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
|
||||
YarnApplicationState.FINISHED.toString());
|
||||
if (noAttempt) {
|
||||
if (!noAttemptId) {
|
||||
eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
|
||||
ApplicationAttemptId.newInstance(appId, 1));
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue