Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6598. History server getApplicationReport NPE when fetching report for pre-2.8 job (Jason Lowe via jeagles)

This commit is contained in:
Jonathan Eagles 2017-05-15 10:32:01 -05:00
parent 6600abbb5c
commit c48f2976a3
2 changed files with 48 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
View File

@ -330,20 +330,19 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
} }
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
long vcoreSeconds=Long.parseLong(entityInfo.get( long vcoreSeconds = parseLong(entityInfo,
ApplicationMetricsConstants.APP_CPU_METRICS).toString()); ApplicationMetricsConstants.APP_CPU_METRICS);
long memorySeconds=Long.parseLong(entityInfo.get( long memorySeconds = parseLong(entityInfo,
ApplicationMetricsConstants.APP_MEM_METRICS).toString()); ApplicationMetricsConstants.APP_MEM_METRICS);
long preemptedMemorySeconds = Long.parseLong(entityInfo.get( long preemptedMemorySeconds = parseLong(entityInfo,
ApplicationMetricsConstants ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
.APP_MEM_PREEMPT_METRICS).toString()); long preemptedVcoreSeconds = parseLong(entityInfo,
long preemptedVcoreSeconds = Long.parseLong(entityInfo.get( ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
ApplicationMetricsConstants appResources = ApplicationResourceUsageReport.newInstance(0, 0, null,
.APP_CPU_PREEMPT_METRICS).toString()); null, null, memorySeconds, vcoreSeconds, 0, 0,
appResources = ApplicationResourceUsageReport preemptedMemorySeconds, preemptedVcoreSeconds);
.newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds, 0,
0, preemptedMemorySeconds, preemptedVcoreSeconds);
} }
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
appTags = new HashSet<String>(); appTags = new HashSet<String>();
Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO); Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
@ -445,6 +444,16 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
amNodeLabelExpression), appViewACLs); amNodeLabelExpression), appViewACLs);
} }
private static long parseLong(Map<String, Object> entityInfo,
String infoKey) {
long result = 0;
Object infoValue = entityInfo.get(infoKey);
if (infoValue != null) {
result = Long.parseLong(infoValue.toString());
}
return result;
}
private static boolean isFinalState(YarnApplicationState state) { private static boolean isFinalState(YarnApplicationState state) {
return state == YarnApplicationState.FINISHED return state == YarnApplicationState.FINISHED
|| state == YarnApplicationState.FAILED || state == YarnApplicationState.FAILED

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
View File

@ -143,6 +143,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
if (i == 2) { if (i == 2) {
entities.addEntity(createApplicationTimelineEntity( entities.addEntity(createApplicationTimelineEntity(
appId, true, false, false, true, YarnApplicationState.FINISHED)); appId, true, false, false, true, YarnApplicationState.FINISHED));
} else if (i == 3) {
entities.addEntity(createApplicationTimelineEntity(
appId, false, false, false, false, YarnApplicationState.FINISHED,
true));
} else { } else {
entities.addEntity(createApplicationTimelineEntity( entities.addEntity(createApplicationTimelineEntity(
appId, false, false, false, false, YarnApplicationState.FINISHED)); appId, false, false, false, false, YarnApplicationState.FINISHED));
@ -176,7 +180,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
@Test @Test
public void testGetApplicationReport() throws Exception { public void testGetApplicationReport() throws Exception {
for (int i = 1; i <= 2; ++i) { for (int i = 1; i <= 3; ++i) {
final ApplicationId appId = ApplicationId.newInstance(0, i); final ApplicationId appId = ApplicationId.newInstance(0, i);
ApplicationReport app; ApplicationReport app;
if (callerUGI == null) { if (callerUGI == null) {
@ -214,7 +218,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2")); Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used. // App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app. // Nobody except admin and owner has access to the details of the app.
if ((i == 1 && callerUGI != null && if ((i != 2 && callerUGI != null &&
callerUGI.getShortUserName().equals("user3")) || callerUGI.getShortUserName().equals("user3")) ||
(i == 2 && callerUGI != null && (i == 2 && callerUGI != null &&
(callerUGI.getShortUserName().equals("user2") || (callerUGI.getShortUserName().equals("user2") ||
@ -245,10 +249,16 @@ public class TestApplicationHistoryManagerOnTimelineStore {
applicationResourceUsageReport.getMemorySeconds()); applicationResourceUsageReport.getMemorySeconds());
Assert Assert
.assertEquals(345, applicationResourceUsageReport.getVcoreSeconds()); .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
Assert.assertEquals(456, long expectedPreemptMemSecs = 456;
long expectedPreemptVcoreSecs = 789;
if (i == 3) {
expectedPreemptMemSecs = 0;
expectedPreemptVcoreSecs = 0;
}
Assert.assertEquals(expectedPreemptMemSecs,
applicationResourceUsageReport.getPreemptedMemorySeconds()); applicationResourceUsageReport.getPreemptedMemorySeconds());
Assert Assert
.assertEquals(789, applicationResourceUsageReport .assertEquals(expectedPreemptVcoreSecs, applicationResourceUsageReport
.getPreemptedVcoreSeconds()); .getPreemptedVcoreSeconds());
Assert.assertEquals(FinalApplicationStatus.UNDEFINED, Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
app.getFinalApplicationStatus()); app.getFinalApplicationStatus());
@ -486,6 +496,14 @@ public class TestApplicationHistoryManagerOnTimelineStore {
ApplicationId appId, boolean emptyACLs, boolean noAttemptId, ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
boolean wrongAppId, boolean enableUpdateEvent, boolean wrongAppId, boolean enableUpdateEvent,
YarnApplicationState state) { YarnApplicationState state) {
return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId,
wrongAppId, enableUpdateEvent, state, false);
}
private static TimelineEntity createApplicationTimelineEntity(
ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
boolean wrongAppId, boolean enableUpdateEvent,
YarnApplicationState state, boolean missingPreemptMetrics) {
TimelineEntity entity = new TimelineEntity(); TimelineEntity entity = new TimelineEntity();
entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE); entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
if (wrongAppId) { if (wrongAppId) {
@ -510,8 +528,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Integer.MAX_VALUE + 1L); Integer.MAX_VALUE + 1L);
entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123); entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345); entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345);
entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,456); if (!missingPreemptMetrics) {
entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,789); entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789);
}
if (emptyACLs) { if (emptyACLs) {
entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, ""); entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "");
} else { } else {