Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6616: YARN AHS shows submitTime for jobs same as startTime. Contributed by Prabhu Joseph 

(cherry picked from commit 04105bbfdb)
This commit is contained in:
Eric E Payne 2019-01-29 17:22:24 +00:00
parent 9688ed3b48
commit c6c5e94de4
10 changed files with 91 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
View File

@ -62,6 +62,22 @@ public abstract class ApplicationReport {
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken) {
return newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url,
startTime, startTime, launchTime, finishTime, finalStatus, appResources,
origTrackingUrl, progress, applicationType, amRmToken);
}
@Private
@Unstable
public static ApplicationReport newInstance(ApplicationId applicationId,
ApplicationAttemptId applicationAttemptId, String user, String queue,
String name, String host, int rpcPort, Token clientToAMToken,
YarnApplicationState state, String diagnostics, String url,
long startTime, long submitTime, long launchTime, long finishTime,
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken) {
ApplicationReport report = Records.newRecord(ApplicationReport.class);
report.setApplicationId(applicationId);
report.setCurrentApplicationAttemptId(applicationAttemptId);
@ -75,6 +91,7 @@ public abstract class ApplicationReport {
report.setDiagnostics(diagnostics);
report.setTrackingUrl(url);
report.setStartTime(startTime);
report.setSubmitTime(submitTime);
report.setLaunchTime(launchTime);
report.setFinishTime(finishTime);
report.setFinalApplicationStatus(finalStatus);
@ -102,7 +119,7 @@ public abstract class ApplicationReport {
ApplicationReport report =
newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url,
startTime, 0, finishTime, finalStatus, appResources,
startTime, 0, 0, finishTime, finalStatus, appResources,
origTrackingUrl, progress, applicationType, amRmToken);
report.setApplicationTags(tags);
report.setUnmanagedApp(unmanagedApplication);
@ -124,10 +141,30 @@ public abstract class ApplicationReport {
float progress, String applicationType, Token amRmToken, Set<String> tags,
boolean unmanagedApplication, Priority priority,
String appNodeLabelExpression, String amNodeLabelExpression) {
return newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url,
startTime, startTime, launchTime, finishTime, finalStatus, appResources,
origTrackingUrl, progress, applicationType, amRmToken, tags,
unmanagedApplication, priority, appNodeLabelExpression,
amNodeLabelExpression);
}
@Private
@Unstable
public static ApplicationReport newInstance(ApplicationId applicationId,
ApplicationAttemptId applicationAttemptId, String user, String queue,
String name, String host, int rpcPort, Token clientToAMToken,
YarnApplicationState state, String diagnostics, String url,
long startTime, long submitTime, long launchTime, long finishTime,
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken, Set<String> tags,
boolean unmanagedApplication, Priority priority,
String appNodeLabelExpression, String amNodeLabelExpression) {
ApplicationReport report =
newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
launchTime, finishTime, finalStatus, appResources,
submitTime, launchTime, finishTime, finalStatus, appResources,
origTrackingUrl, progress, applicationType, amRmToken);
report.setApplicationTags(tags);
report.setUnmanagedApp(unmanagedApplication);
@ -311,6 +348,14 @@ public abstract class ApplicationReport {
@Unstable
public abstract void setStartTime(long startTime);
@Public
@Stable
public abstract long getSubmitTime();
@Private
@Unstable
public abstract void setSubmitTime(long submitTime);
@Private
@Unstable
public abstract void setLaunchTime(long setLaunchTime);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -247,6 +247,7 @@ message ApplicationReportProto {
optional string amNodeLabelExpression = 25;
repeated AppTimeoutsMapProto appTimeouts = 26;
optional int64 launchTime = 27;
optional int64 submitTime = 28;
}
message AppTimeoutsMapProto {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
View File

@ -102,6 +102,8 @@ public class TestAHSClient {
Assert.assertEquals(report, expectedReports.get(0));
Assert.assertEquals(report.getApplicationId().toString(), expectedReports
.get(0).getApplicationId().toString());
Assert.assertEquals(report.getSubmitTime(), expectedReports.get(0)
.getSubmitTime());
client.stop();
}
@ -332,7 +334,7 @@ public class TestAHSClient {
ApplicationReport.newInstance(applicationId,
ApplicationAttemptId.newInstance(applicationId, 1), "user",
"queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0,
YarnApplicationState.RUNNING, "diagnostics", "url", 1, 2, 3, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
null);
List<ApplicationReport> applicationReports =

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
View File

@ -220,6 +220,12 @@ public class ApplicationReportPBImpl extends ApplicationReport {
return p.getStartTime();
}
@Override
public long getSubmitTime() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
return p.getSubmitTime();
}
@Override
public long getLaunchTime() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
@ -430,6 +436,12 @@ public class ApplicationReportPBImpl extends ApplicationReport {
builder.setFinishTime(finishTime);
}
@Override
public void setSubmitTime(long submitTime) {
maybeInitBuilder();
builder.setSubmitTime(submitTime);
}
@Override
public void setFinalApplicationStatus(FinalApplicationStatus finishState) {
maybeInitBuilder();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
View File

@ -137,9 +137,9 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(),
appHistory.getApplicationName(), host, rpcPort, null,
appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(),
trackingUrl, appHistory.getStartTime(), 0, appHistory.getFinishTime(),
appHistory.getFinalApplicationStatus(), null, "", 100,
appHistory.getApplicationType(), null);
trackingUrl, appHistory.getStartTime(), appHistory.getSubmitTime(), 0,
appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(),
null, "", 100, appHistory.getApplicationType(), null);
}
private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId)

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
View File

@ -249,6 +249,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
String type = null;
boolean unmanagedApplication = false;
long createdTime = 0;
long submittedTime = 0;
long finishedTime = 0;
float progress = 0.0f;
int applicationPriority = 0;
@ -280,10 +281,11 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
return new ApplicationReportExt(ApplicationReport.newInstance(
ApplicationId.fromString(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null,
state, diagnosticsInfo, null, createdTime, finishedTime,
finalStatus, null, null, progress, type, null, appTags,
unmanagedApplication, Priority.newInstance(applicationPriority),
appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
state, diagnosticsInfo, null, createdTime, submittedTime, 0,
finishedTime, finalStatus, null, null, progress, type, null,
appTags, unmanagedApplication, Priority.newInstance(
applicationPriority), appNodeLabelExpression,
amNodeLabelExpression), appViewACLs);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
queue =
@ -328,6 +330,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
entityInfo.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)
.toString();
}
submittedTime = parseLong(entityInfo,
ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO);
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
long vcoreSeconds = parseLong(entityInfo,
@ -438,10 +442,10 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
return new ApplicationReportExt(ApplicationReport.newInstance(
ApplicationId.fromString(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
appResources, null, progress, type, null, appTags, unmanagedApplication,
Priority.newInstance(applicationPriority), appNodeLabelExpression,
amNodeLabelExpression), appViewACLs);
diagnosticsInfo, null, createdTime, submittedTime, 0, finishedTime,
finalStatus, appResources, null, progress, type, null, appTags,
unmanagedApplication, Priority.newInstance(applicationPriority),
appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
}
private static long parseLong(Map<String, Object> entityInfo,

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
View File

@ -214,6 +214,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
}
Assert.assertEquals(Integer.MAX_VALUE + 2L
+ app.getApplicationId().getId(), app.getStartTime());
Assert.assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime());
Assert.assertEquals(Integer.MAX_VALUE + 3L
+ +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
View File

@ -401,6 +401,7 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals("test app", app.get("name"));
assertEquals(round == 0 ? "test diagnostics info" : "",
app.get("diagnosticsInfo"));
assertEquals(Integer.MAX_VALUE + 1L, app.get("submittedTime"));
assertEquals("test queue", app.get("queue"));
assertEquals("user1", app.get("user"));
assertEquals("test app type", app.get("type"));

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
View File

@ -53,7 +53,7 @@ public class AppInfo {
protected String originalTrackingUrl;
protected String trackingUrl;
protected FinalApplicationStatus finalAppStatus;
protected long submittedTime;
private long submittedTime;
protected long startedTime;
private long launchTime;
protected long finishedTime;
@ -87,7 +87,7 @@ public class AppInfo {
diagnosticsInfo = app.getDiagnostics();
trackingUrl = app.getTrackingUrl();
originalTrackingUrl = app.getOriginalTrackingUrl();
submittedTime = app.getStartTime();
submittedTime = app.getSubmitTime();
startedTime = app.getStartTime();
launchTime = app.getLaunchTime();
finishedTime = app.getFinishTime();

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
View File

@ -263,6 +263,7 @@ public abstract class MockAsm extends MockApps {
final String name = newAppName();
final String queue = newQueue();
final long start = 123456 + i * 1000;
final long submit = start + i * 50;
final long launch = start + i * 100;
final long finish = 234567 + i * 1000;
final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE;
@ -299,6 +300,11 @@ public abstract class MockAsm extends MockApps {
return start;
}
@Override
public long getSubmitTime() {
return submit;
}
@Override
public long getLaunchTime() {
return launch;
@ -353,9 +359,9 @@ public abstract class MockAsm extends MockApps {
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),
getTrackingUrl(), getLaunchTime(), getStartTime(), getFinishTime(),
getFinalApplicationStatus(), usageReport , null, getProgress(),
type, null);
getTrackingUrl(), getStartTime(), getSubmitTime(), getLaunchTime(),
getFinishTime(), getFinalApplicationStatus(), usageReport, null,
getProgress(), type, null);
return report;
}