Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6616: YARN AHS shows submitTime for jobs same as startTime. Contributed by Prabhu Joseph

Browse Source 
(cherry picked from commit 04105bbfdb041a41062c856632641140de84fba8)
This commit is contained in:
Eric E Payne 2019-01-29 17:22:24 +00:00
parent 9688ed3b48
commit c6c5e94de4
10 changed files with 91 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
View File

@ -62,6 +62,22 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken) {
return newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url,
startTime, startTime, launchTime, finishTime, finalStatus, appResources,
origTrackingUrl, progress, applicationType, amRmToken);
}
@Private
@Unstable
public static ApplicationReport newInstance(ApplicationId applicationId,
ApplicationAttemptId applicationAttemptId, String user, String queue,
String name, String host, int rpcPort, Token clientToAMToken,
YarnApplicationState state, String diagnostics, String url,
long startTime, long submitTime, long launchTime, long finishTime,
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken) {
ApplicationReport report = Records.newRecord(ApplicationReport.class);
report.setApplicationId(applicationId);
report.setCurrentApplicationAttemptId(applicationAttemptId);
@ -75,6 +91,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
report.setDiagnostics(diagnostics);
report.setTrackingUrl(url);
report.setStartTime(startTime);
report.setSubmitTime(submitTime);
report.setLaunchTime(launchTime);
report.setFinishTime(finishTime);
report.setFinalApplicationStatus(finalStatus);
@ -102,7 +119,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
ApplicationReport report =
newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url,
startTime, 0, finishTime, finalStatus, appResources,
startTime, 0, 0, finishTime, finalStatus, appResources,
origTrackingUrl, progress, applicationType, amRmToken);
report.setApplicationTags(tags);
report.setUnmanagedApp(unmanagedApplication);
@ -124,10 +141,30 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
float progress, String applicationType, Token amRmToken, Set<String> tags,
boolean unmanagedApplication, Priority priority,
String appNodeLabelExpression, String amNodeLabelExpression) {
return newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url,
startTime, startTime, launchTime, finishTime, finalStatus, appResources,
origTrackingUrl, progress, applicationType, amRmToken, tags,
unmanagedApplication, priority, appNodeLabelExpression,
amNodeLabelExpression);
}
@Private
@Unstable
public static ApplicationReport newInstance(ApplicationId applicationId,
ApplicationAttemptId applicationAttemptId, String user, String queue,
String name, String host, int rpcPort, Token clientToAMToken,
YarnApplicationState state, String diagnostics, String url,
long startTime, long submitTime, long launchTime, long finishTime,
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken, Set<String> tags,
boolean unmanagedApplication, Priority priority,
String appNodeLabelExpression, String amNodeLabelExpression) {
ApplicationReport report =
newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
launchTime, finishTime, finalStatus, appResources,
submitTime, launchTime, finishTime, finalStatus, appResources,
origTrackingUrl, progress, applicationType, amRmToken);
report.setApplicationTags(tags);
report.setUnmanagedApp(unmanagedApplication);
@ -311,6 +348,14 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
@Unstable
public abstract void setStartTime(long startTime);
@Public
@Stable
public abstract long getSubmitTime();
@Private
@Unstable
public abstract void setSubmitTime(long submitTime);
@Private
@Unstable
public abstract void setLaunchTime(long setLaunchTime);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -247,6 +247,7 @@ message ApplicationReportProto {
optional string amNodeLabelExpression = 25;
repeated AppTimeoutsMapProto appTimeouts = 26;
optional int64 launchTime = 27;
optional int64 submitTime = 28;
}
message AppTimeoutsMapProto {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
View File

@ -102,6 +102,8 @@ public void testGetApplicationReport() throws YarnException, IOException {
Assert.assertEquals(report, expectedReports.get(0));
Assert.assertEquals(report.getApplicationId().toString(), expectedReports
.get(0).getApplicationId().toString());
Assert.assertEquals(report.getSubmitTime(), expectedReports.get(0)
.getSubmitTime());
client.stop();
}
@ -332,7 +334,7 @@ private void createAppReports() {
ApplicationReport.newInstance(applicationId,
ApplicationAttemptId.newInstance(applicationId, 1), "user",
"queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0,
YarnApplicationState.RUNNING, "diagnostics", "url", 1, 2, 3, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
null);
List<ApplicationReport> applicationReports =

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
View File

@ -220,6 +220,12 @@ public long getStartTime() {
return p.getStartTime();
}
@Override
public long getSubmitTime() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
return p.getSubmitTime();
}
@Override
public long getLaunchTime() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
@ -430,6 +436,12 @@ public void setFinishTime(long finishTime) {
builder.setFinishTime(finishTime);
}
@Override
public void setSubmitTime(long submitTime) {
maybeInitBuilder();
builder.setSubmitTime(submitTime);
}
@Override
public void setFinalApplicationStatus(FinalApplicationStatus finishState) {
maybeInitBuilder();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
View File

@ -137,9 +137,9 @@ private ApplicationReport convertToApplicationReport(
currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(),
appHistory.getApplicationName(), host, rpcPort, null,
appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(),
trackingUrl, appHistory.getStartTime(), 0, appHistory.getFinishTime(),
appHistory.getFinalApplicationStatus(), null, "", 100,
appHistory.getApplicationType(), null);
trackingUrl, appHistory.getStartTime(), appHistory.getSubmitTime(), 0,
appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(),
null, "", 100, appHistory.getApplicationType(), null);
}
private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId)

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
View File

@ -249,6 +249,7 @@ private static ApplicationReportExt convertToApplicationReport(
String type = null;
boolean unmanagedApplication = false;
long createdTime = 0;
long submittedTime = 0;
long finishedTime = 0;
float progress = 0.0f;
int applicationPriority = 0;
@ -280,10 +281,11 @@ private static ApplicationReportExt convertToApplicationReport(
return new ApplicationReportExt(ApplicationReport.newInstance(
ApplicationId.fromString(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null,
state, diagnosticsInfo, null, createdTime, finishedTime,
finalStatus, null, null, progress, type, null, appTags,
unmanagedApplication, Priority.newInstance(applicationPriority),
appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
state, diagnosticsInfo, null, createdTime, submittedTime, 0,
finishedTime, finalStatus, null, null, progress, type, null,
appTags, unmanagedApplication, Priority.newInstance(
applicationPriority), appNodeLabelExpression,
amNodeLabelExpression), appViewACLs);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
queue =
@ -328,6 +330,8 @@ private static ApplicationReportExt convertToApplicationReport(
entityInfo.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)
.toString();
}
submittedTime = parseLong(entityInfo,
ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO);
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
long vcoreSeconds = parseLong(entityInfo,
@ -438,10 +442,10 @@ private static ApplicationReportExt convertToApplicationReport(
return new ApplicationReportExt(ApplicationReport.newInstance(
ApplicationId.fromString(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
appResources, null, progress, type, null, appTags, unmanagedApplication,
Priority.newInstance(applicationPriority), appNodeLabelExpression,
amNodeLabelExpression), appViewACLs);
diagnosticsInfo, null, createdTime, submittedTime, 0, finishedTime,
finalStatus, appResources, null, progress, type, null, appTags,
unmanagedApplication, Priority.newInstance(applicationPriority),
appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
}
private static long parseLong(Map<String, Object> entityInfo,

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
View File

@ -214,6 +214,7 @@ public ApplicationReport run() throws Exception {
}
Assert.assertEquals(Integer.MAX_VALUE + 2L
+ app.getApplicationId().getId(), app.getStartTime());
Assert.assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime());
Assert.assertEquals(Integer.MAX_VALUE + 3L
+ +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
View File

@ -401,6 +401,7 @@ public void testSingleApp() throws Exception {
assertEquals("test app", app.get("name"));
assertEquals(round == 0 ? "test diagnostics info" : "",
app.get("diagnosticsInfo"));
assertEquals(Integer.MAX_VALUE + 1L, app.get("submittedTime"));
assertEquals("test queue", app.get("queue"));
assertEquals("user1", app.get("user"));
assertEquals("test app type", app.get("type"));

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
View File

@ -53,7 +53,7 @@ public class AppInfo {
protected String originalTrackingUrl;
protected String trackingUrl;
protected FinalApplicationStatus finalAppStatus;
protected long submittedTime;
private long submittedTime;
protected long startedTime;
private long launchTime;
protected long finishedTime;
@ -87,7 +87,7 @@ public AppInfo(ApplicationReport app) {
diagnosticsInfo = app.getDiagnostics();
trackingUrl = app.getTrackingUrl();
originalTrackingUrl = app.getOriginalTrackingUrl();
submittedTime = app.getStartTime();
submittedTime = app.getSubmitTime();
startedTime = app.getStartTime();
launchTime = app.getLaunchTime();
finishedTime = app.getFinishTime();

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
View File

@ -263,6 +263,7 @@ public static RMApp newApplication(int i) {
final String name = newAppName();
final String queue = newQueue();
final long start = 123456 + i * 1000;
final long submit = start + i * 50;
final long launch = start + i * 100;
final long finish = 234567 + i * 1000;
final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE;
@ -299,6 +300,11 @@ public long getStartTime() {
return start;
}
@Override
public long getSubmitTime() {
return submit;
}
@Override
public long getLaunchTime() {
return launch;
@ -353,9 +359,9 @@ public ApplicationReport createAndGetApplicationReport(
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),
getTrackingUrl(), getLaunchTime(), getStartTime(), getFinishTime(),
getFinalApplicationStatus(), usageReport , null, getProgress(),
type, null);
getTrackingUrl(), getStartTime(), getSubmitTime(), getLaunchTime(),
getFinishTime(), getFinalApplicationStatus(), usageReport, null,
getProgress(), type, null);
return report;
}