From 0c14b6f0d80d847be7ef3eda1614eadbf835181c Mon Sep 17 00:00:00 2001 From: Haibo Chen Date: Tue, 17 Apr 2018 05:55:46 -0700 Subject: [PATCH] YARN-7088. Add application launch time to Resource Manager REST API. (Kanwaljeet Sachdev via Haibo Chen) --- .../apache/hadoop/mapred/NotRunningJob.java | 2 +- .../mapred/TestClientServiceDelegate.java | 4 +- .../apache/hadoop/mapred/TestYARNRunner.java | 3 +- .../yarn/api/records/ApplicationReport.java | 45 +++++++++++++++++-- .../src/main/proto/yarn_protos.proto | 1 + .../yarn/client/ProtocolHATestBase.java | 2 +- .../yarn/client/api/impl/TestAHSClient.java | 8 ++-- .../yarn/client/api/impl/TestYarnClient.java | 8 ++-- .../hadoop/yarn/client/cli/TestYarnCLI.java | 42 ++++++++--------- .../impl/pb/ApplicationReportPBImpl.java | 11 +++++ .../hadoop/yarn/api/TestApplicatonReport.java | 2 +- .../ApplicationHistoryManagerImpl.java | 2 +- .../yarn/server/utils/BuilderUtils.java | 3 +- .../hadoop/yarn/server/webapp/AppBlock.java | 7 ++- .../hadoop/yarn/server/webapp/AppsBlock.java | 5 ++- .../yarn/server/webapp/WebPageUtils.java | 2 +- .../yarn/server/webapp/dao/AppInfo.java | 6 +++ .../amlauncher/AMLauncher.java | 2 +- .../records/ApplicationStateData.java | 24 ++++++++-- .../impl/pb/ApplicationStateDataPBImpl.java | 13 ++++++ .../server/resourcemanager/rmapp/RMApp.java | 10 ++++- .../resourcemanager/rmapp/RMAppEvent.java | 6 +++ .../resourcemanager/rmapp/RMAppEventType.java | 1 + .../resourcemanager/rmapp/RMAppImpl.java | 41 +++++++++++++++-- .../rmapp/attempt/RMAppAttemptEvent.java | 7 +++ .../rmapp/attempt/RMAppAttemptImpl.java | 4 +- .../webapp/FairSchedulerAppsBlock.java | 2 + .../resourcemanager/webapp/RMAppsBlock.java | 2 + .../resourcemanager/webapp/dao/AppInfo.java | 6 +++ ...yarn_server_resourcemanager_recovery.proto | 1 + .../applicationsmanager/MockAsm.java | 15 ++++++- .../recovery/RMStateStoreTestBase.java | 4 +- .../recovery/TestFSRMStateStore.java | 2 +- .../recovery/TestZKRMStateStore.java | 2 +- .../resourcemanager/rmapp/MockRMApp.java | 6 +++ .../rmapp/TestRMAppTransitions.java | 2 +- .../resourcemanager/webapp/TestRMWebApp.java | 2 +- .../webapp/TestRMWebServicesApps.java | 14 +++--- 38 files changed, 249 insertions(+), 70 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index 90635a68854..bfdc6db8abd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -89,7 +89,7 @@ private ApplicationReport getUnknownApplicationReport() { // used for a non running job return ApplicationReport.newInstance(unknownAppId, unknownAttemptId, "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", - "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, + "N/A", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java index b85f18db4eb..275456df837 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java @@ -515,7 +515,7 @@ private ApplicationReport getFinishedApplicationReport() { appId, 0); return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, - "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } @@ -525,7 +525,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) { appId, 0); return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics", - "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, + "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 932e808bf78..babf22b6399 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -272,7 +272,8 @@ public ClientServiceDelegate answer(InvocationOnMock invocation) .thenReturn( ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp", "tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp", - 0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f, + 0L, 0L, 0L, + FinalApplicationStatus.SUCCEEDED, null, null, 0f, "tmp", null)); yarnRunner.killJob(jobId); verify(clientDelegate).killJob(jobId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 962bbba407f..711424b5431 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -58,7 +58,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken) { ApplicationReport report = Records.newRecord(ApplicationReport.class); @@ -74,6 +75,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setLaunchTime(launchTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); @@ -84,13 +86,40 @@ public static ApplicationReport newInstance(ApplicationId applicationId, return report; } + + @Private + @Unstable + public static ApplicationReport newInstance(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, String user, String queue, + String name, String host, int rpcPort, Token clientToAMToken, + YarnApplicationState state, String diagnostics, String url, + long startTime, long finishTime, + FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress, String applicationType, Token amRmToken, + Set tags, boolean unmanagedApplication, Priority priority, + String appNodeLabelExpression, String amNodeLabelExpression) { + ApplicationReport report = + newInstance(applicationId, applicationAttemptId, user, queue, name, + host, rpcPort, clientToAMToken, state, diagnostics, url, + startTime, 0, finishTime, finalStatus, appResources, + origTrackingUrl, progress, applicationType, amRmToken); + report.setApplicationTags(tags); + report.setUnmanagedApp(unmanagedApplication); + report.setPriority(priority); + report.setAppNodeLabelExpression(appNodeLabelExpression); + report.setAmNodeLabelExpression(amNodeLabelExpression); + return report; + } + @Private @Unstable public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken, Set tags, boolean unmanagedApplication, Priority priority, @@ -98,8 +127,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationReport report = newInstance(applicationId, applicationAttemptId, user, queue, name, host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, - finishTime, finalStatus, appResources, origTrackingUrl, progress, - applicationType, amRmToken); + launchTime, finishTime, finalStatus, appResources, + origTrackingUrl, progress, applicationType, amRmToken); report.setApplicationTags(tags); report.setUnmanagedApp(unmanagedApplication); report.setPriority(priority); @@ -282,6 +311,14 @@ public static ApplicationReport newInstance(ApplicationId applicationId, @Unstable public abstract void setStartTime(long startTime); + @Private + @Unstable + public abstract void setLaunchTime(long setLaunchTime); + + @Public + @Unstable + public abstract long getLaunchTime(); + /** * Get the finish time of the application. * @return finish time of the application diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 5e200dc52de..d6138e865ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -281,6 +281,7 @@ message ApplicationReportProto { optional string appNodeLabelExpression = 24; optional string amNodeLabelExpression = 25; repeated AppTimeoutsMapProto appTimeouts = 26; + optional int64 launchTime = 27; } message AppTimeoutsMapProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index 7937b1594bb..474ae788235 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -626,7 +626,7 @@ public ApplicationReport createFakeAppReport() { ApplicationReport.newInstance(appId, attemptId, "fakeUser", "fakeQueue", "fakeApplicationName", "localhost", 0, null, YarnApplicationState.FINISHED, "fake an application report", "", - 1000L, 1200L, FinalApplicationStatus.FAILED, null, "", 50f, + 1000L, 1000L, 1200L, FinalApplicationStatus.FAILED, null, "", 50f, "fakeApplicationType", null); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java index c3e3c414771..f0e3ca2a4cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java @@ -332,7 +332,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = @@ -389,7 +389,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null); applicationReports.add(newApplicationReport2); @@ -399,7 +399,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); @@ -409,7 +409,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index f6e305fab7a..b84b49c734c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -737,7 +737,7 @@ private List createAppReports() { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -812,7 +812,7 @@ private List createAppReports() { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null); applicationReports.add(newApplicationReport2); @@ -821,7 +821,7 @@ private List createAppReports() { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); @@ -832,7 +832,7 @@ private List createAppReports() { applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 1f6488db9e7..82a20ebec58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -136,7 +136,7 @@ public void testGetApplicationReport() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", null, null, false, Priority.newInstance(0), "high-mem", "high-mem"); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); @@ -383,7 +383,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, Sets.newHashSet("tag1", "tag3"), false, Priority.UNDEFINED, "", ""); List applicationReports = @@ -394,7 +394,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null, Sets.newHashSet("tag2", "tag3"), false, Priority.UNDEFINED, "", ""); @@ -404,7 +404,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null, Sets.newHashSet("tag1", "tag4"), false, Priority.UNDEFINED, "", ""); @@ -414,7 +414,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null, Sets.newHashSet("tag1"), false, Priority.UNDEFINED, "", ""); @@ -424,7 +424,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId5, ApplicationAttemptId.newInstance(applicationId5, 5), "user5", "queue5", "appname5", "host5", 128, null, - YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, + YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, 5, FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", null, Sets.newHashSet("tag2", "tag4"), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport5); @@ -433,7 +433,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport6 = ApplicationReport.newInstance( applicationId6, ApplicationAttemptId.newInstance(applicationId6, 6), "user6", "queue6", "appname6", "host6", 129, null, - YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, + YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, 6, FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG", null, new HashSet(), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport6); @@ -1007,7 +1007,7 @@ public void testKillApplication() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); @@ -1020,7 +1020,7 @@ public void testKillApplication() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -1059,12 +1059,12 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport1 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.34344f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport1); @@ -1084,12 +1084,12 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport3); @@ -1127,7 +1127,7 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport5); @@ -1154,12 +1154,12 @@ public void testKillApplicationsOfDifferentEndStates() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport6 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport5); @@ -1182,7 +1182,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); @@ -1197,7 +1197,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -1232,7 +1232,7 @@ public void testMoveApplicationAcrossQueuesWithNewCommand() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(newApplicationReport2); @@ -1247,7 +1247,7 @@ public void testMoveApplicationAcrossQueuesWithNewCommand() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(newApplicationReport); @@ -2015,7 +2015,7 @@ public void testUpdateApplicationPriority() throws Exception { ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 036fa907fe6..ddd9accd898 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -220,6 +220,17 @@ public long getStartTime() { return p.getStartTime(); } + @Override + public long getLaunchTime() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getLaunchTime(); + } + + @Override + public void setLaunchTime(long launchTime) { + maybeInitBuilder(); + builder.setLaunchTime(launchTime); + } @Override public long getFinishTime() { ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index 46fc4d58d92..ea39a4ccdba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -58,7 +58,7 @@ protected static ApplicationReport createApplicationReport( ApplicationReport appReport = ApplicationReport.newInstance(appId, appAttemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, - "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, false, Priority.newInstance(0),"",""); return appReport; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index b8931d81abc..3734e39d91d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -137,7 +137,7 @@ private ApplicationReport convertToApplicationReport( currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(), appHistory.getApplicationName(), host, rpcPort, null, appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), - trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), + trackingUrl, appHistory.getStartTime(), 0, appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), null, "", 100, appHistory.getApplicationType(), null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 4ba1bdfd1ec..0de834c43fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -391,7 +391,7 @@ public static ApplicationReport newApplicationReport( ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, - String url, long startTime, long finishTime, + String url, long startTime, long launchTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String appType, Token amRmToken, Set tags, @@ -410,6 +410,7 @@ public static ApplicationReport newApplicationReport( report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setLaunchTime(launchTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 8600b7252b3..9dd8aa1d249 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -242,10 +242,9 @@ private void generateOverviewTable(AppInfo app, String schedulerPath, .__("FinalStatus Reported by AM:", clairfyAppFinalStatus(app.getFinalAppStatus())) .__("Started:", Times.format(app.getStartedTime())) - .__( - "Elapsed:", - StringUtils.formatTime(Times.elapsed(app.getStartedTime(), - app.getFinishedTime()))) + .__("Launched:", Times.format(app.getLaunchTime())) + .__("Finished:", Times.format(app.getFinishedTime())) + .__("Elapsed:", StringUtils.formatTime(app.getElapsedTime())) .__( "Tracking URL:", app.getTrackingUrl() == null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index b2dd4de8185..cb15449d357 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -150,7 +150,9 @@ protected void renderData(Block html) { html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") .th(".name", "Name").th(".type", "Application Type") .th(".queue", "Queue").th(".priority", "Application Priority") - .th(".starttime", "StartTime").th(".finishtime", "FinishTime") + .th(".starttime", "StartTime") + .th(".launchtime", "LaunchTime") + .th(".finishtime", "FinishTime") .th(".state", "State").th(".finalstatus", "FinalStatus") .th(".progress", "Progress").th(".ui", "Tracking UI").__().__().tbody(); @@ -188,6 +190,7 @@ protected void renderData(Block html) { .getQueue()))).append("\",\"").append(String .valueOf(app.getPriority())) .append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getLaunchTime()) .append("\",\"").append(app.getFinishedTime()) .append("\",\"") .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java index 29f1e634563..b2f65a84504 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java @@ -51,7 +51,7 @@ private static String getAppsTableColumnDefs( sb.append("[\n") .append("{'sType':'natural', 'aTargets': [0]") .append(", 'mRender': parseHadoopID }") - .append("\n, {'sType':'numeric', 'aTargets': [6, 7]") + .append("\n, {'sType':'numeric', 'aTargets': [6, 7, 8]") .append(", 'mRender': renderHadoopDate }") .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':"); if (isFairSchedulerPage) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index ac2f8da6aa2..b4687a42607 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -55,6 +55,7 @@ public class AppInfo { protected FinalApplicationStatus finalAppStatus; protected long submittedTime; protected long startedTime; + private long launchTime; protected long finishedTime; protected long elapsedTime; protected String applicationTags; @@ -88,6 +89,7 @@ public AppInfo(ApplicationReport app) { originalTrackingUrl = app.getOriginalTrackingUrl(); submittedTime = app.getStartTime(); startedTime = app.getStartTime(); + launchTime = app.getLaunchTime(); finishedTime = app.getFinishTime(); elapsedTime = Times.elapsed(startedTime, finishedTime); finalAppStatus = app.getFinalApplicationStatus(); @@ -198,6 +200,10 @@ public long getSubmittedTime() { return submittedTime; } + public long getLaunchTime() { + return launchTime; + } + public long getStartedTime() { return startedTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 7051f8cca26..58968089f29 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -303,7 +303,7 @@ public void run() { LOG.info("Launching master" + application.getAppAttemptId()); launch(); handler.handle(new RMAppAttemptEvent(application.getAppAttemptId(), - RMAppAttemptEventType.LAUNCHED)); + RMAppAttemptEventType.LAUNCHED, System.currentTimeMillis())); } catch(Exception ie) { String message = "Error launching " + application.getAppAttemptId() + ". Got exception: " + StringUtils.stringifyException(ie); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java index 79a5de25846..2b0bd2b5bad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java @@ -47,7 +47,8 @@ public abstract class ApplicationStateData { public static ApplicationStateData newInstance(long submitTime, long startTime, String user, ApplicationSubmissionContext submissionContext, RMAppState state, - String diagnostics, long finishTime, CallerContext callerContext) { + String diagnostics, long launchTime, long finishTime, + CallerContext callerContext) { ApplicationStateData appState = Records.newRecord(ApplicationStateData.class); appState.setSubmitTime(submitTime); appState.setStartTime(startTime); @@ -55,6 +56,7 @@ public static ApplicationStateData newInstance(long submitTime, appState.setApplicationSubmissionContext(submissionContext); appState.setState(state); appState.setDiagnostics(diagnostics); + appState.setLaunchTime(launchTime); appState.setFinishTime(finishTime); appState.setCallerContext(callerContext); return appState; @@ -63,7 +65,8 @@ public static ApplicationStateData newInstance(long submitTime, public static ApplicationStateData newInstance(long submitTime, long startTime, String user, ApplicationSubmissionContext submissionContext, RMAppState state, - String diagnostics, long finishTime, CallerContext callerContext, + String diagnostics, long launchTime, long finishTime, + CallerContext callerContext, Map applicationTimeouts) { ApplicationStateData appState = Records.newRecord(ApplicationStateData.class); @@ -73,6 +76,7 @@ public static ApplicationStateData newInstance(long submitTime, appState.setApplicationSubmissionContext(submissionContext); appState.setState(state); appState.setDiagnostics(diagnostics); + appState.setLaunchTime(launchTime); appState.setFinishTime(finishTime); appState.setCallerContext(callerContext); appState.setApplicationTimeouts(applicationTimeouts); @@ -82,7 +86,7 @@ public static ApplicationStateData newInstance(long submitTime, public static ApplicationStateData newInstance(long submitTime, long startTime, ApplicationSubmissionContext context, String user, CallerContext callerContext) { - return newInstance(submitTime, startTime, user, context, null, "", 0, + return newInstance(submitTime, startTime, user, context, null, "", 0, 0, callerContext); } @@ -136,6 +140,20 @@ public int getFirstAttemptId() { @Unstable public abstract void setStartTime(long startTime); + + + /** + * Get the launch time of the application. + * @return launch time of the application + */ + @Public + @Stable + public abstract long getLaunchTime(); + + @Private + @Unstable + public abstract void setLaunchTime(long launchTime); + /** * The application submitter */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java index d037e68a12a..f5cd107e173 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java @@ -120,6 +120,19 @@ public void setStartTime(long startTime) { builder.setStartTime(startTime); } + + @Override + public long getLaunchTime() { + ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getLaunchTime(); + } + + @Override + public void setLaunchTime(long launchTime) { + maybeInitBuilder(); + builder.setLaunchTime(launchTime); + } + @Override public String getUser() { ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index e286834cd4b..99cce87102d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -180,7 +180,15 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, * @return the submit time of the application. */ long getSubmitTime(); - + + /** + * The launch time of the application. + * Since getStartTime() returns what is essentially submit time, + * this new field is to prevent potential backwards compatibility issues. + * @return the launch time of the application. + */ + long getLaunchTime(); + /** * The tracking url for the application master. * @return the tracking url for the application master. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java index 5c46945754e..3212d6c504c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java @@ -37,6 +37,12 @@ public RMAppEvent(ApplicationId appId, RMAppEventType type, this.diagnosticMsg = diagnostic; } + public RMAppEvent(ApplicationId appId, RMAppEventType type, long timeStamp) { + super(type, timeStamp); + this.appId = appId; + this.diagnosticMsg = ""; + } + public ApplicationId getApplicationId() { return this.appId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java index 514efd44dd6..4b55d389540 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java @@ -37,6 +37,7 @@ public enum RMAppEventType { ATTEMPT_FAILED, ATTEMPT_KILLED, NODE_UPDATE, + ATTEMPT_LAUNCHED, // Source: Container and ResourceTracker APP_RUNNING_ON_NODE, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 4234e582573..daf14c46632 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -164,6 +164,7 @@ public class RMAppImpl implements RMApp, Recoverable { // Mutable fields private long startTime; + private long launchTime = 0; private long finishTime = 0; private long storedFinishTime = 0; private int firstAttemptIdInStateStore = 1; @@ -290,6 +291,10 @@ RMAppEventType.KILL, new KillAttemptTransition()) .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, RMAppEventType.APP_RUNNING_ON_NODE, new AppRunningOnNodeTransition()) + // Handle AppAttemptLaunch to upate the launchTime and publish to ATS + .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, + RMAppEventType.ATTEMPT_LAUNCHED, + new AttemptLaunchedTransition()) // Transitions from RUNNING state .addTransition(RMAppState.RUNNING, RMAppState.RUNNING, @@ -784,9 +789,9 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, this.applicationId, currentApplicationAttemptId, this.user, this.queue, this.name, host, rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl, this.startTime, - this.finishTime, finishState, appUsageReport, origTrackingUrl, - progress, this.applicationType, amrmToken, applicationTags, - this.getApplicationPriority()); + this.launchTime, this.finishTime, finishState, appUsageReport, + origTrackingUrl, progress, this.applicationType, amrmToken, + applicationTags, this.getApplicationPriority()); report.setLogAggregationStatus(logAggregationStatus); report.setUnmanagedApp(submissionContext.getUnmanagedAM()); report.setAppNodeLabelExpression(getAppNodeLabelExpression()); @@ -839,6 +844,17 @@ public long getStartTime() { } } + @Override + public long getLaunchTime() { + this.readLock.lock(); + + try { + return this.launchTime; + } finally { + this.readLock.unlock(); + } + } + @Override public long getSubmitTime() { return this.submitTime; @@ -936,6 +952,7 @@ public void recover(RMState state) { .getDiagnostics()); this.storedFinishTime = appState.getFinishTime(); this.startTime = appState.getStartTime(); + this.launchTime = appState.getLaunchTime(); this.callerContext = appState.getCallerContext(); this.applicationTimeouts = appState.getApplicationTimeouts(); // If interval > 0, some attempts might have been deleted. @@ -1038,6 +1055,21 @@ public void transition(RMAppImpl app, RMAppEvent event) { }; } + private static final class AttemptLaunchedTransition + extends RMAppTransition { + @Override + public void transition(RMAppImpl app, RMAppEvent event) { + + if(app.launchTime == 0) { + LOG.info("update the launch time for applicationId: "+ + app.getApplicationId()+", attemptId: "+ + app.getCurrentAppAttempt().getAppAttemptId()+ + "launchTime: "+event.getTimestamp()); + app.launchTime = event.getTimestamp(); + } + } + } + private static final class AppRunningOnNodeTransition extends RMAppTransition { public void transition(RMAppImpl app, RMAppEvent event) { RMAppRunningOnNodeEvent nodeAddedEvent = (RMAppRunningOnNodeEvent) event; @@ -1297,7 +1329,8 @@ private void rememberTargetTransitionsAndStoreState(RMAppEvent event, ApplicationStateData appState = ApplicationStateData.newInstance(this.submitTime, this.startTime, this.user, this.submissionContext, - stateToBeStored, diags, this.storedFinishTime, this.callerContext); + stateToBeStored, diags, this.launchTime, this.storedFinishTime, + this.callerContext); appState.setApplicationTimeouts(this.applicationTimeouts); this.rmContext.getStateStore().updateApplicationState(appState); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java index 6df6b19f978..2ddc981067a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java @@ -38,6 +38,13 @@ public RMAppAttemptEvent(ApplicationAttemptId appAttemptId, this.diagnosticMsg = diagnostics; } + public RMAppAttemptEvent(ApplicationAttemptId appAttemptId, + RMAppAttemptEventType type, long timeStamp) { + super(type, timeStamp); + this.appAttemptId = appAttemptId; + this.diagnosticMsg = ""; + } + public ApplicationAttemptId getApplicationAttemptId() { return this.appAttemptId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 1b1e2c447c6..32f275f8079 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1557,7 +1557,9 @@ public void transition(RMAppAttemptImpl appAttempt, appAttempt.launchAMStartTime; ClusterMetrics.getMetrics().addAMLaunchDelay(delay); } - + appAttempt.eventHandler.handle( + new RMAppEvent(appAttempt.getAppAttemptId().getApplicationId(), + RMAppEventType.ATTEMPT_LAUNCHED, event.getTimestamp())); appAttempt .updateAMLaunchDiagnostics(AMState.LAUNCHED.getDiagnosticMessage()); // Register with AMLivelinessMonitor diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java index ac88f861feb..2bbbcc54a30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -89,6 +89,7 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, th(".queue", "Queue"). th(".fairshare", "Fair Share"). th(".starttime", "StartTime"). + th(".launchTime", "LaunchTime"). th(".finishtime", "FinishTime"). th(".state", "State"). th(".finalstatus", "FinalStatus"). @@ -135,6 +136,7 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, appInfo.getQueue()))).append("\",\"") .append(fairShare).append("\",\"") .append(appInfo.getStartTime()).append("\",\"") + .append(appInfo.getLaunchTime()).append("\",\"") .append(appInfo.getFinishTime()).append("\",\"") .append(appInfo.getState()).append("\",\"") .append(appInfo.getFinalStatus()).append("\",\"") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index d0dccab6e5c..a525cffdd0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -62,6 +62,7 @@ protected void renderData(Block html) { .th(".name", "Name").th(".type", "Application Type") .th(".queue", "Queue").th(".priority", "Application Priority") .th(".starttime", "StartTime") + .th("launchtime", "LaunchTime") .th(".finishtime", "FinishTime").th(".state", "State") .th(".finalstatus", "FinalStatus") .th(".runningcontainer", "Running Containers") @@ -134,6 +135,7 @@ protected void renderData(Block html) { .getQueue()))).append("\",\"").append(String .valueOf(app.getPriority())) .append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getLaunchTime()) .append("\",\"").append(app.getFinishedTime()) .append("\",\"") .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 631c9084193..d47f13d9586 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -87,6 +87,7 @@ public class AppInfo { // these are only allowed if acls allow protected long startedTime; + private long launchTime; protected long finishedTime; protected long elapsedTime; protected String amContainerLogs; @@ -181,6 +182,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { this.startedTime = app.getStartTime(); + this.launchTime = app.getLaunchTime(); this.finishedTime = app.getFinishTime(); this.elapsedTime = Times.elapsed(app.getStartTime(), app.getFinishTime()); @@ -394,6 +396,10 @@ public long getStartTime() { return this.startedTime; } + public long getLaunchTime() { + return this.launchTime; + } + public long getFinishTime() { return this.finishedTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto index 39a56a811a3..35c77ab78b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -70,6 +70,7 @@ message ApplicationStateDataProto { optional int64 finish_time = 7; optional hadoop.common.RPCCallerContextProto caller_context = 8; repeated ApplicationTimeoutMapProto application_timeouts = 9; + optional int64 launch_time = 10; } message ApplicationAttemptStateDataProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 72de27cf95a..6c6c4b4e803 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -92,7 +92,12 @@ public long getStartTime() { public long getSubmitTime() { throw new UnsupportedOperationException("Not supported yet."); } - + + @Override + public long getLaunchTime() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public long getFinishTime() { throw new UnsupportedOperationException("Not supported yet."); @@ -272,6 +277,7 @@ public static RMApp newApplication(int i) { final String name = newAppName(); final String queue = newQueue(); final long start = 123456 + i * 1000; + final long launch = start + i * 100; final long finish = 234567 + i * 1000; final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE; YarnApplicationState[] allStates = YarnApplicationState.values(); @@ -307,6 +313,11 @@ public long getStartTime() { return start; } + @Override + public long getLaunchTime() { + return launch; + } + @Override public long getFinishTime() { return finish; @@ -357,7 +368,7 @@ public ApplicationReport createAndGetApplicationReport( ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), - getTrackingUrl(), getStartTime(), getFinishTime(), + getTrackingUrl(), getLaunchTime(), getStartTime(), getFinishTime(), getFinalApplicationStatus(), usageReport , null, getProgress(), type, null); return report; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index dbb214897a6..957d4ce8224 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -358,7 +358,7 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), appState.getApplicationSubmissionContext(), RMAppState.FINISHED, - "appDiagnostics", 1234, appState.getCallerContext()); + "appDiagnostics", 123, 1234, appState.getCallerContext()); appState2.attempts.putAll(appState.attempts); store.updateApplicationState(appState2); @@ -384,7 +384,7 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, ApplicationStateData dummyApp = ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), dummyContext, - RMAppState.FINISHED, "appDiagnostics", 1234, null); + RMAppState.FINISHED, "appDiagnostics", 123, 1234, null); store.updateApplicationState(dummyApp); ApplicationAttemptId dummyAttemptId = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index 0738730aabc..fe4a7019baf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -407,7 +407,7 @@ public void run() { store.storeApplicationStateInternal( ApplicationId.newInstance(100L, 1), ApplicationStateData.newInstance(111, 111, "user", null, - RMAppState.ACCEPTED, "diagnostics", 333, null)); + RMAppState.ACCEPTED, "diagnostics", 222, 333, null)); } catch (Exception e) { assertionFailedInThread.set(true); e.printStackTrace(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 0a1b15275cf..d8718e0b88b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -790,7 +790,7 @@ private static ApplicationStateData createAppState( long finishTime, boolean isFinished) { return ApplicationStateData.newInstance(submitTime, startTime, "test", ctxt, isFinished ? RMAppState.FINISHED : null, isFinished ? - "appDiagnostics" : "", isFinished ? finishTime : 0, null); + "appDiagnostics" : "", 0, isFinished ? finishTime : 0, null); } private static ApplicationAttemptStateData createFinishedAttempt( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 664fae2d061..ad29d274a4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -57,6 +57,7 @@ public class MockRMApp implements RMApp { String name = MockApps.newAppName(); String queue = MockApps.newQueue(); long start = System.currentTimeMillis() - (int) (Math.random() * DT); + private long launch = start; long submit = start - (int) (Math.random() * DT); long finish = 0; RMAppState state = RMAppState.NEW; @@ -194,6 +195,11 @@ public long getSubmitTime() { return submit; } + @Override + public long getLaunchTime() { + return launch; + } + public void setStartTime(long time) { this.start = time; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 920bd4fd2b9..70887e0f4bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -1239,7 +1239,7 @@ public void createRMStateForApplications( ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(), app.getUser(), app.getApplicationSubmissionContext(), rmAppState, - null, app.getFinishTime(), null); + null, app.getLaunchTime(), app.getFinishTime(), null); applicationState.put(app.getApplicationId(), appState); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index b1256083ef8..93377be1ea9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -257,7 +257,7 @@ public static ClientRMService mockClientRMService(RMContext rmContext) { app.getName(), (String) null, 0, (Token) null, app.createApplicationState(), app.getDiagnostics().toString(), (String) null, - app.getStartTime(), app.getFinishTime(), + app.getStartTime(), app.getLaunchTime(), app.getFinishTime(), app.getFinalApplicationStatus(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(), app.getProgress(), app.getApplicationType(), (Token) null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index a381ed4020c..6c6f400a623 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1553,6 +1553,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) WebServicesTestUtils.getXmlString(element, "diagnostics"), WebServicesTestUtils.getXmlLong(element, "clusterId"), WebServicesTestUtils.getXmlLong(element, "startedTime"), + WebServicesTestUtils.getXmlLong(element, "launchTime"), WebServicesTestUtils.getXmlLong(element, "finishedTime"), WebServicesTestUtils.getXmlLong(element, "elapsedTime"), WebServicesTestUtils.getXmlString(element, "amHostHttpAddress"), @@ -1603,7 +1604,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) throws JSONException, Exception { - int expectedNumberOfElements = 39 + (hasResourceReqs ? 2 : 0); + int expectedNumberOfElements = 40 + (hasResourceReqs ? 2 : 0); String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() @@ -1629,8 +1630,10 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) info.getString("state"), info.getString("finalStatus"), (float) info.getDouble("progress"), info.getString("trackingUI"), info.getString("diagnostics"), info.getLong("clusterId"), - info.getLong("startedTime"), info.getLong("finishedTime"), - info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), + info.getLong("startedTime"), info.getLong("launchTime"), + info.getLong("finishedTime"), + info.getLong("elapsedTime"), + info.getString("amHostHttpAddress"), info.getString("amContainerLogs"), info.getInt("allocatedMB"), info.getInt("allocatedVCores"), info.getInt("runningContainers"), (float) info.getDouble("queueUsagePercentage"), @@ -1653,8 +1656,9 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) public void verifyAppInfoGeneric(RMApp app, String id, String user, String name, String applicationType, String queue, int prioirty, String state, String finalStatus, float progress, String trackingUI, - String diagnostics, long clusterId, long startedTime, long finishedTime, - long elapsedTime, String amHostHttpAddress, String amContainerLogs, + String diagnostics, long clusterId, long startedTime, + long launchTime, long finishedTime, long elapsedTime, + String amHostHttpAddress, String amContainerLogs, int allocatedMB, int allocatedVCores, int numContainers, float queueUsagePerc, float clusterUsagePerc, int preemptedResourceMB, int preemptedResourceVCores,