From c02c19121efe638df84c44525e72dfa867543e8c Mon Sep 17 00:00:00 2001 From: Haibo Chen Date: Tue, 17 Apr 2018 11:15:20 -0700 Subject: [PATCH] YARN-7088. Add application launch time to Resource Manager REST API. (Kanwaljeet Sachdev via Haibo Chen) (cherry picked from commit bb92bfb4ef96baa234966b60e464d1773fbf3f22) (cherry picked from commit 3a764250262e745a6762dd6551b0d0a6af39e891) (cherry picked from commit 75df8bed6eed86ad7c34460a1167c35d48b559d7) --- .../apache/hadoop/mapred/NotRunningJob.java | 2 +- .../mapred/TestClientServiceDelegate.java | 4 +- .../apache/hadoop/mapred/TestYARNRunner.java | 3 +- .../yarn/api/records/ApplicationReport.java | 45 +++++++++++++++++-- .../src/main/proto/yarn_protos.proto | 1 + .../yarn/client/ProtocolHATestBase.java | 2 +- .../yarn/client/api/impl/TestAHSClient.java | 8 ++-- .../yarn/client/api/impl/TestYarnClient.java | 8 ++-- .../hadoop/yarn/client/cli/TestYarnCLI.java | 42 ++++++++--------- .../impl/pb/ApplicationReportPBImpl.java | 11 +++++ .../hadoop/yarn/api/TestApplicatonReport.java | 2 +- .../ApplicationHistoryManagerImpl.java | 2 +- .../yarn/server/utils/BuilderUtils.java | 3 +- .../hadoop/yarn/server/webapp/AppBlock.java | 7 ++- .../hadoop/yarn/server/webapp/AppsBlock.java | 5 ++- .../yarn/server/webapp/WebPageUtils.java | 2 +- .../yarn/server/webapp/dao/AppInfo.java | 6 +++ .../amlauncher/AMLauncher.java | 2 +- .../records/ApplicationStateData.java | 24 ++++++++-- .../impl/pb/ApplicationStateDataPBImpl.java | 13 ++++++ .../server/resourcemanager/rmapp/RMApp.java | 10 ++++- .../resourcemanager/rmapp/RMAppEvent.java | 6 +++ .../resourcemanager/rmapp/RMAppEventType.java | 1 + .../resourcemanager/rmapp/RMAppImpl.java | 41 +++++++++++++++-- .../rmapp/attempt/RMAppAttemptEvent.java | 7 +++ .../rmapp/attempt/RMAppAttemptImpl.java | 4 +- .../webapp/FairSchedulerAppsBlock.java | 2 + .../resourcemanager/webapp/RMAppsBlock.java | 2 + .../resourcemanager/webapp/dao/AppInfo.java | 6 +++ ...yarn_server_resourcemanager_recovery.proto | 1 + .../applicationsmanager/MockAsm.java | 15 ++++++- .../recovery/RMStateStoreTestBase.java | 4 +- .../recovery/TestFSRMStateStore.java | 2 +- .../recovery/TestZKRMStateStore.java | 2 +- .../resourcemanager/rmapp/MockRMApp.java | 6 +++ .../rmapp/TestRMAppTransitions.java | 2 +- .../resourcemanager/webapp/TestRMWebApp.java | 2 +- .../webapp/TestRMWebServicesApps.java | 14 +++--- 38 files changed, 249 insertions(+), 70 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index 90635a68854..bfdc6db8abd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -89,7 +89,7 @@ private ApplicationReport getUnknownApplicationReport() { // used for a non running job return ApplicationReport.newInstance(unknownAppId, unknownAttemptId, "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", - "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, + "N/A", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java index b85f18db4eb..275456df837 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java @@ -515,7 +515,7 @@ private ApplicationReport getFinishedApplicationReport() { appId, 0); return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, - "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } @@ -525,7 +525,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) { appId, 0); return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics", - "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, + "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index b4df7f1a4bb..c79b08e52e4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -206,7 +206,8 @@ public ClientServiceDelegate answer(InvocationOnMock invocation) .thenReturn( ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp", "tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp", - 0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f, + 0L, 0L, 0L, + FinalApplicationStatus.SUCCEEDED, null, null, 0f, "tmp", null)); yarnRunner.killJob(jobId); verify(clientDelegate).killJob(jobId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 9e9ec3cd2c9..d572d62867a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -58,7 +58,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken) { ApplicationReport report = Records.newRecord(ApplicationReport.class); @@ -74,6 +75,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setLaunchTime(launchTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); @@ -84,13 +86,40 @@ public static ApplicationReport newInstance(ApplicationId applicationId, return report; } + + @Private + @Unstable + public static ApplicationReport newInstance(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, String user, String queue, + String name, String host, int rpcPort, Token clientToAMToken, + YarnApplicationState state, String diagnostics, String url, + long startTime, long finishTime, + FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress, String applicationType, Token amRmToken, + Set tags, boolean unmanagedApplication, Priority priority, + String appNodeLabelExpression, String amNodeLabelExpression) { + ApplicationReport report = + newInstance(applicationId, applicationAttemptId, user, queue, name, + host, rpcPort, clientToAMToken, state, diagnostics, url, + startTime, 0, finishTime, finalStatus, appResources, + origTrackingUrl, progress, applicationType, amRmToken); + report.setApplicationTags(tags); + report.setUnmanagedApp(unmanagedApplication); + report.setPriority(priority); + report.setAppNodeLabelExpression(appNodeLabelExpression); + report.setAmNodeLabelExpression(amNodeLabelExpression); + return report; + } + @Private @Unstable public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken, Set tags, boolean unmanagedApplication, Priority priority, @@ -98,8 +127,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationReport report = newInstance(applicationId, applicationAttemptId, user, queue, name, host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, - finishTime, finalStatus, appResources, origTrackingUrl, progress, - applicationType, amRmToken); + launchTime, finishTime, finalStatus, appResources, + origTrackingUrl, progress, applicationType, amRmToken); report.setApplicationTags(tags); report.setUnmanagedApp(unmanagedApplication); report.setPriority(priority); @@ -282,6 +311,14 @@ public static ApplicationReport newInstance(ApplicationId applicationId, @Unstable public abstract void setStartTime(long startTime); + @Private + @Unstable + public abstract void setLaunchTime(long setLaunchTime); + + @Public + @Unstable + public abstract long getLaunchTime(); + /** * Get the finish time of the application. * @return finish time of the application diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 80c043fbab9..7cc85dedde2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -246,6 +246,7 @@ message ApplicationReportProto { optional string appNodeLabelExpression = 24; optional string amNodeLabelExpression = 25; repeated AppTimeoutsMapProto appTimeouts = 26; + optional int64 launchTime = 27; } message AppTimeoutsMapProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index d90e1317b13..5a947e4b04a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -627,7 +627,7 @@ public ApplicationReport createFakeAppReport() { ApplicationReport.newInstance(appId, attemptId, "fakeUser", "fakeQueue", "fakeApplicationName", "localhost", 0, null, YarnApplicationState.FINISHED, "fake an application report", "", - 1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f, + 1000l, 1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f, "fakeApplicationType", null); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java index c3e3c414771..f0e3ca2a4cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java @@ -332,7 +332,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = @@ -389,7 +389,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null); applicationReports.add(newApplicationReport2); @@ -399,7 +399,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); @@ -409,7 +409,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index f84b73109f8..69b5b191fb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -738,7 +738,7 @@ private List createAppReports() { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -813,7 +813,7 @@ private List createAppReports() { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null); applicationReports.add(newApplicationReport2); @@ -822,7 +822,7 @@ private List createAppReports() { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); @@ -833,7 +833,7 @@ private List createAppReports() { applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 0f0c9bfd086..bec7e5f974a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -124,7 +124,7 @@ public void testGetApplicationReport() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", null, null, false, Priority.newInstance(0), "high-mem", "high-mem"); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); @@ -371,7 +371,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, Sets.newHashSet("tag1", "tag3"), false, Priority.UNDEFINED, "", ""); List applicationReports = @@ -382,7 +382,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null, Sets.newHashSet("tag2", "tag3"), false, Priority.UNDEFINED, "", ""); @@ -392,7 +392,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null, Sets.newHashSet("tag1", "tag4"), false, Priority.UNDEFINED, "", ""); @@ -402,7 +402,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null, Sets.newHashSet("tag1"), false, Priority.UNDEFINED, "", ""); @@ -412,7 +412,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId5, ApplicationAttemptId.newInstance(applicationId5, 5), "user5", "queue5", "appname5", "host5", 128, null, - YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, + YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, 5, FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", null, Sets.newHashSet("tag2", "tag4"), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport5); @@ -421,7 +421,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport6 = ApplicationReport.newInstance( applicationId6, ApplicationAttemptId.newInstance(applicationId6, 6), "user6", "queue6", "appname6", "host6", 129, null, - YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, + YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, 6, FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG", null, new HashSet(), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport6); @@ -995,7 +995,7 @@ public void testKillApplication() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); @@ -1008,7 +1008,7 @@ public void testKillApplication() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -1047,12 +1047,12 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport1 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.34344f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport1); @@ -1072,12 +1072,12 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport3); @@ -1115,7 +1115,7 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport5); @@ -1142,12 +1142,12 @@ public void testKillApplicationsOfDifferentEndStates() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport6 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport5); @@ -1170,7 +1170,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); @@ -1185,7 +1185,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -1220,7 +1220,7 @@ public void testMoveApplicationAcrossQueuesWithNewCommand() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(newApplicationReport2); @@ -1235,7 +1235,7 @@ public void testMoveApplicationAcrossQueuesWithNewCommand() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(newApplicationReport); @@ -2003,7 +2003,7 @@ public void testUpdateApplicationPriority() throws Exception { ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 036fa907fe6..ddd9accd898 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -220,6 +220,17 @@ public long getStartTime() { return p.getStartTime(); } + @Override + public long getLaunchTime() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getLaunchTime(); + } + + @Override + public void setLaunchTime(long launchTime) { + maybeInitBuilder(); + builder.setLaunchTime(launchTime); + } @Override public long getFinishTime() { ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index 46fc4d58d92..ea39a4ccdba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -58,7 +58,7 @@ protected static ApplicationReport createApplicationReport( ApplicationReport appReport = ApplicationReport.newInstance(appId, appAttemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, - "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, false, Priority.newInstance(0),"",""); return appReport; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index eeec5b1b6a4..e9d2c5e1b50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -137,7 +137,7 @@ private ApplicationReport convertToApplicationReport( currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(), appHistory.getApplicationName(), host, rpcPort, null, appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), - trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), + trackingUrl, appHistory.getStartTime(), 0, appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), null, "", 100, appHistory.getApplicationType(), null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index cd146a17a63..27ee855965c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -375,7 +375,7 @@ public static ApplicationReport newApplicationReport( ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, - String url, long startTime, long finishTime, + String url, long startTime, long launchTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String appType, Token amRmToken, Set tags, @@ -394,6 +394,7 @@ public static ApplicationReport newApplicationReport( report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setLaunchTime(launchTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index e82dc158af1..a7b95c0aae0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -194,10 +194,9 @@ public ApplicationReport run() throws Exception { ._("FinalStatus Reported by AM:", clairfyAppFinalStatus(app.getFinalAppStatus())) ._("Started:", Times.format(app.getStartedTime())) - ._( - "Elapsed:", - StringUtils.formatTime(Times.elapsed(app.getStartedTime(), - app.getFinishedTime()))) + ._("Launched:", Times.format(app.getLaunchTime())) + ._("Finished:", Times.format(app.getFinishedTime())) + ._("Elapsed:", StringUtils.formatTime(app.getElapsedTime())) ._( "Tracking URL:", app.getTrackingUrl() == null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index 04aa3d4618f..75607473fd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -150,7 +150,9 @@ protected void renderData(Block html) { html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") .th(".name", "Name").th(".type", "Application Type") .th(".queue", "Queue").th(".priority", "Application Priority") - .th(".starttime", "StartTime").th(".finishtime", "FinishTime") + .th(".starttime", "StartTime") + .th(".launchtime", "LaunchTime") + .th(".finishtime", "FinishTime") .th(".state", "State").th(".finalstatus", "FinalStatus") .th(".progress", "Progress").th(".ui", "Tracking UI")._()._().tbody(); @@ -188,6 +190,7 @@ protected void renderData(Block html) { .getQueue()))).append("\",\"").append(String .valueOf(app.getPriority())) .append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getLaunchTime()) .append("\",\"").append(app.getFinishedTime()) .append("\",\"") .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java index 29f1e634563..b2f65a84504 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java @@ -51,7 +51,7 @@ private static String getAppsTableColumnDefs( sb.append("[\n") .append("{'sType':'natural', 'aTargets': [0]") .append(", 'mRender': parseHadoopID }") - .append("\n, {'sType':'numeric', 'aTargets': [6, 7]") + .append("\n, {'sType':'numeric', 'aTargets': [6, 7, 8]") .append(", 'mRender': renderHadoopDate }") .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':"); if (isFairSchedulerPage) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index ac2f8da6aa2..b4687a42607 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -55,6 +55,7 @@ public class AppInfo { protected FinalApplicationStatus finalAppStatus; protected long submittedTime; protected long startedTime; + private long launchTime; protected long finishedTime; protected long elapsedTime; protected String applicationTags; @@ -88,6 +89,7 @@ public AppInfo(ApplicationReport app) { originalTrackingUrl = app.getOriginalTrackingUrl(); submittedTime = app.getStartTime(); startedTime = app.getStartTime(); + launchTime = app.getLaunchTime(); finishedTime = app.getFinishTime(); elapsedTime = Times.elapsed(startedTime, finishedTime); finalAppStatus = app.getFinalApplicationStatus(); @@ -198,6 +200,10 @@ public long getSubmittedTime() { return submittedTime; } + public long getLaunchTime() { + return launchTime; + } + public long getStartedTime() { return startedTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 19f1d81e15e..8213420dd90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -310,7 +310,7 @@ public void run() { LOG.info("Launching master" + application.getAppAttemptId()); launch(); handler.handle(new RMAppAttemptEvent(application.getAppAttemptId(), - RMAppAttemptEventType.LAUNCHED)); + RMAppAttemptEventType.LAUNCHED, System.currentTimeMillis())); } catch(Exception ie) { onAMLaunchFailed(masterContainer.getId(), ie); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java index 79a5de25846..2b0bd2b5bad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java @@ -47,7 +47,8 @@ public abstract class ApplicationStateData { public static ApplicationStateData newInstance(long submitTime, long startTime, String user, ApplicationSubmissionContext submissionContext, RMAppState state, - String diagnostics, long finishTime, CallerContext callerContext) { + String diagnostics, long launchTime, long finishTime, + CallerContext callerContext) { ApplicationStateData appState = Records.newRecord(ApplicationStateData.class); appState.setSubmitTime(submitTime); appState.setStartTime(startTime); @@ -55,6 +56,7 @@ public static ApplicationStateData newInstance(long submitTime, appState.setApplicationSubmissionContext(submissionContext); appState.setState(state); appState.setDiagnostics(diagnostics); + appState.setLaunchTime(launchTime); appState.setFinishTime(finishTime); appState.setCallerContext(callerContext); return appState; @@ -63,7 +65,8 @@ public static ApplicationStateData newInstance(long submitTime, public static ApplicationStateData newInstance(long submitTime, long startTime, String user, ApplicationSubmissionContext submissionContext, RMAppState state, - String diagnostics, long finishTime, CallerContext callerContext, + String diagnostics, long launchTime, long finishTime, + CallerContext callerContext, Map applicationTimeouts) { ApplicationStateData appState = Records.newRecord(ApplicationStateData.class); @@ -73,6 +76,7 @@ public static ApplicationStateData newInstance(long submitTime, appState.setApplicationSubmissionContext(submissionContext); appState.setState(state); appState.setDiagnostics(diagnostics); + appState.setLaunchTime(launchTime); appState.setFinishTime(finishTime); appState.setCallerContext(callerContext); appState.setApplicationTimeouts(applicationTimeouts); @@ -82,7 +86,7 @@ public static ApplicationStateData newInstance(long submitTime, public static ApplicationStateData newInstance(long submitTime, long startTime, ApplicationSubmissionContext context, String user, CallerContext callerContext) { - return newInstance(submitTime, startTime, user, context, null, "", 0, + return newInstance(submitTime, startTime, user, context, null, "", 0, 0, callerContext); } @@ -136,6 +140,20 @@ public int getFirstAttemptId() { @Unstable public abstract void setStartTime(long startTime); + + + /** + * Get the launch time of the application. + * @return launch time of the application + */ + @Public + @Stable + public abstract long getLaunchTime(); + + @Private + @Unstable + public abstract void setLaunchTime(long launchTime); + /** * The application submitter */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java index d037e68a12a..f5cd107e173 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java @@ -120,6 +120,19 @@ public void setStartTime(long startTime) { builder.setStartTime(startTime); } + + @Override + public long getLaunchTime() { + ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getLaunchTime(); + } + + @Override + public void setLaunchTime(long launchTime) { + maybeInitBuilder(); + builder.setLaunchTime(launchTime); + } + @Override public String getUser() { ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 93c41b6747c..434a7e70b37 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -176,7 +176,15 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, * @return the submit time of the application. */ long getSubmitTime(); - + + /** + * The launch time of the application. + * Since getStartTime() returns what is essentially submit time, + * this new field is to prevent potential backwards compatibility issues. + * @return the launch time of the application. + */ + long getLaunchTime(); + /** * The tracking url for the application master. * @return the tracking url for the application master. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java index 5c46945754e..3212d6c504c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java @@ -37,6 +37,12 @@ public RMAppEvent(ApplicationId appId, RMAppEventType type, this.diagnosticMsg = diagnostic; } + public RMAppEvent(ApplicationId appId, RMAppEventType type, long timeStamp) { + super(type, timeStamp); + this.appId = appId; + this.diagnosticMsg = ""; + } + public ApplicationId getApplicationId() { return this.appId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java index 514efd44dd6..4b55d389540 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java @@ -37,6 +37,7 @@ public enum RMAppEventType { ATTEMPT_FAILED, ATTEMPT_KILLED, NODE_UPDATE, + ATTEMPT_LAUNCHED, // Source: Container and ResourceTracker APP_RUNNING_ON_NODE, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index af5ef5eb26d..63acaeab100 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -159,6 +159,7 @@ public class RMAppImpl implements RMApp, Recoverable { // Mutable fields private long startTime; + private long launchTime = 0; private long finishTime = 0; private long storedFinishTime = 0; private int firstAttemptIdInStateStore = 1; @@ -283,6 +284,10 @@ RMAppEventType.KILL, new KillAttemptTransition()) .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, RMAppEventType.APP_RUNNING_ON_NODE, new AppRunningOnNodeTransition()) + // Handle AppAttemptLaunch to upate the launchTime and publish to ATS + .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, + RMAppEventType.ATTEMPT_LAUNCHED, + new AttemptLaunchedTransition()) // Transitions from RUNNING state .addTransition(RMAppState.RUNNING, RMAppState.RUNNING, @@ -769,9 +774,9 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, this.applicationId, currentApplicationAttemptId, this.user, this.queue, this.name, host, rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl, this.startTime, - this.finishTime, finishState, appUsageReport, origTrackingUrl, - progress, this.applicationType, amrmToken, applicationTags, - this.getApplicationPriority()); + this.launchTime, this.finishTime, finishState, appUsageReport, + origTrackingUrl, progress, this.applicationType, amrmToken, + applicationTags, this.getApplicationPriority()); report.setLogAggregationStatus(logAggregationStatus); report.setUnmanagedApp(submissionContext.getUnmanagedAM()); report.setAppNodeLabelExpression(getAppNodeLabelExpression()); @@ -824,6 +829,17 @@ public long getStartTime() { } } + @Override + public long getLaunchTime() { + this.readLock.lock(); + + try { + return this.launchTime; + } finally { + this.readLock.unlock(); + } + } + @Override public long getSubmitTime() { return this.submitTime; @@ -921,6 +937,7 @@ public void recover(RMState state) { .getDiagnostics()); this.storedFinishTime = appState.getFinishTime(); this.startTime = appState.getStartTime(); + this.launchTime = appState.getLaunchTime(); this.callerContext = appState.getCallerContext(); this.applicationTimeouts = appState.getApplicationTimeouts(); // If interval > 0, some attempts might have been deleted. @@ -1023,6 +1040,21 @@ public void transition(RMAppImpl app, RMAppEvent event) { }; } + private static final class AttemptLaunchedTransition + extends RMAppTransition { + @Override + public void transition(RMAppImpl app, RMAppEvent event) { + + if(app.launchTime == 0) { + LOG.info("update the launch time for applicationId: "+ + app.getApplicationId()+", attemptId: "+ + app.getCurrentAppAttempt().getAppAttemptId()+ + "launchTime: "+event.getTimestamp()); + app.launchTime = event.getTimestamp(); + } + } + } + private static final class AppRunningOnNodeTransition extends RMAppTransition { public void transition(RMAppImpl app, RMAppEvent event) { RMAppRunningOnNodeEvent nodeAddedEvent = (RMAppRunningOnNodeEvent) event; @@ -1278,7 +1310,8 @@ private void rememberTargetTransitionsAndStoreState(RMAppEvent event, ApplicationStateData appState = ApplicationStateData.newInstance(this.submitTime, this.startTime, this.user, this.submissionContext, - stateToBeStored, diags, this.storedFinishTime, this.callerContext); + stateToBeStored, diags, this.launchTime, this.storedFinishTime, + this.callerContext); appState.setApplicationTimeouts(this.applicationTimeouts); this.rmContext.getStateStore().updateApplicationState(appState); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java index 6df6b19f978..2ddc981067a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java @@ -38,6 +38,13 @@ public RMAppAttemptEvent(ApplicationAttemptId appAttemptId, this.diagnosticMsg = diagnostics; } + public RMAppAttemptEvent(ApplicationAttemptId appAttemptId, + RMAppAttemptEventType type, long timeStamp) { + super(type, timeStamp); + this.appAttemptId = appAttemptId; + this.diagnosticMsg = ""; + } + public ApplicationAttemptId getApplicationAttemptId() { return this.appAttemptId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 0b56f092012..93b5383dccf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1519,7 +1519,9 @@ public void transition(RMAppAttemptImpl appAttempt, appAttempt.launchAMStartTime; ClusterMetrics.getMetrics().addAMLaunchDelay(delay); } - + appAttempt.eventHandler.handle( + new RMAppEvent(appAttempt.getAppAttemptId().getApplicationId(), + RMAppEventType.ATTEMPT_LAUNCHED, event.getTimestamp())); appAttempt .updateAMLaunchDiagnostics(AMState.LAUNCHED.getDiagnosticMessage()); // Register with AMLivelinessMonitor diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java index b7a7a9372c8..e49d7b640ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -89,6 +89,7 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, th(".queue", "Queue"). th(".fairshare", "Fair Share"). th(".starttime", "StartTime"). + th(".launchTime", "LaunchTime"). th(".finishtime", "FinishTime"). th(".state", "State"). th(".finalstatus", "FinalStatus"). @@ -135,6 +136,7 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, appInfo.getQueue()))).append("\",\"") .append(fairShare).append("\",\"") .append(appInfo.getStartTime()).append("\",\"") + .append(appInfo.getLaunchTime()).append("\",\"") .append(appInfo.getFinishTime()).append("\",\"") .append(appInfo.getState()).append("\",\"") .append(appInfo.getFinalStatus()).append("\",\"") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index f56069d08d3..e0baf10021c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -63,6 +63,7 @@ protected void renderData(Block html) { .th(".name", "Name").th(".type", "Application Type") .th(".queue", "Queue").th(".priority", "Application Priority") .th(".starttime", "StartTime") + .th("launchtime", "LaunchTime") .th(".finishtime", "FinishTime").th(".state", "State") .th(".finalstatus", "FinalStatus") .th(".runningcontainer", "Running Containers") @@ -135,6 +136,7 @@ protected void renderData(Block html) { .getQueue()))).append("\",\"").append(String .valueOf(app.getPriority())) .append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getLaunchTime()) .append("\",\"").append(app.getFinishedTime()) .append("\",\"") .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 7fed3dfae06..71fbfc44e5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -87,6 +87,7 @@ public class AppInfo { // these are only allowed if acls allow protected long startedTime; + private long launchTime; protected long finishedTime; protected long elapsedTime; protected String amContainerLogs; @@ -176,6 +177,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { this.startedTime = app.getStartTime(); + this.launchTime = app.getLaunchTime(); this.finishedTime = app.getFinishTime(); this.elapsedTime = Times.elapsed(app.getStartTime(), app.getFinishTime()); @@ -386,6 +388,10 @@ public long getStartTime() { return this.startedTime; } + public long getLaunchTime() { + return this.launchTime; + } + public long getFinishTime() { return this.finishedTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto index 247cd2195d9..0300d65f857 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -70,6 +70,7 @@ message ApplicationStateDataProto { optional int64 finish_time = 7; optional hadoop.common.RPCCallerContextProto caller_context = 8; repeated ApplicationTimeoutMapProto application_timeouts = 9; + optional int64 launch_time = 10; } message ApplicationAttemptStateDataProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index f826631a21d..c6d68b3208a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -89,7 +89,12 @@ public long getStartTime() { public long getSubmitTime() { throw new UnsupportedOperationException("Not supported yet."); } - + + @Override + public long getLaunchTime() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public long getFinishTime() { throw new UnsupportedOperationException("Not supported yet."); @@ -258,6 +263,7 @@ public static RMApp newApplication(int i) { final String name = newAppName(); final String queue = newQueue(); final long start = 123456 + i * 1000; + final long launch = start + i * 100; final long finish = 234567 + i * 1000; final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE; YarnApplicationState[] allStates = YarnApplicationState.values(); @@ -293,6 +299,11 @@ public long getStartTime() { return start; } + @Override + public long getLaunchTime() { + return launch; + } + @Override public long getFinishTime() { return finish; @@ -342,7 +353,7 @@ public ApplicationReport createAndGetApplicationReport( ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), - getTrackingUrl(), getStartTime(), getFinishTime(), + getTrackingUrl(), getLaunchTime(), getStartTime(), getFinishTime(), getFinalApplicationStatus(), usageReport , null, getProgress(), type, null); return report; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index fdac89ce993..f01f730bb26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -356,7 +356,7 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), appState.getApplicationSubmissionContext(), RMAppState.FINISHED, - "appDiagnostics", 1234, appState.getCallerContext()); + "appDiagnostics", 123, 1234, appState.getCallerContext()); appState2.attempts.putAll(appState.attempts); store.updateApplicationState(appState2); @@ -381,7 +381,7 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, ApplicationStateData dummyApp = ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), dummyContext, - RMAppState.FINISHED, "appDiagnostics", 1234, null); + RMAppState.FINISHED, "appDiagnostics", 123, 1234, null); store.updateApplicationState(dummyApp); ApplicationAttemptId dummyAttemptId = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index 68ed5c1a851..4ec64715ab6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -410,7 +410,7 @@ public void run() { store.storeApplicationStateInternal( ApplicationId.newInstance(100L, 1), ApplicationStateData.newInstance(111, 111, "user", null, - RMAppState.ACCEPTED, "diagnostics", 333, null)); + RMAppState.ACCEPTED, "diagnostics", 222, 333, null)); } catch (Exception e) { assertionFailedInThread.set(true); e.printStackTrace(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index fbacac0cc86..b5b792578e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -797,7 +797,7 @@ private static ApplicationStateData createAppState( long finishTime, boolean isFinished) { return ApplicationStateData.newInstance(submitTime, startTime, "test", ctxt, isFinished ? RMAppState.FINISHED : null, isFinished ? - "appDiagnostics" : "", isFinished ? finishTime : 0, null); + "appDiagnostics" : "", 0, isFinished ? finishTime : 0, null); } private static ApplicationAttemptStateData createFinishedAttempt( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 17cafefc65f..6afa0615559 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -55,6 +55,7 @@ public class MockRMApp implements RMApp { String name = MockApps.newAppName(); String queue = MockApps.newQueue(); long start = System.currentTimeMillis() - (int) (Math.random() * DT); + private long launch = start; long submit = start - (int) (Math.random() * DT); long finish = 0; RMAppState state = RMAppState.NEW; @@ -185,6 +186,11 @@ public long getSubmitTime() { return submit; } + @Override + public long getLaunchTime() { + return launch; + } + public void setStartTime(long time) { this.start = time; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 920bd4fd2b9..70887e0f4bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -1239,7 +1239,7 @@ public void createRMStateForApplications( ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(), app.getUser(), app.getApplicationSubmissionContext(), rmAppState, - null, app.getFinishTime(), null); + null, app.getLaunchTime(), app.getFinishTime(), null); applicationState.put(app.getApplicationId(), appState); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index b1256083ef8..93377be1ea9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -257,7 +257,7 @@ public static ClientRMService mockClientRMService(RMContext rmContext) { app.getName(), (String) null, 0, (Token) null, app.createApplicationState(), app.getDiagnostics().toString(), (String) null, - app.getStartTime(), app.getFinishTime(), + app.getStartTime(), app.getLaunchTime(), app.getFinishTime(), app.getFinalApplicationStatus(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(), app.getProgress(), app.getApplicationType(), (Token) null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 5cc86d43a6f..aea158a0a1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1498,6 +1498,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) WebServicesTestUtils.getXmlString(element, "diagnostics"), WebServicesTestUtils.getXmlLong(element, "clusterId"), WebServicesTestUtils.getXmlLong(element, "startedTime"), + WebServicesTestUtils.getXmlLong(element, "launchTime"), WebServicesTestUtils.getXmlLong(element, "finishedTime"), WebServicesTestUtils.getXmlLong(element, "elapsedTime"), WebServicesTestUtils.getXmlString(element, "amHostHttpAddress"), @@ -1550,7 +1551,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) throws JSONException, Exception { - int expectedNumberOfElements = 36 + (hasResourceReqs ? 2 : 0); + int expectedNumberOfElements = 37 + (hasResourceReqs ? 2 : 0); String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() @@ -1576,8 +1577,10 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) info.getString("state"), info.getString("finalStatus"), (float) info.getDouble("progress"), info.getString("trackingUI"), info.getString("diagnostics"), info.getLong("clusterId"), - info.getLong("startedTime"), info.getLong("finishedTime"), - info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), + info.getLong("startedTime"), info.getLong("launchTime"), + info.getLong("finishedTime"), + info.getLong("elapsedTime"), + info.getString("amHostHttpAddress"), info.getString("amContainerLogs"), info.getInt("allocatedMB"), info.getInt("allocatedVCores"), info.getInt("runningContainers"), (float) info.getDouble("queueUsagePercentage"), @@ -1600,8 +1603,9 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) public void verifyAppInfoGeneric(RMApp app, String id, String user, String name, String applicationType, String queue, int prioirty, String state, String finalStatus, float progress, String trackingUI, - String diagnostics, long clusterId, long startedTime, long finishedTime, - long elapsedTime, String amHostHttpAddress, String amContainerLogs, + String diagnostics, long clusterId, long startedTime, + long launchTime, long finishedTime, long elapsedTime, + String amHostHttpAddress, String amContainerLogs, int allocatedMB, int allocatedVCores, int numContainers, float queueUsagePerc, float clusterUsagePerc, int preemptedResourceMB, int preemptedResourceVCores,