YARN-6616: YARN AHS shows submitTime for jobs same as startTime. Contributed by Prabhu Joseph

(cherry picked from commit 04105bbfdb)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java

@@ -62,6 +62,22 @@ public abstract class ApplicationReport {
       FinalApplicationStatus finalStatus,
       ApplicationResourceUsageReport appResources, String origTrackingUrl,
       float progress, String applicationType, Token amRmToken) {
+    return newInstance(applicationId, applicationAttemptId, user, queue, name,
+        host, rpcPort, clientToAMToken, state, diagnostics, url,
+        startTime, startTime, launchTime, finishTime, finalStatus, appResources,
+        origTrackingUrl, progress, applicationType, amRmToken);
+  }
+
+  @Private
+  @Unstable
+  public static ApplicationReport newInstance(ApplicationId applicationId,
+      ApplicationAttemptId applicationAttemptId, String user, String queue,
+      String name, String host, int rpcPort, Token clientToAMToken,
+      YarnApplicationState state, String diagnostics, String url,
+      long startTime, long submitTime, long launchTime, long finishTime,
+      FinalApplicationStatus finalStatus,
+      ApplicationResourceUsageReport appResources, String origTrackingUrl,
+      float progress, String applicationType, Token amRmToken) {
     ApplicationReport report = Records.newRecord(ApplicationReport.class);
     report.setApplicationId(applicationId);
     report.setCurrentApplicationAttemptId(applicationAttemptId);
@@ -75,6 +91,7 @@ public abstract class ApplicationReport {
     report.setDiagnostics(diagnostics);
     report.setTrackingUrl(url);
     report.setStartTime(startTime);
+    report.setSubmitTime(submitTime);
     report.setLaunchTime(launchTime);
     report.setFinishTime(finishTime);
     report.setFinalApplicationStatus(finalStatus);
@@ -102,7 +119,7 @@ public abstract class ApplicationReport {
     ApplicationReport report =
             newInstance(applicationId, applicationAttemptId, user, queue, name,
                     host, rpcPort, clientToAMToken, state, diagnostics, url,
-                    startTime, 0, finishTime, finalStatus, appResources,
+                    startTime, 0, 0, finishTime, finalStatus, appResources,
                     origTrackingUrl, progress, applicationType, amRmToken);
     report.setApplicationTags(tags);
     report.setUnmanagedApp(unmanagedApplication);
@@ -124,10 +141,30 @@ public abstract class ApplicationReport {
       float progress, String applicationType, Token amRmToken, Set<String> tags,
       boolean unmanagedApplication, Priority priority,
       String appNodeLabelExpression, String amNodeLabelExpression) {
+    return newInstance(applicationId, applicationAttemptId, user, queue, name,
+        host, rpcPort, clientToAMToken, state, diagnostics, url,
+        startTime, startTime, launchTime, finishTime, finalStatus, appResources,
+        origTrackingUrl, progress, applicationType, amRmToken, tags,
+        unmanagedApplication, priority, appNodeLabelExpression,
+        amNodeLabelExpression);
+  }
+
+  @Private
+  @Unstable
+  public static ApplicationReport newInstance(ApplicationId applicationId,
+      ApplicationAttemptId applicationAttemptId, String user, String queue,
+      String name, String host, int rpcPort, Token clientToAMToken,
+      YarnApplicationState state, String diagnostics, String url,
+      long startTime, long submitTime, long launchTime, long finishTime,
+      FinalApplicationStatus finalStatus,
+      ApplicationResourceUsageReport appResources, String origTrackingUrl,
+      float progress, String applicationType, Token amRmToken, Set<String> tags,
+      boolean unmanagedApplication, Priority priority,
+      String appNodeLabelExpression, String amNodeLabelExpression) {
     ApplicationReport report =
         newInstance(applicationId, applicationAttemptId, user, queue, name,
           host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
-          launchTime, finishTime, finalStatus, appResources,
+          submitTime, launchTime, finishTime, finalStatus, appResources,
           origTrackingUrl, progress, applicationType, amRmToken);
     report.setApplicationTags(tags);
     report.setUnmanagedApp(unmanagedApplication);
@@ -311,6 +348,14 @@ public abstract class ApplicationReport {
   @Unstable
   public abstract void setStartTime(long startTime);
 
+  @Public
+  @Stable
+  public abstract long getSubmitTime();
+
+  @Private
+  @Unstable
+  public abstract void setSubmitTime(long submitTime);
+
   @Private
   @Unstable
   public abstract void setLaunchTime(long setLaunchTime);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

@@ -247,6 +247,7 @@ message ApplicationReportProto {
   optional string amNodeLabelExpression = 25;
   repeated AppTimeoutsMapProto appTimeouts = 26;
   optional int64 launchTime = 27;
+  optional int64 submitTime = 28;
 }
 
 message AppTimeoutsMapProto {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java

@@ -102,6 +102,8 @@ public class TestAHSClient {
     Assert.assertEquals(report, expectedReports.get(0));
     Assert.assertEquals(report.getApplicationId().toString(), expectedReports
       .get(0).getApplicationId().toString());
+    Assert.assertEquals(report.getSubmitTime(), expectedReports.get(0)
+      .getSubmitTime());
     client.stop();
   }
 
@@ -332,7 +334,7 @@ public class TestAHSClient {
           ApplicationReport.newInstance(applicationId,
             ApplicationAttemptId.newInstance(applicationId, 1), "user",
             "queue", "appname", "host", 124, null,
-            YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0,
+            YarnApplicationState.RUNNING, "diagnostics", "url", 1, 2, 3, 4,
             FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
             null);
       List<ApplicationReport> applicationReports =

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java

@@ -220,6 +220,12 @@ public class ApplicationReportPBImpl extends ApplicationReport {
     return p.getStartTime();
   }
 
+  @Override
+  public long getSubmitTime() {
+    ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getSubmitTime();
+  }
+
   @Override
   public long getLaunchTime() {
     ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
@@ -430,6 +436,12 @@ public class ApplicationReportPBImpl extends ApplicationReport {
     builder.setFinishTime(finishTime);
   }
 
+  @Override
+  public void setSubmitTime(long submitTime) {
+    maybeInitBuilder();
+    builder.setSubmitTime(submitTime);
+  }
+
   @Override
   public void setFinalApplicationStatus(FinalApplicationStatus finishState) {
     maybeInitBuilder();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java

@@ -137,9 +137,9 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
       currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(),
       appHistory.getApplicationName(), host, rpcPort, null,
       appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(),
-      trackingUrl, appHistory.getStartTime(), 0, appHistory.getFinishTime(),
+      trackingUrl, appHistory.getStartTime(), appHistory.getSubmitTime(), 0,
-      appHistory.getFinalApplicationStatus(), null, "", 100,
+      appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(),
-      appHistory.getApplicationType(), null);
+      null, "", 100, appHistory.getApplicationType(), null);
   }
 
   private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java

@@ -249,6 +249,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
     String type = null;
     boolean unmanagedApplication = false;
     long createdTime = 0;
+    long submittedTime = 0;
     long finishedTime = 0;
     float progress = 0.0f;
     int applicationPriority = 0;
@@ -280,10 +281,11 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
         return new ApplicationReportExt(ApplicationReport.newInstance(
             ApplicationId.fromString(entity.getEntityId()),
             latestApplicationAttemptId, user, queue, name, null, -1, null,
-            state, diagnosticsInfo, null, createdTime, finishedTime,
+            state, diagnosticsInfo, null, createdTime, submittedTime, 0,
-            finalStatus, null, null, progress, type, null, appTags,
+            finishedTime, finalStatus, null, null, progress, type, null,
-            unmanagedApplication, Priority.newInstance(applicationPriority),
+            appTags, unmanagedApplication, Priority.newInstance(
-            appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
+            applicationPriority), appNodeLabelExpression,
+            amNodeLabelExpression), appViewACLs);
       }
       if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
         queue =
@@ -328,6 +330,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
             entityInfo.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)
                 .toString();
       }
+      submittedTime = parseLong(entityInfo,
+          ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO);
 
       if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
         long vcoreSeconds = parseLong(entityInfo,
@@ -438,10 +442,10 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
     return new ApplicationReportExt(ApplicationReport.newInstance(
         ApplicationId.fromString(entity.getEntityId()),
         latestApplicationAttemptId, user, queue, name, null, -1, null, state,
-        diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
+        diagnosticsInfo, null, createdTime, submittedTime, 0, finishedTime,
-        appResources, null, progress, type, null, appTags, unmanagedApplication,
+        finalStatus, appResources, null, progress, type, null, appTags,
-        Priority.newInstance(applicationPriority), appNodeLabelExpression,
+        unmanagedApplication, Priority.newInstance(applicationPriority),
-        amNodeLabelExpression), appViewACLs);
+        appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
   }
 
   private static long parseLong(Map<String, Object> entityInfo,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java

@@ -214,6 +214,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
       }
       Assert.assertEquals(Integer.MAX_VALUE + 2L
           + app.getApplicationId().getId(), app.getStartTime());
+      Assert.assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime());
       Assert.assertEquals(Integer.MAX_VALUE + 3L
           + +app.getApplicationId().getId(), app.getFinishTime());
       Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java

@@ -401,6 +401,7 @@ public class TestAHSWebServices extends JerseyTestBase {
     assertEquals("test app", app.get("name"));
     assertEquals(round == 0 ? "test diagnostics info" : "",
         app.get("diagnosticsInfo"));
+    assertEquals(Integer.MAX_VALUE + 1L, app.get("submittedTime"));
     assertEquals("test queue", app.get("queue"));
     assertEquals("user1", app.get("user"));
     assertEquals("test app type", app.get("type"));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java

@@ -53,7 +53,7 @@ public class AppInfo {
   protected String originalTrackingUrl;
   protected String trackingUrl;
   protected FinalApplicationStatus finalAppStatus;
-  protected long submittedTime;
+  private long submittedTime;
   protected long startedTime;
   private long launchTime;
   protected long finishedTime;
@@ -87,7 +87,7 @@ public class AppInfo {
     diagnosticsInfo = app.getDiagnostics();
     trackingUrl = app.getTrackingUrl();
     originalTrackingUrl = app.getOriginalTrackingUrl();
-    submittedTime = app.getStartTime();
+    submittedTime = app.getSubmitTime();
     startedTime = app.getStartTime();
     launchTime = app.getLaunchTime();
     finishedTime = app.getFinishTime();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java

@@ -263,6 +263,7 @@ public abstract class MockAsm extends MockApps {
     final String name = newAppName();
     final String queue = newQueue();
     final long start = 123456 + i * 1000;
+    final long submit = start + i * 50;
     final long launch = start + i * 100;
     final long finish = 234567 + i * 1000;
     final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE;
@@ -299,6 +300,11 @@ public abstract class MockAsm extends MockApps {
         return start;
       }
 
+      @Override
+      public long getSubmitTime() {
+        return submit;
+      }
+
       @Override
       public long getLaunchTime() {
         return launch;
@@ -353,9 +359,9 @@ public abstract class MockAsm extends MockApps {
         ApplicationReport report = ApplicationReport.newInstance(
             getApplicationId(), appAttemptId, getUser(), getQueue(), 
             getName(), null, 0, null, null, getDiagnostics().toString(), 
-            getTrackingUrl(), getLaunchTime(), getStartTime(), getFinishTime(),
+            getTrackingUrl(), getStartTime(), getSubmitTime(), getLaunchTime(),
-            getFinalApplicationStatus(), usageReport , null, getProgress(),
+            getFinishTime(), getFinalApplicationStatus(), usageReport, null,
-            type, null);
+            getProgress(), type, null);
         return report;
       }