YARN-4023. Publish Application Priority to TimelineServer. (Sunil G via rohithsharmaks)

(cherry picked from commit 1c12adb71f)

hadoop-yarn-project/CHANGES.txt

@@ -112,6 +112,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3887. Support changing Application priority during runtime. (Sunil G
     via jianhe)
 
+    YARN-4023. Publish Application Priority to TimelineServer. (Sunil G 
+    via rohithsharmaks)
+
   IMPROVEMENTS
 
     YARN-644. Basic null check is not performed on passed in arguments before

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java

@@ -92,7 +92,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
       long startTime, long finishTime, FinalApplicationStatus finalStatus,
       ApplicationResourceUsageReport appResources, String origTrackingUrl,
       float progress, String applicationType, Token amRmToken,
-      Set<String> tags, boolean unmanagedApplication) {
+      Set<String> tags, boolean unmanagedApplication, Priority priority) {
     ApplicationReport report =
         newInstance(applicationId, applicationAttemptId, user, queue, name,
           host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
@@ -100,6 +100,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
           applicationType, amRmToken);
     report.setApplicationTags(tags);
     report.setUnmanagedApp(unmanagedApplication);
+    report.setPriority(priority);
     return report;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java

@@ -104,7 +104,7 @@ public void testGetApplicationReport() throws Exception {
           "user", "queue", "appname", "host", 124, null,
           YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
           FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
-          null, null, false);
+          null, null, false, Priority.newInstance(0));
       newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
       newApplicationReport.setPriority(Priority.newInstance(0));
       when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java

@@ -22,6 +22,7 @@
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.junit.Assert;
@@ -59,7 +60,7 @@ protected static ApplicationReport createApplicationReport(
           "appname", "host", 124, null, YarnApplicationState.FINISHED,
           "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
           "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null,
-          null,false);
+          null, false, Priority.newInstance(0));
     return appReport;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java

@@ -251,6 +251,7 @@ private static ApplicationReportExt convertToApplicationReport(
     long createdTime = 0;
     long finishedTime = 0;
     float progress = 0.0f;
+    int applicationPriority = 0;
     ApplicationAttemptId latestApplicationAttemptId = null;
     String diagnosticsInfo = null;
     FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
@@ -276,10 +277,11 @@ private static ApplicationReportExt convertToApplicationReport(
       if (field == ApplicationReportField.USER_AND_ACLS) {
         return new ApplicationReportExt(ApplicationReport.newInstance(
             ConverterUtils.toApplicationId(entity.getEntityId()),
-            latestApplicationAttemptId, user, queue, name, null, -1, null, state,
+            latestApplicationAttemptId, user, queue, name, null, -1, null,
-            diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
+            state, diagnosticsInfo, null, createdTime, finishedTime,
-            null, progress, type, null, appTags,
+            finalStatus, null, null, progress, type, null, appTags,
-            unmanagedApplication), appViewACLs);
+            unmanagedApplication, Priority.newInstance(applicationPriority)),
+            appViewACLs);
       }
       if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
         queue =
@@ -303,6 +305,11 @@ private static ApplicationReportExt convertToApplicationReport(
                 ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)
                 .toString());
       }
+      if (entityInfo
+          .containsKey(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO)) {
+        applicationPriority = Integer.parseInt(entityInfo.get(
+            ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO).toString());
+      }
       if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
         long vcoreSeconds=Long.parseLong(entityInfo.get(
                 ApplicationMetricsConstants.APP_CPU_METRICS).toString());
@@ -373,8 +380,10 @@ private static ApplicationReportExt convertToApplicationReport(
     return new ApplicationReportExt(ApplicationReport.newInstance(
         ConverterUtils.toApplicationId(entity.getEntityId()),
         latestApplicationAttemptId, user, queue, name, null, -1, null, state,
-        diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
+        diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
-        null, progress, type, null, appTags, unmanagedApplication), appViewACLs);
+        appResources, null, progress, type, null, appTags,
+        unmanagedApplication, Priority.newInstance(applicationPriority)),
+        appViewACLs);
   }
 
   private static ApplicationAttemptReport convertToApplicationAttemptReport(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java

@@ -477,6 +477,8 @@ private static TimelineEntity createApplicationTimelineEntity(
     entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue");
     entityInfo.put(
         ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false");
+    entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
+        Priority.newInstance(0));
     entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
         Integer.MAX_VALUE + 1L);
     entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java

@@ -77,4 +77,7 @@ public class ApplicationMetricsConstants {
 
   public static final String UNMANAGED_APPLICATION_ENTITY_INFO =
       "YARN_APPLICATION_UNMANAGED_APPLICATION";
+
+  public static final String APPLICATION_PRIORITY_INFO =
+      "YARN_APPLICATION_PRIORITY";
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java

@@ -144,10 +144,10 @@ protected void renderData(Block html) {
     TBODY<TABLE<Hamlet>> tbody =
         html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
           .th(".name", "Name").th(".type", "Application Type")
-          .th(".queue", "Queue").th(".starttime", "StartTime")
+          .th(".queue", "Queue").th(".priority", "Application Priority")
-          .th(".finishtime", "FinishTime").th(".state", "State")
+          .th(".starttime", "StartTime").th(".finishtime", "FinishTime")
-          .th(".finalstatus", "FinalStatus").th(".progress", "Progress")
+          .th(".state", "State").th(".finalstatus", "FinalStatus")
-          .th(".ui", "Tracking UI")._()._().tbody();
+          .th(".progress", "Progress").th(".ui", "Tracking UI")._()._().tbody();
 
     StringBuilder appsTableData = new StringBuilder("[\n");
     for (ApplicationReport appReport : appReports) {
@@ -180,7 +180,9 @@ protected void renderData(Block html) {
         .append("\",\"")
         .append(
           StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
-            .getQueue()))).append("\",\"").append(app.getStartedTime())
+            .getQueue()))).append("\",\"").append(String
+                .valueOf(app.getPriority()))
+        .append("\",\"").append(app.getStartedTime())
         .append("\",\"").append(app.getFinishedTime())
         .append("\",\"")
         .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState())

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java

@@ -21,6 +21,7 @@
 import java.util.Set;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Priority;
 
 public class ApplicationCreatedEvent extends
     SystemMetricsEvent {
@@ -33,6 +34,7 @@ public class ApplicationCreatedEvent extends
   private long submittedTime;
   private Set<String> appTags;
   private boolean unmanagedApplication;
+  private Priority applicationPriority;
 
   public ApplicationCreatedEvent(ApplicationId appId,
       String name,
@@ -42,7 +44,8 @@ public ApplicationCreatedEvent(ApplicationId appId,
       long submittedTime,
       long createdTime,
       Set<String> appTags,
-      boolean unmanagedApplication) {
+      boolean unmanagedApplication,
+      Priority applicationPriority) {
     super(SystemMetricsEventType.APP_CREATED, createdTime);
     this.appId = appId;
     this.name = name;
@@ -52,6 +55,7 @@ public ApplicationCreatedEvent(ApplicationId appId,
     this.submittedTime = submittedTime;
     this.appTags = appTags;
     this.unmanagedApplication = unmanagedApplication;
+    this.applicationPriority = applicationPriority;
   }
 
   @Override
@@ -90,4 +94,8 @@ public Set<String> getAppTags() {
   public boolean isUnmanagedApp() {
     return unmanagedApplication;
   }
+
+  public Priority getApplicationPriority() {
+    return applicationPriority;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java

@@ -107,7 +107,8 @@ public void appCreated(RMApp app, long createdTime) {
               app.getQueue(),
               app.getSubmitTime(),
               createdTime, app.getApplicationTags(),
-              app.getApplicationSubmissionContext().getUnmanagedAM()));
+              app.getApplicationSubmissionContext().getUnmanagedAM(),
+              app.getApplicationSubmissionContext().getPriority()));
     }
   }
 
@@ -257,6 +258,8 @@ private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
     entityInfo.put(
         ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO,
         event.isUnmanagedApp());
+    entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
+        event.getApplicationPriority().getPriority());
     entity.setOtherInfo(entityInfo);
     TimelineEvent tEvent = new TimelineEvent();
     tEvent.setEventType(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java

@@ -141,6 +141,11 @@ public void testPublishApplicationMetrics() throws Exception {
           entity.getOtherInfo().get(
               ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO));
 
+      Assert.assertEquals(
+          app.getApplicationSubmissionContext().getPriority().getPriority(),
+          entity.getOtherInfo().get(
+              ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO));
+
       Assert
           .assertEquals(
               app.getUser(),
@@ -371,6 +376,7 @@ private static RMApp createRMApp(ApplicationId appId) {
     when(app.getApplicationTags()).thenReturn(appTags);
     ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class);
     when(asc.getUnmanagedAM()).thenReturn(false);
+    when(asc.getPriority()).thenReturn(Priority.newInstance(0));
     when(app.getApplicationSubmissionContext()).thenReturn(asc);
     return app;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md

@@ -1088,7 +1088,8 @@ Response Body:
           "startedTime":1430425001004,
           "finishedTime":1430425008861,
           "elapsedTime":7857,
-          "unmanagedApplication":"false"},
+          "unmanagedApplication":"false",
+          "applicationPriority":0},
           {
           "appId":"application_1430424020775_0003",
           "currentAppAttemptId":"appattempt_1430424020775_0003_000001",
@@ -1108,7 +1109,8 @@ Response Body:
           "startedTime":1430424956650,
           "finishedTime":1430424963907,
           "elapsedTime":7257,
-          "unmanagedApplication":"false"},
+          "unmanagedApplication":"false",
+          "applicationPriority":0},
           {
           "appId":"application_1430424020775_0002",
           "currentAppAttemptId":"appattempt_1430424020775_0002_000001",
@@ -1128,7 +1130,8 @@ Response Body:
           "startedTime":1430424769395,
           "finishedTime":1430424776594,
           "elapsedTime":7199,
-          "unmanagedApplication":"false"},
+          "unmanagedApplication":"false",
+          "applicationPriority":0},
           {
           "appId":"application_1430424020775_0001",
           "currentAppAttemptId":"appattempt_1430424020775_0001_000001",
@@ -1149,7 +1152,8 @@ Response Body:
           "finishedTime":1430424776594,
           "elapsedTime":18344,
           "applicationTags":"mrapplication,ta-example",
-          "unmanagedApplication":"false"
+          "unmanagedApplication":"false",
+          "applicationPriority":0
           }
       ]
     }
@@ -1192,6 +1196,7 @@ Response Body:
         <finishedTime>1430425008861</finishedTime>
         <elapsedTime>7857</elapsedTime>
         <unmanagedApplication>false</unmanagedApplication>
+        <applicationPriority>0</applicationPriority>
       </app>
       <app>
         <appId>application_1430424020775_0003</appId>
@@ -1213,6 +1218,7 @@ Response Body:
         <finishedTime>1430424963907</finishedTime>
         <elapsedTime>7257</elapsedTime>
         <unmanagedApplication>false</unmanagedApplication>
+        <applicationPriority>0</applicationPriority>
       </app>
       <app>
         <appId>application_1430424020775_0002</appId>
@@ -1234,6 +1240,7 @@ Response Body:
         <finishedTime>1430424776594</finishedTime>
         <elapsedTime>7199</elapsedTime>
         <unmanagedApplication>false</unmanagedApplication>
+        <applicationPriority>0</applicationPriority>
       </app>
       <app>
         <appId>application_1430424020775_0001</appId>
@@ -1256,6 +1263,7 @@ Response Body:
         <elapsedTime>18344</elapsedTime>
         <applicationTags>mrapplication,ta-example</applicationTags>
         <unmanagedApplication>false</unmanagedApplication>
+        <applicationPriority>0</applicationPriority>
       </app>
     </apps>
 
@@ -1307,6 +1315,7 @@ None
 | `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared |
 | `applicationTags` | string | The application tags. |
 | `unmanagedApplication` | boolean | Is the application unmanaged. |
+| `applicationPriority` | int | Priority of the submitted application. |
 
 ### Response Examples:
 
@@ -1345,6 +1354,7 @@ Response Body:
       "elapsedTime": 18344,
       "applicationTags": mrapplication,tag-example,
       "unmanagedApplication":"false"
+      "applicationPriority":0
     }
 
 #### XML response
@@ -1384,6 +1394,7 @@ Response Body:
        <elapsedTime>18344</elapsedTime>
        <applicationTags>mrapplication,ta-example</applicationTags>
        <unmanagedApplication>false</unmanagedApplication>
+       <applicationPriority>0</applicationPriority>
      </app>
 
 ## <a name="REST_API_APPLICATION_ATTEMPT_LIST"></a>Application Attempt List