diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0b76e982dc1..54a5d85ff20 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -112,6 +112,9 @@ Release 2.8.0 - UNRELEASED YARN-3887. Support changing Application priority during runtime. (Sunil G via jianhe) + YARN-4023. Publish Application Priority to TimelineServer. (Sunil G + via rohithsharmaks) + IMPROVEMENTS YARN-644. Basic null check is not performed on passed in arguments before diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 258b991df30..fa3b1e552b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -92,7 +92,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken, - Set tags, boolean unmanagedApplication) { + Set tags, boolean unmanagedApplication, Priority priority) { ApplicationReport report = newInstance(applicationId, applicationAttemptId, user, queue, name, host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, @@ -100,6 +100,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, applicationType, amRmToken); report.setApplicationTags(tags); report.setUnmanagedApp(unmanagedApplication); + report.setPriority(priority); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index eb50e003e93..8f17c8f9157 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -104,7 +104,7 @@ public void testGetApplicationReport() throws Exception { "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", - null, null, false); + null, null, false, Priority.newInstance(0)); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); newApplicationReport.setPriority(Priority.newInstance(0)); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index e2fa64805de..d0d1d407b03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.Assert; @@ -59,7 +60,7 @@ protected static ApplicationReport createApplicationReport( "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, - null,false); + null, false, Priority.newInstance(0)); return appReport; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 9ae99c5ef33..f02e83c7855 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -251,6 +251,7 @@ private static ApplicationReportExt convertToApplicationReport( long createdTime = 0; long finishedTime = 0; float progress = 0.0f; + int applicationPriority = 0; ApplicationAttemptId latestApplicationAttemptId = null; String diagnosticsInfo = null; FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED; @@ -276,10 +277,11 @@ private static ApplicationReportExt convertToApplicationReport( if (field == ApplicationReportField.USER_AND_ACLS) { return new ApplicationReportExt(ApplicationReport.newInstance( ConverterUtils.toApplicationId(entity.getEntityId()), - latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, - null, progress, type, null, appTags, - unmanagedApplication), appViewACLs); + latestApplicationAttemptId, user, queue, name, null, -1, null, + state, diagnosticsInfo, null, createdTime, finishedTime, + finalStatus, null, null, progress, type, null, appTags, + unmanagedApplication, Priority.newInstance(applicationPriority)), + appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -303,6 +305,11 @@ private static ApplicationReportExt convertToApplicationReport( ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO) .toString()); } + if (entityInfo + .containsKey(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO)) { + applicationPriority = Integer.parseInt(entityInfo.get( + ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO).toString()); + } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { long vcoreSeconds=Long.parseLong(entityInfo.get( ApplicationMetricsConstants.APP_CPU_METRICS).toString()); @@ -373,8 +380,10 @@ private static ApplicationReportExt convertToApplicationReport( return new ApplicationReportExt(ApplicationReport.newInstance( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, - null, progress, type, null, appTags, unmanagedApplication), appViewACLs); + diagnosticsInfo, null, createdTime, finishedTime, finalStatus, + appResources, null, progress, type, null, appTags, + unmanagedApplication, Priority.newInstance(applicationPriority)), + appViewACLs); } private static ApplicationAttemptReport convertToApplicationAttemptReport( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index fb4af85bb39..e24c11ca8ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -477,6 +477,8 @@ private static TimelineEntity createApplicationTimelineEntity( entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue"); entityInfo.put( ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false"); + entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, + Priority.newInstance(0)); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L); entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index 26b113f1325..481b84c2b54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -77,4 +77,7 @@ public class ApplicationMetricsConstants { public static final String UNMANAGED_APPLICATION_ENTITY_INFO = "YARN_APPLICATION_UNMANAGED_APPLICATION"; + + public static final String APPLICATION_PRIORITY_INFO = + "YARN_APPLICATION_PRIORITY"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index 19ab1ce50f9..7f42343658d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -144,10 +144,10 @@ protected void renderData(Block html) { TBODY> tbody = html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") .th(".name", "Name").th(".type", "Application Type") - .th(".queue", "Queue").th(".starttime", "StartTime") - .th(".finishtime", "FinishTime").th(".state", "State") - .th(".finalstatus", "FinalStatus").th(".progress", "Progress") - .th(".ui", "Tracking UI")._()._().tbody(); + .th(".queue", "Queue").th(".priority", "Application Priority") + .th(".starttime", "StartTime").th(".finishtime", "FinishTime") + .th(".state", "State").th(".finalstatus", "FinalStatus") + .th(".progress", "Progress").th(".ui", "Tracking UI")._()._().tbody(); StringBuilder appsTableData = new StringBuilder("[\n"); for (ApplicationReport appReport : appReports) { @@ -180,7 +180,9 @@ protected void renderData(Block html) { .append("\",\"") .append( StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app - .getQueue()))).append("\",\"").append(app.getStartedTime()) + .getQueue()))).append("\",\"").append(String + .valueOf(app.getPriority())) + .append("\",\"").append(app.getStartedTime()) .append("\",\"").append(app.getFinishedTime()) .append("\",\"") .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java index df0e39671e4..343641300d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java @@ -21,6 +21,7 @@ import java.util.Set; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Priority; public class ApplicationCreatedEvent extends SystemMetricsEvent { @@ -33,6 +34,7 @@ public class ApplicationCreatedEvent extends private long submittedTime; private Set appTags; private boolean unmanagedApplication; + private Priority applicationPriority; public ApplicationCreatedEvent(ApplicationId appId, String name, @@ -42,7 +44,8 @@ public ApplicationCreatedEvent(ApplicationId appId, long submittedTime, long createdTime, Set appTags, - boolean unmanagedApplication) { + boolean unmanagedApplication, + Priority applicationPriority) { super(SystemMetricsEventType.APP_CREATED, createdTime); this.appId = appId; this.name = name; @@ -52,6 +55,7 @@ public ApplicationCreatedEvent(ApplicationId appId, this.submittedTime = submittedTime; this.appTags = appTags; this.unmanagedApplication = unmanagedApplication; + this.applicationPriority = applicationPriority; } @Override @@ -90,4 +94,8 @@ public Set getAppTags() { public boolean isUnmanagedApp() { return unmanagedApplication; } + + public Priority getApplicationPriority() { + return applicationPriority; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index 07553a73bba..3d7ac9f6f21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -107,7 +107,8 @@ public void appCreated(RMApp app, long createdTime) { app.getQueue(), app.getSubmitTime(), createdTime, app.getApplicationTags(), - app.getApplicationSubmissionContext().getUnmanagedAM())); + app.getApplicationSubmissionContext().getUnmanagedAM(), + app.getApplicationSubmissionContext().getPriority())); } } @@ -257,6 +258,8 @@ private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) { entityInfo.put( ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, event.isUnmanagedApp()); + entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, + event.getApplicationPriority().getPriority()); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 890dfb105d2..eb48cc74f16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -141,6 +141,11 @@ public void testPublishApplicationMetrics() throws Exception { entity.getOtherInfo().get( ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)); + Assert.assertEquals( + app.getApplicationSubmissionContext().getPriority().getPriority(), + entity.getOtherInfo().get( + ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO)); + Assert .assertEquals( app.getUser(), @@ -371,6 +376,7 @@ private static RMApp createRMApp(ApplicationId appId) { when(app.getApplicationTags()).thenReturn(appTags); ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class); when(asc.getUnmanagedAM()).thenReturn(false); + when(asc.getPriority()).thenReturn(Priority.newInstance(0)); when(app.getApplicationSubmissionContext()).thenReturn(asc); return app; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md index bbf3e5936e4..72b813a2574 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md @@ -1088,7 +1088,8 @@ Response Body: "startedTime":1430425001004, "finishedTime":1430425008861, "elapsedTime":7857, - "unmanagedApplication":"false"}, + "unmanagedApplication":"false", + "applicationPriority":0}, { "appId":"application_1430424020775_0003", "currentAppAttemptId":"appattempt_1430424020775_0003_000001", @@ -1108,7 +1109,8 @@ Response Body: "startedTime":1430424956650, "finishedTime":1430424963907, "elapsedTime":7257, - "unmanagedApplication":"false"}, + "unmanagedApplication":"false", + "applicationPriority":0}, { "appId":"application_1430424020775_0002", "currentAppAttemptId":"appattempt_1430424020775_0002_000001", @@ -1128,7 +1130,8 @@ Response Body: "startedTime":1430424769395, "finishedTime":1430424776594, "elapsedTime":7199, - "unmanagedApplication":"false"}, + "unmanagedApplication":"false", + "applicationPriority":0}, { "appId":"application_1430424020775_0001", "currentAppAttemptId":"appattempt_1430424020775_0001_000001", @@ -1149,7 +1152,8 @@ Response Body: "finishedTime":1430424776594, "elapsedTime":18344, "applicationTags":"mrapplication,ta-example", - "unmanagedApplication":"false" + "unmanagedApplication":"false", + "applicationPriority":0 } ] } @@ -1192,6 +1196,7 @@ Response Body: 1430425008861 7857 false + 0 application_1430424020775_0003 @@ -1213,6 +1218,7 @@ Response Body: 1430424963907 7257 false + 0 application_1430424020775_0002 @@ -1234,6 +1240,7 @@ Response Body: 1430424776594 7199 false + 0 application_1430424020775_0001 @@ -1256,6 +1263,7 @@ Response Body: 18344 mrapplication,ta-example false + 0 @@ -1307,6 +1315,7 @@ None | `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared | | `applicationTags` | string | The application tags. | | `unmanagedApplication` | boolean | Is the application unmanaged. | +| `applicationPriority` | int | Priority of the submitted application. | ### Response Examples: @@ -1345,6 +1354,7 @@ Response Body: "elapsedTime": 18344, "applicationTags": mrapplication,tag-example, "unmanagedApplication":"false" + "applicationPriority":0 } #### XML response @@ -1384,6 +1394,7 @@ Response Body: 18344 mrapplication,ta-example false + 0 ## Application Attempt List