Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-4023. Publish Application Priority to TimelineServer. (Sunil G via rohithsharmaks) 

(cherry picked from commit 1c12adb71f)
This commit is contained in:
rohithsharmaks 2015-08-12 14:45:41 +05:30
parent d53296281a
commit c5b20e0db6
12 changed files with 69 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -112,6 +112,9 @@ Release 2.8.0 - UNRELEASED
YARN-3887. Support changing Application priority during runtime. (Sunil G
via jianhe)
YARN-4023. Publish Application Priority to TimelineServer. (Sunil G
via rohithsharmaks)
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
View File

@ -92,7 +92,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
long startTime, long finishTime, FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken,
Set<String> tags, boolean unmanagedApplication) {
Set<String> tags, boolean unmanagedApplication, Priority priority) {
ApplicationReport report =
newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
@ -100,6 +100,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
applicationType, amRmToken);
report.setApplicationTags(tags);
report.setUnmanagedApp(unmanagedApplication);
report.setPriority(priority);
return report;
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -104,7 +104,7 @@ public void testGetApplicationReport() throws Exception {
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
null, null, false);
null, null, false, Priority.newInstance(0));
newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
newApplicationReport.setPriority(Priority.newInstance(0));
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
View File

@ -22,6 +22,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.Assert;
@ -59,7 +60,7 @@ protected static ApplicationReport createApplicationReport(
"appname", "host", 124, null, YarnApplicationState.FINISHED,
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
"N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null,
null,false);
null, false, Priority.newInstance(0));
return appReport;
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
View File

@ -251,6 +251,7 @@ private static ApplicationReportExt convertToApplicationReport(
long createdTime = 0;
long finishedTime = 0;
float progress = 0.0f;
int applicationPriority = 0;
ApplicationAttemptId latestApplicationAttemptId = null;
String diagnosticsInfo = null;
FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
@ -276,10 +277,11 @@ private static ApplicationReportExt convertToApplicationReport(
if (field == ApplicationReportField.USER_AND_ACLS) {
return new ApplicationReportExt(ApplicationReport.newInstance(
ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
null, progress, type, null, appTags,
unmanagedApplication), appViewACLs);
latestApplicationAttemptId, user, queue, name, null, -1, null,
state, diagnosticsInfo, null, createdTime, finishedTime,
finalStatus, null, null, progress, type, null, appTags,
unmanagedApplication, Priority.newInstance(applicationPriority)),
appViewACLs);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
queue =
@ -303,6 +305,11 @@ private static ApplicationReportExt convertToApplicationReport(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)
.toString());
}
if (entityInfo
.containsKey(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO)) {
applicationPriority = Integer.parseInt(entityInfo.get(
ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO).toString());
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
long vcoreSeconds=Long.parseLong(entityInfo.get(
ApplicationMetricsConstants.APP_CPU_METRICS).toString());
@ -373,8 +380,10 @@ private static ApplicationReportExt convertToApplicationReport(
return new ApplicationReportExt(ApplicationReport.newInstance(
ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
null, progress, type, null, appTags, unmanagedApplication), appViewACLs);
diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
appResources, null, progress, type, null, appTags,
unmanagedApplication, Priority.newInstance(applicationPriority)),
appViewACLs);
}
private static ApplicationAttemptReport convertToApplicationAttemptReport(

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
View File

@ -477,6 +477,8 @@ private static TimelineEntity createApplicationTimelineEntity(
entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue");
entityInfo.put(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false");
entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
Priority.newInstance(0));
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
Integer.MAX_VALUE + 1L);
entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123);

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
View File

@ -77,4 +77,7 @@ public class ApplicationMetricsConstants {
public static final String UNMANAGED_APPLICATION_ENTITY_INFO =
"YARN_APPLICATION_UNMANAGED_APPLICATION";
public static final String APPLICATION_PRIORITY_INFO =
"YARN_APPLICATION_PRIORITY";
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
View File

@ -144,10 +144,10 @@ protected void renderData(Block html) {
TBODY<TABLE<Hamlet>> tbody =
html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
.th(".name", "Name").th(".type", "Application Type")
.th(".queue", "Queue").th(".starttime", "StartTime")
.th(".finishtime", "FinishTime").th(".state", "State")
.th(".finalstatus", "FinalStatus").th(".progress", "Progress")
.th(".ui", "Tracking UI")._()._().tbody();
.th(".queue", "Queue").th(".priority", "Application Priority")
.th(".starttime", "StartTime").th(".finishtime", "FinishTime")
.th(".state", "State").th(".finalstatus", "FinalStatus")
.th(".progress", "Progress").th(".ui", "Tracking UI")._()._().tbody();
StringBuilder appsTableData = new StringBuilder("[\n");
for (ApplicationReport appReport : appReports) {
@ -180,7 +180,9 @@ protected void renderData(Block html) {
.append("\",\"")
.append(
StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
.getQueue()))).append("\",\"").append(app.getStartedTime())
.getQueue()))).append("\",\"").append(String
.valueOf(app.getPriority()))
.append("\",\"").append(app.getStartedTime())
.append("\",\"").append(app.getFinishedTime())
.append("\",\"")
.append(app.getAppState() == null ? UNAVAILABLE : app.getAppState())

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
View File

@ -21,6 +21,7 @@
import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Priority;
public class ApplicationCreatedEvent extends
SystemMetricsEvent {
@ -33,6 +34,7 @@ public class ApplicationCreatedEvent extends
private long submittedTime;
private Set<String> appTags;
private boolean unmanagedApplication;
private Priority applicationPriority;
public ApplicationCreatedEvent(ApplicationId appId,
String name,
@ -42,7 +44,8 @@ public ApplicationCreatedEvent(ApplicationId appId,
long submittedTime,
long createdTime,
Set<String> appTags,
boolean unmanagedApplication) {
boolean unmanagedApplication,
Priority applicationPriority) {
super(SystemMetricsEventType.APP_CREATED, createdTime);
this.appId = appId;
this.name = name;
@ -52,6 +55,7 @@ public ApplicationCreatedEvent(ApplicationId appId,
this.submittedTime = submittedTime;
this.appTags = appTags;
this.unmanagedApplication = unmanagedApplication;
this.applicationPriority = applicationPriority;
}
@Override
@ -90,4 +94,8 @@ public Set<String> getAppTags() {
public boolean isUnmanagedApp() {
return unmanagedApplication;
}
public Priority getApplicationPriority() {
return applicationPriority;
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
View File

@ -107,7 +107,8 @@ public void appCreated(RMApp app, long createdTime) {
app.getQueue(),
app.getSubmitTime(),
createdTime, app.getApplicationTags(),
app.getApplicationSubmissionContext().getUnmanagedAM()));
app.getApplicationSubmissionContext().getUnmanagedAM(),
app.getApplicationSubmissionContext().getPriority()));
}
}
@ -257,6 +258,8 @@ private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
entityInfo.put(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO,
event.isUnmanagedApp());
entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
event.getApplicationPriority().getPriority());
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
View File

@ -141,6 +141,11 @@ public void testPublishApplicationMetrics() throws Exception {
entity.getOtherInfo().get(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO));
Assert.assertEquals(
app.getApplicationSubmissionContext().getPriority().getPriority(),
entity.getOtherInfo().get(
ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO));
Assert
.assertEquals(
app.getUser(),
@ -371,6 +376,7 @@ private static RMApp createRMApp(ApplicationId appId) {
when(app.getApplicationTags()).thenReturn(appTags);
ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class);
when(asc.getUnmanagedAM()).thenReturn(false);
when(asc.getPriority()).thenReturn(Priority.newInstance(0));
when(app.getApplicationSubmissionContext()).thenReturn(asc);
return app;
}

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
View File

@ -1088,7 +1088,8 @@ Response Body:
"startedTime":1430425001004,
"finishedTime":1430425008861,
"elapsedTime":7857,
"unmanagedApplication":"false"},
"unmanagedApplication":"false",
"applicationPriority":0},
{
"appId":"application_1430424020775_0003",
"currentAppAttemptId":"appattempt_1430424020775_0003_000001",
@ -1108,7 +1109,8 @@ Response Body:
"startedTime":1430424956650,
"finishedTime":1430424963907,
"elapsedTime":7257,
"unmanagedApplication":"false"},
"unmanagedApplication":"false",
"applicationPriority":0},
{
"appId":"application_1430424020775_0002",
"currentAppAttemptId":"appattempt_1430424020775_0002_000001",
@ -1128,7 +1130,8 @@ Response Body:
"startedTime":1430424769395,
"finishedTime":1430424776594,
"elapsedTime":7199,
"unmanagedApplication":"false"},
"unmanagedApplication":"false",
"applicationPriority":0},
{
"appId":"application_1430424020775_0001",
"currentAppAttemptId":"appattempt_1430424020775_0001_000001",
@ -1149,7 +1152,8 @@ Response Body:
"finishedTime":1430424776594,
"elapsedTime":18344,
"applicationTags":"mrapplication,ta-example",
"unmanagedApplication":"false"
"unmanagedApplication":"false",
"applicationPriority":0
}
]
}
@ -1192,6 +1196,7 @@ Response Body:
<finishedTime>1430425008861</finishedTime>
<elapsedTime>7857</elapsedTime>
<unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority>
</app>
<app>
<appId>application_1430424020775_0003</appId>
@ -1213,6 +1218,7 @@ Response Body:
<finishedTime>1430424963907</finishedTime>
<elapsedTime>7257</elapsedTime>
<unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority>
</app>
<app>
<appId>application_1430424020775_0002</appId>
@ -1234,6 +1240,7 @@ Response Body:
<finishedTime>1430424776594</finishedTime>
<elapsedTime>7199</elapsedTime>
<unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority>
</app>
<app>
<appId>application_1430424020775_0001</appId>
@ -1256,6 +1263,7 @@ Response Body:
<elapsedTime>18344</elapsedTime>
<applicationTags>mrapplication,ta-example</applicationTags>
<unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority>
</app>
</apps>
@ -1307,6 +1315,7 @@ None
| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared |
| `applicationTags` | string | The application tags. |
| `unmanagedApplication` | boolean | Is the application unmanaged. |
| `applicationPriority` | int | Priority of the submitted application. |
### Response Examples:
@ -1345,6 +1354,7 @@ Response Body:
"elapsedTime": 18344,
"applicationTags": mrapplication,tag-example,
"unmanagedApplication":"false"
"applicationPriority":0
}
#### XML response
@ -1384,6 +1394,7 @@ Response Body:
<elapsedTime>18344</elapsedTime>
<applicationTags>mrapplication,ta-example</applicationTags>
<unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority>
</app>
## <a name="REST_API_APPLICATION_ATTEMPT_LIST"></a>Application Attempt List