Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-3948. Display Application Priority in RM Web UI.(Sunil G via rohithsharmaks)

This commit is contained in:
Rohith Sharma K S 2015-08-07 10:43:41 +05:30
parent 6d4eee718a
commit b6265d39c5
15 changed files with 116 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -159,6 +159,8 @@ Release 2.8.0 - UNRELEASED
YARN-3736. Add RMStateStore apis to store and load accepted reservations for
failover (adhoot via asuresh)
YARN-3948. Display Application Priority in RM Web UI.(Sunil G via rohithsharmaks)
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
View File

@ -408,4 +408,17 @@ public abstract class ApplicationReport {
@Public
@Unstable
public abstract void setUnmanagedApp(boolean unmanagedApplication);
/**
* Get priority of the application
*
* @return Application's priority
*/
@Public
@Stable
public abstract Priority getPriority();
@Private
@Unstable
public abstract void setPriority(Priority priority);
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -196,6 +196,7 @@ message ApplicationReportProto {
repeated string applicationTags = 20;
optional LogAggregationStatusProto log_aggregation_status = 21;
optional bool unmanaged_application = 22 [default = false];
optional PriorityProto priority = 23;
}
enum LogAggregationStatusProto {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
View File

@ -501,6 +501,8 @@ public class ApplicationCLI extends YarnCLI {
appReportStr.println(appReport.getUser());
appReportStr.print("\tQueue : ");
appReportStr.println(appReport.getQueue());
appReportStr.print("\tApplication Priority : ");
appReportStr.println(appReport.getPriority());
appReportStr.print("\tStart-Time : ");
appReportStr.println(appReport.getStartTime());
appReportStr.print("\tFinish-Time : ");

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -106,6 +106,7 @@ public class TestYarnCLI {
FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
null, null, false);
newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
newApplicationReport.setPriority(Priority.newInstance(0));
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
@ -119,6 +120,7 @@ public class TestYarnCLI {
pw.println("\tApplication-Type : YARN");
pw.println("\tUser : user");
pw.println("\tQueue : queue");
pw.println("\tApplication Priority : 0");
pw.println("\tStart-Time : 0");
pw.println("\tFinish-Time : 0");
pw.println("\tProgress : 53.79%");

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
View File

@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto;
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import com.google.protobuf.TextFormat;
@ -55,6 +57,7 @@ public class ApplicationReportPBImpl extends ApplicationReport {
private Token clientToAMToken = null;
private Token amRmToken = null;
private Set<String> applicationTags = null;
private Priority priority = null;
public ApplicationReportPBImpl() {
builder = ApplicationReportProto.newBuilder();
@ -484,6 +487,11 @@ public class ApplicationReportPBImpl extends ApplicationReport {
builder.clearApplicationTags();
builder.addAllApplicationTags(this.applicationTags);
}
if (this.priority != null
&& !((PriorityPBImpl) this.priority).getProto().equals(
builder.getPriority())) {
builder.setPriority(convertToProtoFormat(this.priority));
}
}
private void mergeLocalToProto() {
@ -551,6 +559,14 @@ public class ApplicationReportPBImpl extends ApplicationReport {
return ((TokenPBImpl)t).getProto();
}
private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
return new PriorityPBImpl(p);
}
private PriorityProto convertToProtoFormat(Priority t) {
return ((PriorityPBImpl)t).getProto();
}
@Override
public LogAggregationStatus getLogAggregationStatus() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
@ -593,4 +609,25 @@ public class ApplicationReportPBImpl extends ApplicationReport {
maybeInitBuilder();
builder.setUnmanagedApplication(unmanagedApplication);
}
@Override
public Priority getPriority() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
if (this.priority != null) {
return this.priority;
}
if (!p.hasPriority()) {
return null;
}
this.priority = convertFromProtoFormat(p.getPriority());
return this.priority;
}
@Override
public void setPriority(Priority priority) {
maybeInitBuilder();
if (priority == null)
builder.clearPriority();
this.priority = priority;
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
View File

@ -324,7 +324,8 @@ public class BuilderUtils {
String url, long startTime, long finishTime,
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String appType, Token amRmToken, Set<String> tags) {
float progress, String appType, Token amRmToken, Set<String> tags,
Priority priority) {
ApplicationReport report = recordFactory
.newRecordInstance(ApplicationReport.class);
report.setApplicationId(applicationId);
@ -347,6 +348,7 @@ public class BuilderUtils {
report.setApplicationType(appType);
report.setAMRMToken(amRmToken);
report.setApplicationTags(tags);
report.setPriority(priority);
return report;
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
View File

@ -167,6 +167,7 @@ public class AppBlock extends HtmlBlock {
._("Application Type:", app.getType())
._("Application Tags:",
app.getApplicationTags() == null ? "" : app.getApplicationTags())
._("Application Priority:", clarifyAppPriority(app.getPriority()))
._(
"YarnApplicationState:",
app.getAppState() == null ? UNAVAILABLE : clarifyAppState(app
@ -342,6 +343,10 @@ public class AppBlock extends HtmlBlock {
}
}
private String clarifyAppPriority(int priority) {
return priority + " (Higher Integer value indicates higher priority)";
}
private String clairfyAppFinalStatus(FinalApplicationStatus status) {
if (status == FinalApplicationStatus.UNDEFINED) {
return "Application has not completed yet.";

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
View File

@ -52,13 +52,13 @@ public class WebPageUtils {
.append("{'sType':'string', 'aTargets': [0]")
.append(", 'mRender': parseHadoopID }")
.append("\n, {'sType':'numeric', 'aTargets': " +
(isFairSchedulerPage ? "[6, 7]": "[5, 6]"))
(isFairSchedulerPage ? "[6, 7]": "[6, 7]"))
.append(", 'mRender': renderHadoopDate }")
.append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':");
if (isFairSchedulerPage) {
sb.append("[13]");
} else if (isResourceManager) {
sb.append("[12]");
sb.append("[13]");
} else {
sb.append("[9]");
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
View File

@ -58,6 +58,7 @@ public class AppInfo {
protected long finishedTime;
protected long elapsedTime;
protected String applicationTags;
protected int priority;
private int allocatedCpuVcores;
private int allocatedMemoryMB;
protected boolean unmanagedApplication;
@ -86,6 +87,10 @@ public class AppInfo {
finishedTime = app.getFinishTime();
elapsedTime = Times.elapsed(startedTime, finishedTime);
finalAppStatus = app.getFinalApplicationStatus();
priority = 0;
if (app.getPriority() != null) {
priority = app.getPriority().getPriority();
}
if (app.getApplicationResourceUsageReport() != null) {
runningContainers = app.getApplicationResourceUsageReport()
.getNumUsedContainers();
@ -194,4 +199,8 @@ public class AppInfo {
public boolean isUnmanagedApp() {
return unmanagedApplication;
}
public int getPriority() {
return priority;
}
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
View File

@ -660,10 +660,10 @@ public class RMAppImpl implements RMApp, Recoverable {
ApplicationReport report = BuilderUtils.newApplicationReport(
this.applicationId, currentApplicationAttemptId, this.user,
this.queue, this.name, host, rpcPort, clientToAMToken,
createApplicationState(), diags,
trackingUrl, this.startTime, this.finishTime, finishState,
appUsageReport, origTrackingUrl, progress, this.applicationType,
amrmToken, applicationTags);
createApplicationState(), diags, trackingUrl, this.startTime,
this.finishTime, finishState, appUsageReport, origTrackingUrl,
progress, this.applicationType, amrmToken, applicationTags,
this.submissionContext.getPriority());
report.setLogAggregationStatus(logAggregationStatus);
report.setUnmanagedApp(submissionContext.getUnmanagedAM());
return report;

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
View File

@ -56,7 +56,8 @@ public class RMAppsBlock extends AppsBlock {
TBODY<TABLE<Hamlet>> tbody =
html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
.th(".name", "Name").th(".type", "Application Type")
.th(".queue", "Queue").th(".starttime", "StartTime")
.th(".queue", "Queue").th(".priority", "Application Priority")
.th(".starttime", "StartTime")
.th(".finishtime", "FinishTime").th(".state", "State")
.th(".finalstatus", "FinalStatus")
.th(".runningcontainer", "Running Containers")
@ -106,7 +107,9 @@ public class RMAppsBlock extends AppsBlock {
.append("\",\"")
.append(
StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
.getQueue()))).append("\",\"").append(app.getStartedTime())
.getQueue()))).append("\",\"").append(String
.valueOf(app.getPriority()))
.append("\",\"").append(app.getStartedTime())
.append("\",\"").append(app.getFinishedTime())
.append("\",\"")
.append(app.getAppState() == null ? UNAVAILABLE : app.getAppState())

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -74,7 +74,8 @@ public class AppInfo {
protected long clusterId;
protected String applicationType;
protected String applicationTags = "";
protected int priority;
// these are only allowed if acls allow
protected long startedTime;
protected long finishedTime;
@ -130,6 +131,11 @@ public class AppInfo {
this.user = app.getUser().toString();
this.name = app.getName().toString();
this.queue = app.getQueue().toString();
this.priority = 0;
if (app.getApplicationSubmissionContext().getPriority() != null) {
this.priority = app.getApplicationSubmissionContext().getPriority()
.getPriority();
}
this.progress = app.getProgress() * 100;
this.diagnostics = app.getDiagnostics().toString();
if (diagnostics == null || diagnostics.isEmpty()) {
@ -328,4 +334,8 @@ public class AppInfo {
public boolean isUnmanagedApp() {
return unmanagedApplication;
}
public int getPriority() {
return this.priority;
}
}

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -1290,6 +1290,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
WebServicesTestUtils.getXmlString(element, "name"),
WebServicesTestUtils.getXmlString(element, "applicationType"),
WebServicesTestUtils.getXmlString(element, "queue"),
WebServicesTestUtils.getXmlInt(element, "priority"),
WebServicesTestUtils.getXmlString(element, "state"),
WebServicesTestUtils.getXmlString(element, "finalStatus"),
WebServicesTestUtils.getXmlFloat(element, "progress"),
@ -1316,18 +1317,18 @@ public class TestRMWebServicesApps extends JerseyTestBase {
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
assertEquals("incorrect number of elements", 29, info.length());
assertEquals("incorrect number of elements", 30, info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
info.getString("name"), info.getString("applicationType"),
info.getString("queue"), info.getString("state"),
info.getString("finalStatus"), (float) info.getDouble("progress"),
info.getString("trackingUI"), info.getString("diagnostics"),
info.getLong("clusterId"), info.getLong("startedTime"),
info.getLong("finishedTime"), info.getLong("elapsedTime"),
info.getString("amHostHttpAddress"), info.getString("amContainerLogs"),
info.getInt("allocatedMB"), info.getInt("allocatedVCores"),
info.getInt("runningContainers"),
info.getString("queue"), info.getInt("priority"),
info.getString("state"), info.getString("finalStatus"),
(float) info.getDouble("progress"), info.getString("trackingUI"),
info.getString("diagnostics"), info.getLong("clusterId"),
info.getLong("startedTime"), info.getLong("finishedTime"),
info.getLong("elapsedTime"), info.getString("amHostHttpAddress"),
info.getString("amContainerLogs"), info.getInt("allocatedMB"),
info.getInt("allocatedVCores"), info.getInt("runningContainers"),
info.getInt("preemptedResourceMB"),
info.getInt("preemptedResourceVCores"),
info.getInt("numNonAMContainerPreempted"),
@ -1337,8 +1338,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
}
public void verifyAppInfoGeneric(RMApp app, String id, String user,
String name, String applicationType, String queue, String state,
String finalStatus, float progress, String trackingUI,
String name, String applicationType, String queue, int prioirty,
String state, String finalStatus, float progress, String trackingUI,
String diagnostics, long clusterId, long startedTime, long finishedTime,
long elapsedTime, String amHostHttpAddress, String amContainerLogs,
int allocatedMB, int allocatedVCores, int numContainers,
@ -1355,6 +1356,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
WebServicesTestUtils.checkStringMatch("applicationType",
app.getApplicationType(), applicationType);
WebServicesTestUtils.checkStringMatch("queue", app.getQueue(), queue);
assertEquals("priority doesn't match", 0, prioirty);
WebServicesTestUtils.checkStringMatch("state", app.getState().toString(),
state);
WebServicesTestUtils.checkStringMatch("finalStatus", app

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
View File

@ -1382,6 +1382,7 @@ Response Body:
"memorySeconds" : 151730,
"vcoreSeconds" : 103,
"unmanagedApplication":"false"
"applicationPriority":0
},
{
"finishedTime" : 1326815789546,
@ -1408,6 +1409,7 @@ Response Body:
"memorySeconds" : 640064,
"vcoreSeconds" : 442
"unmanagedApplication":"false"
"applicationPriority":0
}
]
}
@ -1458,6 +1460,7 @@ Response Body:
<memorySeconds>151730</memorySeconds>
<vcoreSeconds>103</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority>
</app>
<app>
<id>application_1326815542473_0002</id>
@ -1484,6 +1487,7 @@ Response Body:
<memorySeconds>640064</memorySeconds>
<vcoreSeconds>442</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority>
</app>
</apps>
```
@ -1644,6 +1648,7 @@ Note that depending on security settings a user might not be able to see all the
| memorySeconds | long | The amount of memory the application has allocated (megabyte-seconds) |
| vcoreSeconds | long | The amount of CPU resources the application has allocated (virtual core-seconds) |
| unmanagedApplication | boolean | Is the application unmanaged. |
| applicationPriority | int | priority of the submitted application |
### Response Examples
@ -1685,6 +1690,7 @@ Response Body:
"memorySeconds" : 151730,
"vcoreSeconds" : 103,
"unmanagedApplication":"false"
"applicationPriority":0
}
}
```
@ -1727,6 +1733,7 @@ Response Body:
<memorySeconds>151730</memorySeconds>
<vcoreSeconds>103</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority>
</app>
```