YARN-3717. Expose app/am/queue's node-label-expression to RM web UI / CLI / REST-API. (Naganarasimha G R via wangda)

This commit is contained in:
Wangda Tan 2015-09-15 11:40:50 -07:00
parent b2017d9b03
commit ae5308fe1d
29 changed files with 412 additions and 63 deletions

View File

@ -436,6 +436,9 @@ Release 2.8.0 - UNRELEASED
YARN-2005. Blacklisting support for scheduling AMs. (Anubhav Dhoot via kasha) YARN-2005. Blacklisting support for scheduling AMs. (Anubhav Dhoot via kasha)
YARN-3717. Expose app/am/queue's node-label-expression to RM web UI /
CLI / REST-API. (Naganarasimha G R via wangda)
OPTIMIZATIONS OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not YARN-3339. TestDockerContainerExecutor should pull a single image and not

View File

@ -91,8 +91,9 @@ public abstract class ApplicationReport {
YarnApplicationState state, String diagnostics, String url, YarnApplicationState state, String diagnostics, String url,
long startTime, long finishTime, FinalApplicationStatus finalStatus, long startTime, long finishTime, FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl, ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken, float progress, String applicationType, Token amRmToken, Set<String> tags,
Set<String> tags, boolean unmanagedApplication, Priority priority) { boolean unmanagedApplication, Priority priority,
String appNodeLabelExpression, String amNodeLabelExpression) {
ApplicationReport report = ApplicationReport report =
newInstance(applicationId, applicationAttemptId, user, queue, name, newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
@ -101,6 +102,8 @@ public abstract class ApplicationReport {
report.setApplicationTags(tags); report.setApplicationTags(tags);
report.setUnmanagedApp(unmanagedApplication); report.setUnmanagedApp(unmanagedApplication);
report.setPriority(priority); report.setPriority(priority);
report.setAppNodeLabelExpression(appNodeLabelExpression);
report.setAmNodeLabelExpression(amNodeLabelExpression);
return report; return report;
} }
@ -422,4 +425,26 @@ public abstract class ApplicationReport {
@Private @Private
@Unstable @Unstable
public abstract void setPriority(Priority priority); public abstract void setPriority(Priority priority);
/**
* Get the default Node Label expression for all the application's containers
*
* @return Application's NodeLabelExpression
*/
@Unstable
public abstract String getAppNodeLabelExpression();
@Unstable
public abstract void setAppNodeLabelExpression(String appNodeLabelExpression);
/**
* Get the default Node Label expression for all the application's containers
*
* @return Application's NodeLabelExpression
*/
@Unstable
public abstract String getAmNodeLabelExpression();
@Unstable
public abstract void setAmNodeLabelExpression(String amNodeLabelExpression);
} }

View File

@ -27,6 +27,22 @@ import org.apache.hadoop.yarn.util.Records;
@Public @Public
@Unstable @Unstable
public abstract class NodeLabel implements Comparable<NodeLabel> { public abstract class NodeLabel implements Comparable<NodeLabel> {
/**
* Default node label partition.
*/
@Private
@Unstable
public static final String DEFAULT_NODE_LABEL_PARTITION =
"<DEFAULT_PARTITION>";
/**
* Node Label expression not set .
*/
@Private
@Unstable
public static final String NODE_LABEL_EXPRESSION_NOT_SET = "<Not set>";
/** /**
* By default, node label is exclusive or not * By default, node label is exclusive or not
*/ */

View File

@ -197,6 +197,8 @@ message ApplicationReportProto {
optional LogAggregationStatusProto log_aggregation_status = 21; optional LogAggregationStatusProto log_aggregation_status = 21;
optional bool unmanaged_application = 22 [default = false]; optional bool unmanaged_application = 22 [default = false];
optional PriorityProto priority = 23; optional PriorityProto priority = 23;
optional string appNodeLabelExpression = 24;
optional string amNodeLabelExpression = 25;
} }
enum LogAggregationStatusProto { enum LogAggregationStatusProto {

View File

@ -554,7 +554,11 @@ public class ApplicationCLI extends YarnCLI {
appReportStr.print("\tDiagnostics : "); appReportStr.print("\tDiagnostics : ");
appReportStr.println(appReport.getDiagnostics()); appReportStr.println(appReport.getDiagnostics());
appReportStr.print("\tUnmanaged Application : "); appReportStr.print("\tUnmanaged Application : ");
appReportStr.print(appReport.isUnmanagedApp()); appReportStr.println(appReport.isUnmanagedApp());
appReportStr.print("\tApplication Node Label Expression : ");
appReportStr.println(appReport.getAppNodeLabelExpression());
appReportStr.print("\tAM container Node Label Expression : ");
appReportStr.print(appReport.getAmNodeLabelExpression());
} else { } else {
appReportStr.print("Application with id '" + applicationId appReportStr.print("Application with id '" + applicationId
+ "' doesn't exist in RM."); + "' doesn't exist in RM.");

View File

@ -32,6 +32,7 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
@ -135,11 +136,11 @@ public class QueueCLI extends YarnCLI {
writer.print("\tMaximum Capacity : "); writer.print("\tMaximum Capacity : ");
writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%"); writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%");
writer.print("\tDefault Node Label expression : "); writer.print("\tDefault Node Label expression : ");
if (null != queueInfo.getDefaultNodeLabelExpression()) { String nodeLabelExpression = queueInfo.getDefaultNodeLabelExpression();
writer.println(queueInfo.getDefaultNodeLabelExpression()); nodeLabelExpression =
} else { (nodeLabelExpression == null || nodeLabelExpression.trim().isEmpty())
writer.println(); ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : nodeLabelExpression;
} writer.println(nodeLabelExpression);
Set<String> nodeLabels = queueInfo.getAccessibleNodeLabels(); Set<String> nodeLabels = queueInfo.getAccessibleNodeLabels();
StringBuilder labelList = new StringBuilder(); StringBuilder labelList = new StringBuilder();

View File

@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Priority;
@ -104,7 +105,7 @@ public class TestYarnCLI {
"user", "queue", "appname", "host", 124, null, "user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
null, null, false, Priority.newInstance(0)); null, null, false, Priority.newInstance(0), "high-mem", "high-mem");
newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
newApplicationReport.setPriority(Priority.newInstance(0)); newApplicationReport.setPriority(Priority.newInstance(0));
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
@ -134,6 +135,8 @@ public class TestYarnCLI {
pw.println("\tLog Aggregation Status : SUCCEEDED"); pw.println("\tLog Aggregation Status : SUCCEEDED");
pw.println("\tDiagnostics : diagnostics"); pw.println("\tDiagnostics : diagnostics");
pw.println("\tUnmanaged Application : false"); pw.println("\tUnmanaged Application : false");
pw.println("\tApplication Node Label Expression : high-mem");
pw.println("\tAM container Node Label Expression : high-mem");
pw.close(); pw.close();
String appReportStr = baos.toString("UTF-8"); String appReportStr = baos.toString("UTF-8");
Assert.assertEquals(appReportStr, sysOutStream.toString()); Assert.assertEquals(appReportStr, sysOutStream.toString());
@ -1311,7 +1314,8 @@ public class TestYarnCLI {
pw.println("\tCapacity : " + "40.0%"); pw.println("\tCapacity : " + "40.0%");
pw.println("\tCurrent Capacity : " + "50.0%"); pw.println("\tCurrent Capacity : " + "50.0%");
pw.println("\tMaximum Capacity : " + "80.0%"); pw.println("\tMaximum Capacity : " + "80.0%");
pw.println("\tDefault Node Label expression : "); pw.println("\tDefault Node Label expression : "
+ NodeLabel.DEFAULT_NODE_LABEL_PARTITION);
pw.println("\tAccessible Node Labels : "); pw.println("\tAccessible Node Labels : ");
pw.close(); pw.close();
String queueInfoStr = baos.toString("UTF-8"); String queueInfoStr = baos.toString("UTF-8");

View File

@ -630,4 +630,42 @@ public class ApplicationReportPBImpl extends ApplicationReport {
builder.clearPriority(); builder.clearPriority();
this.priority = priority; this.priority = priority;
} }
@Override
public String getAppNodeLabelExpression() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasAppNodeLabelExpression()) {
return null;
}
return p.getAppNodeLabelExpression();
}
@Override
public void setAppNodeLabelExpression(String appNodeLabelExpression) {
maybeInitBuilder();
if (appNodeLabelExpression == null) {
builder.clearAppNodeLabelExpression();
return;
}
builder.setAppNodeLabelExpression((appNodeLabelExpression));
}
@Override
public String getAmNodeLabelExpression() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasAmNodeLabelExpression()) {
return null;
}
return p.getAmNodeLabelExpression();
}
@Override
public void setAmNodeLabelExpression(String amNodeLabelExpression) {
maybeInitBuilder();
if (amNodeLabelExpression == null) {
builder.clearAmNodeLabelExpression();
return;
}
builder.setAmNodeLabelExpression((amNodeLabelExpression));
}
} }

View File

@ -60,7 +60,7 @@ public class TestApplicatonReport {
"appname", "host", 124, null, YarnApplicationState.FINISHED, "appname", "host", 124, null, YarnApplicationState.FINISHED,
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
"N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null,
null, false, Priority.newInstance(0)); null, false, Priority.newInstance(0),"","");
return appReport; return appReport;
} }

View File

@ -260,6 +260,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
Set<String> appTags = null; Set<String> appTags = null;
Map<ApplicationAccessType, String> appViewACLs = Map<ApplicationAccessType, String> appViewACLs =
new HashMap<ApplicationAccessType, String>(); new HashMap<ApplicationAccessType, String>();
String appNodeLabelExpression = null;
String amNodeLabelExpression = null;
Map<String, Object> entityInfo = entity.getOtherInfo(); Map<String, Object> entityInfo = entity.getOtherInfo();
if (entityInfo != null) { if (entityInfo != null) {
if (entityInfo.containsKey(ApplicationMetricsConstants.USER_ENTITY_INFO)) { if (entityInfo.containsKey(ApplicationMetricsConstants.USER_ENTITY_INFO)) {
@ -280,8 +282,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
latestApplicationAttemptId, user, queue, name, null, -1, null, latestApplicationAttemptId, user, queue, name, null, -1, null,
state, diagnosticsInfo, null, createdTime, finishedTime, state, diagnosticsInfo, null, createdTime, finishedTime,
finalStatus, null, null, progress, type, null, appTags, finalStatus, null, null, progress, type, null, appTags,
unmanagedApplication, Priority.newInstance(applicationPriority)), unmanagedApplication, Priority.newInstance(applicationPriority),
appViewACLs); appNodeLabelExpression, amNodeLabelExpression), appViewACLs);
} }
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
queue = queue =
@ -298,6 +300,11 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO)
.toString(); .toString();
} }
if (entityInfo.containsKey(ApplicationMetricsConstants.TYPE_ENTITY_INFO)) {
type =
entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO)
.toString();
}
if (entityInfo if (entityInfo
.containsKey(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)) { .containsKey(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)) {
unmanagedApplication = unmanagedApplication =
@ -310,6 +317,18 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
applicationPriority = Integer.parseInt(entityInfo.get( applicationPriority = Integer.parseInt(entityInfo.get(
ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO).toString()); ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO).toString());
} }
if (entityInfo
.containsKey(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION)) {
appNodeLabelExpression = entityInfo
.get(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION).toString();
}
if (entityInfo
.containsKey(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)) {
amNodeLabelExpression =
entityInfo.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)
.toString();
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
long vcoreSeconds=Long.parseLong(entityInfo.get( long vcoreSeconds=Long.parseLong(entityInfo.get(
ApplicationMetricsConstants.APP_CPU_METRICS).toString()); ApplicationMetricsConstants.APP_CPU_METRICS).toString());
@ -381,9 +400,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
ConverterUtils.toApplicationId(entity.getEntityId()), ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state, latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
appResources, null, progress, type, null, appTags, appResources, null, progress, type, null, appTags, unmanagedApplication,
unmanagedApplication, Priority.newInstance(applicationPriority)), Priority.newInstance(applicationPriority), appNodeLabelExpression,
appViewACLs); amNodeLabelExpression), appViewACLs);
} }
private static ApplicationAttemptReport convertToApplicationAttemptReport( private static ApplicationAttemptReport convertToApplicationAttemptReport(

View File

@ -80,4 +80,10 @@ public class ApplicationMetricsConstants {
public static final String APPLICATION_PRIORITY_INFO = public static final String APPLICATION_PRIORITY_INFO =
"YARN_APPLICATION_PRIORITY"; "YARN_APPLICATION_PRIORITY";
public static final String APP_NODE_LABEL_EXPRESSION =
"YARN_APP_NODE_LABEL_EXPRESSION";
public static final String AM_NODE_LABEL_EXPRESSION =
"YARN_AM_NODE_LABEL_EXPRESSION";
} }

View File

@ -208,6 +208,12 @@ public class AppBlock extends HtmlBlock {
overviewTable._("Diagnostics:", overviewTable._("Diagnostics:",
app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo()); app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo());
overviewTable._("Unmanaged Application:", app.isUnmanagedApp()); overviewTable._("Unmanaged Application:", app.isUnmanagedApp());
overviewTable._("Application Node Label expression:",
app.getAppNodeLabelExpression() == null ? "<Not set>"
: app.getAppNodeLabelExpression());
overviewTable._("AM container Node Label expression:",
app.getAmNodeLabelExpression() == null ? "<Not set>"
: app.getAmNodeLabelExpression());
Collection<ApplicationAttemptReport> attempts; Collection<ApplicationAttemptReport> attempts;
try { try {

View File

@ -62,6 +62,8 @@ public class AppInfo {
private int allocatedCpuVcores; private int allocatedCpuVcores;
private int allocatedMemoryMB; private int allocatedMemoryMB;
protected boolean unmanagedApplication; protected boolean unmanagedApplication;
private String appNodeLabelExpression;
private String amNodeLabelExpression;
public AppInfo() { public AppInfo() {
// JAXB needs this // JAXB needs this
@ -106,6 +108,8 @@ public class AppInfo {
this.applicationTags = CSV_JOINER.join(app.getApplicationTags()); this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
} }
unmanagedApplication = app.isUnmanagedApp(); unmanagedApplication = app.isUnmanagedApp();
appNodeLabelExpression = app.getAppNodeLabelExpression();
amNodeLabelExpression = app.getAmNodeLabelExpression();
} }
public String getAppId() { public String getAppId() {
@ -203,4 +207,12 @@ public class AppInfo {
public int getPriority() { public int getPriority() {
return priority; return priority;
} }
public String getAppNodeLabelExpression() {
return appNodeLabelExpression;
}
public String getAmNodeLabelExpression() {
return amNodeLabelExpression;
}
} }

View File

@ -35,6 +35,8 @@ public class ApplicationCreatedEvent extends
private Set<String> appTags; private Set<String> appTags;
private boolean unmanagedApplication; private boolean unmanagedApplication;
private Priority applicationPriority; private Priority applicationPriority;
private String appNodeLabelsExpression;
private String amNodeLabelsExpression;
public ApplicationCreatedEvent(ApplicationId appId, public ApplicationCreatedEvent(ApplicationId appId,
String name, String name,
@ -45,7 +47,9 @@ public class ApplicationCreatedEvent extends
long createdTime, long createdTime,
Set<String> appTags, Set<String> appTags,
boolean unmanagedApplication, boolean unmanagedApplication,
Priority applicationPriority) { Priority applicationPriority,
String appNodeLabelsExpression,
String amNodeLabelsExpression) {
super(SystemMetricsEventType.APP_CREATED, createdTime); super(SystemMetricsEventType.APP_CREATED, createdTime);
this.appId = appId; this.appId = appId;
this.name = name; this.name = name;
@ -56,6 +60,8 @@ public class ApplicationCreatedEvent extends
this.appTags = appTags; this.appTags = appTags;
this.unmanagedApplication = unmanagedApplication; this.unmanagedApplication = unmanagedApplication;
this.applicationPriority = applicationPriority; this.applicationPriority = applicationPriority;
this.appNodeLabelsExpression = appNodeLabelsExpression;
this.amNodeLabelsExpression = amNodeLabelsExpression;
} }
@Override @Override
@ -98,4 +104,12 @@ public class ApplicationCreatedEvent extends
public Priority getApplicationPriority() { public Priority getApplicationPriority() {
return applicationPriority; return applicationPriority;
} }
public String getAppNodeLabelsExpression() {
return appNodeLabelsExpression;
}
public String getAmNodeLabelsExpression() {
return amNodeLabelsExpression;
}
} }

View File

@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@ -98,6 +100,8 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public void appCreated(RMApp app, long createdTime) { public void appCreated(RMApp app, long createdTime) {
if (publishSystemMetrics) { if (publishSystemMetrics) {
ApplicationSubmissionContext appSubmissionContext =
app.getApplicationSubmissionContext();
dispatcher.getEventHandler().handle( dispatcher.getEventHandler().handle(
new ApplicationCreatedEvent( new ApplicationCreatedEvent(
app.getApplicationId(), app.getApplicationId(),
@ -107,8 +111,10 @@ public class SystemMetricsPublisher extends CompositeService {
app.getQueue(), app.getQueue(),
app.getSubmitTime(), app.getSubmitTime(),
createdTime, app.getApplicationTags(), createdTime, app.getApplicationTags(),
app.getApplicationSubmissionContext().getUnmanagedAM(), appSubmissionContext.getUnmanagedAM(),
app.getApplicationSubmissionContext().getPriority())); appSubmissionContext.getPriority(),
app.getAppNodeLabelExpression(),
app.getAmNodeLabelExpression()));
} }
} }
@ -260,6 +266,10 @@ public class SystemMetricsPublisher extends CompositeService {
event.isUnmanagedApp()); event.isUnmanagedApp());
entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
event.getApplicationPriority().getPriority()); event.getApplicationPriority().getPriority());
entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
event.getAppNodeLabelsExpression());
entityInfo.put(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION,
event.getAmNodeLabelsExpression());
entity.setOtherInfo(entityInfo); entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent(); TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType( tEvent.setEventType(

View File

@ -248,4 +248,10 @@ public interface RMApp extends EventHandler<RMAppEvent> {
Map<NodeId, LogAggregationReport> getLogAggregationReportsForApp(); Map<NodeId, LogAggregationReport> getLogAggregationReportsForApp();
LogAggregationStatus getLogAggregationStatusForAppReport(); LogAggregationStatus getLogAggregationStatusForAppReport();
/**
* Return the node label expression of the AM container.
*/
String getAmNodeLabelExpression();
String getAppNodeLabelExpression();
} }

View File

@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
@ -683,6 +684,8 @@ public class RMAppImpl implements RMApp, Recoverable {
this.submissionContext.getPriority()); this.submissionContext.getPriority());
report.setLogAggregationStatus(logAggregationStatus); report.setLogAggregationStatus(logAggregationStatus);
report.setUnmanagedApp(submissionContext.getUnmanagedAM()); report.setUnmanagedApp(submissionContext.getUnmanagedAM());
report.setAppNodeLabelExpression(getAppNodeLabelExpression());
report.setAmNodeLabelExpression(getAmNodeLabelExpression());
return report; return report;
} finally { } finally {
this.readLock.unlock(); this.readLock.unlock();
@ -1700,4 +1703,28 @@ public class RMAppImpl implements RMApp, Recoverable {
this.readLock.unlock(); this.readLock.unlock();
} }
} }
@Override
public String getAppNodeLabelExpression() {
String appNodeLabelExpression =
getApplicationSubmissionContext().getNodeLabelExpression();
appNodeLabelExpression = (appNodeLabelExpression == null)
? NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET : appNodeLabelExpression;
appNodeLabelExpression = (appNodeLabelExpression.trim().isEmpty())
? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appNodeLabelExpression;
return appNodeLabelExpression;
}
@Override
public String getAmNodeLabelExpression() {
String amNodeLabelExpression = null;
if (!getApplicationSubmissionContext().getUnmanagedAM()) {
amNodeLabelExpression = getAMResourceRequest().getNodeLabelExpression();
amNodeLabelExpression = (amNodeLabelExpression == null)
? NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET : amNodeLabelExpression;
amNodeLabelExpression = (amNodeLabelExpression.trim().isEmpty())
? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : amNodeLabelExpression;
}
return amNodeLabelExpression;
}
} }

View File

@ -27,6 +27,7 @@ import java.util.Map;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; import org.apache.hadoop.yarn.nodelabels.RMNodeLabel;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
@ -91,7 +92,8 @@ class CapacitySchedulerPage extends RmView {
} }
private void renderLeafQueueInfoWithPartition(Block html) { private void renderLeafQueueInfoWithPartition(Block html) {
nodeLabel = nodeLabel.length() == 0 ? "<DEFAULT_PARTITION>" : nodeLabel; nodeLabel = nodeLabel.length() == 0
? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : nodeLabel;
// first display the queue's label specific details : // first display the queue's label specific details :
ResponseInfo ri = ResponseInfo ri =
info("\'" + lqinfo.getQueuePath().substring(5) info("\'" + lqinfo.getQueuePath().substring(5)
@ -152,7 +154,11 @@ class CapacitySchedulerPage extends RmView {
"%.1f", lqinfo.getUserLimitFactor())). "%.1f", lqinfo.getUserLimitFactor())).
_("Accessible Node Labels:", StringUtils.join(",", lqinfo.getNodeLabels())). _("Accessible Node Labels:", StringUtils.join(",", lqinfo.getNodeLabels())).
_("Ordering Policy: ", lqinfo.getOrderingPolicyInfo()). _("Ordering Policy: ", lqinfo.getOrderingPolicyInfo()).
_("Preemption:", lqinfo.getPreemptionDisabled() ? "disabled" : "enabled"); _("Preemption:", lqinfo.getPreemptionDisabled() ? "disabled" : "enabled").
_("Default Node Label Expression:",
lqinfo.getDefaultNodeLabelExpression() == null
? NodeLabel.DEFAULT_NODE_LABEL_PARTITION
: lqinfo.getDefaultNodeLabelExpression());
} }
} }
@ -363,9 +369,8 @@ class CapacitySchedulerPage extends RmView {
csqinfo.csinfo = sinfo; csqinfo.csinfo = sinfo;
csqinfo.qinfo = null; csqinfo.qinfo = null;
csqinfo.label = label.getLabelName(); csqinfo.label = label.getLabelName();
String nodeLabel = String nodeLabel = csqinfo.label.length() == 0
csqinfo.label.length() == 0 ? "<DEFAULT_PARTITION>" ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : csqinfo.label;
: csqinfo.label;
QueueCapacities queueCapacities = root.getQueueCapacities(); QueueCapacities queueCapacities = root.getQueueCapacities();
used = queueCapacities.getUsedCapacity(label.getLabelName()); used = queueCapacities.getUsedCapacity(label.getLabelName());
String partitionUiTag = String partitionUiTag =

View File

@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; import org.apache.hadoop.yarn.nodelabels.RMNodeLabel;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
@ -58,9 +59,8 @@ public class NodeLabelsPage extends RmView {
RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager(); RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager();
for (RMNodeLabel info : nlm.pullRMNodeLabelsInfo()) { for (RMNodeLabel info : nlm.pullRMNodeLabelsInfo()) {
TR<TBODY<TABLE<Hamlet>>> row = TR<TBODY<TABLE<Hamlet>>> row =
tbody.tr().td( tbody.tr().td(info.getLabelName().isEmpty()
info.getLabelName().isEmpty() ? "<DEFAULT_PARTITION>" : info ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : info.getLabelName());
.getLabelName());
String type = String type =
(info.getIsExclusive()) ? "Exclusive Partition" (info.getIsExclusive()) ? "Exclusive Partition"
: "Non Exclusive Partition"; : "Non Exclusive Partition";

View File

@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
@ -98,6 +99,8 @@ public class AppInfo {
protected LogAggregationStatus logAggregationStatus; protected LogAggregationStatus logAggregationStatus;
protected boolean unmanagedApplication; protected boolean unmanagedApplication;
protected String appNodeLabelExpression;
protected String amNodeLabelExpression;
public AppInfo() { public AppInfo() {
} // JAXB needs this } // JAXB needs this
@ -132,8 +135,10 @@ public class AppInfo {
this.name = app.getName().toString(); this.name = app.getName().toString();
this.queue = app.getQueue().toString(); this.queue = app.getQueue().toString();
this.priority = 0; this.priority = 0;
if (app.getApplicationSubmissionContext().getPriority() != null) { ApplicationSubmissionContext appSubmissionContext =
this.priority = app.getApplicationSubmissionContext().getPriority() app.getApplicationSubmissionContext();
if (appSubmissionContext.getPriority() != null) {
this.priority = appSubmissionContext.getPriority()
.getPriority(); .getPriority();
} }
this.progress = app.getProgress() * 100; this.progress = app.getProgress() * 100;
@ -191,7 +196,11 @@ public class AppInfo {
memorySeconds = appMetrics.getMemorySeconds(); memorySeconds = appMetrics.getMemorySeconds();
vcoreSeconds = appMetrics.getVcoreSeconds(); vcoreSeconds = appMetrics.getVcoreSeconds();
unmanagedApplication = unmanagedApplication =
app.getApplicationSubmissionContext().getUnmanagedAM(); appSubmissionContext.getUnmanagedAM();
appNodeLabelExpression =
app.getApplicationSubmissionContext().getNodeLabelExpression();
amNodeLabelExpression = (unmanagedApplication) ? null
: app.getAMResourceRequest().getNodeLabelExpression();
} }
} }
@ -338,4 +347,12 @@ public class AppInfo {
public int getPriority() { public int getPriority() {
return this.priority; return this.priority;
} }
public String getAppNodeLabelExpression() {
return this.appNodeLabelExpression;
}
public String getAmNodeLabelExpression() {
return this.amNodeLabelExpression;
}
} }

View File

@ -40,6 +40,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected ResourceInfo usedAMResource; protected ResourceInfo usedAMResource;
protected ResourceInfo userAMResourceLimit; protected ResourceInfo userAMResourceLimit;
protected boolean preemptionDisabled; protected boolean preemptionDisabled;
protected String defaultNodeLabelExpression;
@XmlTransient @XmlTransient
protected String orderingPolicyInfo; protected String orderingPolicyInfo;
@ -62,6 +63,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit()); userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit());
preemptionDisabled = q.getPreemptionDisabled(); preemptionDisabled = q.getPreemptionDisabled();
orderingPolicyInfo = q.getOrderingPolicy().getInfo(); orderingPolicyInfo = q.getOrderingPolicy().getInfo();
defaultNodeLabelExpression = q.getDefaultNodeLabelExpression();
} }
public int getNumActiveApplications() { public int getNumActiveApplications() {
@ -116,4 +118,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public String getOrderingPolicyInfo() { public String getOrderingPolicyInfo() {
return orderingPolicyInfo; return orderingPolicyInfo;
} }
public String getDefaultNodeLabelExpression() {
return defaultNodeLabelExpression;
}
} }

View File

@ -300,6 +300,33 @@ public class TestClientRMService {
report.getApplicationResourceUsageReport(); report.getApplicationResourceUsageReport();
Assert.assertEquals(10, usageReport.getMemorySeconds()); Assert.assertEquals(10, usageReport.getMemorySeconds());
Assert.assertEquals(3, usageReport.getVcoreSeconds()); Assert.assertEquals(3, usageReport.getVcoreSeconds());
Assert.assertEquals("<Not set>", report.getAmNodeLabelExpression());
Assert.assertEquals("<Not set>", report.getAppNodeLabelExpression());
// if application has am node label set to blank
ApplicationId appId2 = getApplicationId(2);
when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
ApplicationAccessType.VIEW_APP, null, appId2)).thenReturn(true);
request.setApplicationId(appId2);
response = rmService.getApplicationReport(request);
report = response.getApplicationReport();
Assert.assertEquals(NodeLabel.DEFAULT_NODE_LABEL_PARTITION,
report.getAmNodeLabelExpression());
Assert.assertEquals(NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET,
report.getAppNodeLabelExpression());
// if application has am node label set to blank
ApplicationId appId3 = getApplicationId(3);
when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
ApplicationAccessType.VIEW_APP, null, appId3)).thenReturn(true);
request.setApplicationId(appId3);
response = rmService.getApplicationReport(request);
report = response.getApplicationReport();
Assert.assertEquals("high-mem", report.getAmNodeLabelExpression());
Assert.assertEquals("high-mem", report.getAppNodeLabelExpression());
// if application id is null // if application id is null
GetApplicationReportRequest invalidRequest = recordFactory GetApplicationReportRequest invalidRequest = recordFactory
@ -951,11 +978,11 @@ public class TestClientRMService {
ApplicationId applicationId3 = getApplicationId(3); ApplicationId applicationId3 = getApplicationId(3);
YarnConfiguration config = new YarnConfiguration(); YarnConfiguration config = new YarnConfiguration();
apps.put(applicationId1, getRMApp(rmContext, yarnScheduler, applicationId1, apps.put(applicationId1, getRMApp(rmContext, yarnScheduler, applicationId1,
config, "testqueue", 10, 3)); config, "testqueue", 10, 3,null,null));
apps.put(applicationId2, getRMApp(rmContext, yarnScheduler, applicationId2, apps.put(applicationId2, getRMApp(rmContext, yarnScheduler, applicationId2,
config, "a", 20, 2)); config, "a", 20, 2,null,""));
apps.put(applicationId3, getRMApp(rmContext, yarnScheduler, applicationId3, apps.put(applicationId3, getRMApp(rmContext, yarnScheduler, applicationId3,
config, "testqueue", 40, 5)); config, "testqueue", 40, 5,"high-mem","high-mem"));
return apps; return apps;
} }
@ -978,10 +1005,11 @@ public class TestClientRMService {
private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler,
ApplicationId applicationId3, YarnConfiguration config, String queueName, ApplicationId applicationId3, YarnConfiguration config, String queueName,
final long memorySeconds, final long vcoreSeconds) { final long memorySeconds, final long vcoreSeconds,
String appNodeLabelExpression, String amNodeLabelExpression) {
ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class); ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class);
when(asContext.getMaxAppAttempts()).thenReturn(1); when(asContext.getMaxAppAttempts()).thenReturn(1);
when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression);
RMAppImpl app = RMAppImpl app =
spy(new RMAppImpl(applicationId3, rmContext, config, null, null, spy(new RMAppImpl(applicationId3, rmContext, config, null, null,
queueName, asContext, yarnScheduler, null, queueName, asContext, yarnScheduler, null,
@ -1002,7 +1030,7 @@ public class TestClientRMService {
return report; return report;
} }
}); });
app.getAMResourceRequest().setNodeLabelExpression(amNodeLabelExpression);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
ApplicationId.newInstance(123456, 1), 1); ApplicationId.newInstance(123456, 1), 1);
RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId, RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId,

View File

@ -202,6 +202,16 @@ public abstract class MockAsm extends MockApps {
public LogAggregationStatus getLogAggregationStatusForAppReport() { public LogAggregationStatus getLogAggregationStatusForAppReport() {
throw new UnsupportedOperationException("Not supported yet."); throw new UnsupportedOperationException("Not supported yet.");
} }
@Override
public String getAmNodeLabelExpression() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public String getAppNodeLabelExpression() {
throw new UnsupportedOperationException("Not supported yet.");
}
} }
public static RMApp newApplication(int i) { public static RMApp newApplication(int i) {

View File

@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
@ -47,6 +48,7 @@ import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@ -146,6 +148,14 @@ public class TestSystemMetricsPublisher {
entity.getOtherInfo().get( entity.getOtherInfo().get(
ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO)); ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO));
Assert.assertEquals(app.getAmNodeLabelExpression(), entity.getOtherInfo()
.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION));
Assert.assertEquals(
app.getApplicationSubmissionContext().getNodeLabelExpression(),
entity.getOtherInfo()
.get(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION));
Assert Assert
.assertEquals( .assertEquals(
app.getUser(), app.getUser(),
@ -351,7 +361,7 @@ public class TestSystemMetricsPublisher {
} }
private static RMApp createRMApp(ApplicationId appId) { private static RMApp createRMApp(ApplicationId appId) {
RMApp app = mock(RMApp.class); RMApp app = mock(RMAppImpl.class);
when(app.getApplicationId()).thenReturn(appId); when(app.getApplicationId()).thenReturn(appId);
when(app.getName()).thenReturn("test app"); when(app.getName()).thenReturn("test app");
when(app.getApplicationType()).thenReturn("test app type"); when(app.getApplicationType()).thenReturn("test app type");
@ -376,8 +386,14 @@ public class TestSystemMetricsPublisher {
when(app.getApplicationTags()).thenReturn(appTags); when(app.getApplicationTags()).thenReturn(appTags);
ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class); ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class);
when(asc.getUnmanagedAM()).thenReturn(false); when(asc.getUnmanagedAM()).thenReturn(false);
when(asc.getPriority()).thenReturn(Priority.newInstance(0)); when(asc.getPriority()).thenReturn(Priority.newInstance(10));
when(asc.getNodeLabelExpression()).thenReturn("high-cpu");
when(app.getApplicationSubmissionContext()).thenReturn(asc); when(app.getApplicationSubmissionContext()).thenReturn(asc);
when(app.getAppNodeLabelExpression()).thenCallRealMethod();
ResourceRequest amReq = mock(ResourceRequest.class);
when(amReq.getNodeLabelExpression()).thenReturn("high-mem");
when(app.getAMResourceRequest()).thenReturn(amReq);
when(app.getAmNodeLabelExpression()).thenCallRealMethod();
return app; return app;
} }

View File

@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest;
@ -64,6 +65,8 @@ public class MockRMApp implements RMApp {
finish = time; finish = time;
id = MockApps.newAppID(newid); id = MockApps.newAppID(newid);
state = newState; state = newState;
amReq = ResourceRequest.newInstance(Priority.UNDEFINED, "0.0.0.0",
Resource.newInstance(0, 0), 1);
} }
public MockRMApp(int newid, long time, RMAppState newState, String userName) { public MockRMApp(int newid, long time, RMAppState newState, String userName) {
@ -283,4 +286,14 @@ public class MockRMApp implements RMApp {
public LogAggregationStatus getLogAggregationStatusForAppReport() { public LogAggregationStatus getLogAggregationStatusForAppReport() {
return null; return null;
} }
@Override
public String getAmNodeLabelExpression() {
return null;
}
@Override
public String getAppNodeLabelExpression() {
return null;
}
} }

View File

@ -261,10 +261,9 @@ public class TestRMAppTransitions {
// but applicationId is still set for safety // but applicationId is still set for safety
submissionContext.setApplicationId(applicationId); submissionContext.setApplicationId(applicationId);
RMApp application = RMApp application = new RMAppImpl(applicationId, rmContext, conf, name,
new RMAppImpl(applicationId, rmContext, conf, name, user, queue, user, queue, submissionContext, scheduler, masterService,
submissionContext, scheduler, masterService, System.currentTimeMillis(), "YARN", null, mock(ResourceRequest.class));
System.currentTimeMillis(), "YARN", null, null);
testAppStartState(applicationId, user, name, queue, application); testAppStartState(applicationId, user, name, queue, application);
this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),

View File

@ -1310,14 +1310,29 @@ public class TestRMWebServicesApps extends JerseyTestBase {
WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"), WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"),
WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted"), WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted"),
WebServicesTestUtils.getXmlString(element, "logAggregationStatus"), WebServicesTestUtils.getXmlString(element, "logAggregationStatus"),
WebServicesTestUtils.getXmlBoolean(element, "unmanagedApplication")); WebServicesTestUtils.getXmlBoolean(element, "unmanagedApplication"),
WebServicesTestUtils.getXmlString(element, "appNodeLabelExpression"),
WebServicesTestUtils.getXmlString(element, "amNodeLabelExpression"));
} }
} }
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception { Exception {
assertEquals("incorrect number of elements", 30, info.length()); int expectedNumberOfElements = 30;
String appNodeLabelExpression = null;
String amNodeLabelExpression = null;
if (app.getApplicationSubmissionContext()
.getNodeLabelExpression() != null) {
expectedNumberOfElements++;
appNodeLabelExpression = info.getString("appNodeLabelExpression");
}
if (app.getAMResourceRequest().getNodeLabelExpression() != null) {
expectedNumberOfElements++;
amNodeLabelExpression = info.getString("amNodeLabelExpression");
}
assertEquals("incorrect number of elements", expectedNumberOfElements,
info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
info.getString("name"), info.getString("applicationType"), info.getString("name"), info.getString("applicationType"),
@ -1334,7 +1349,9 @@ public class TestRMWebServicesApps extends JerseyTestBase {
info.getInt("numNonAMContainerPreempted"), info.getInt("numNonAMContainerPreempted"),
info.getInt("numAMContainerPreempted"), info.getInt("numAMContainerPreempted"),
info.getString("logAggregationStatus"), info.getString("logAggregationStatus"),
info.getBoolean("unmanagedApplication")); info.getBoolean("unmanagedApplication"),
appNodeLabelExpression,
amNodeLabelExpression);
} }
public void verifyAppInfoGeneric(RMApp app, String id, String user, public void verifyAppInfoGeneric(RMApp app, String id, String user,
@ -1345,7 +1362,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
int allocatedMB, int allocatedVCores, int numContainers, int allocatedMB, int allocatedVCores, int numContainers,
int preemptedResourceMB, int preemptedResourceVCores, int preemptedResourceMB, int preemptedResourceVCores,
int numNonAMContainerPreempted, int numAMContainerPreempted, int numNonAMContainerPreempted, int numAMContainerPreempted,
String logAggregationStatus, boolean unmanagedApplication) String logAggregationStatus, boolean unmanagedApplication,
String appNodeLabelExpression, String amNodeLabelExpression)
throws JSONException, throws JSONException,
Exception { Exception {
@ -1400,6 +1418,12 @@ public class TestRMWebServicesApps extends JerseyTestBase {
assertEquals("unmanagedApplication doesn't match", app assertEquals("unmanagedApplication doesn't match", app
.getApplicationSubmissionContext().getUnmanagedAM(), .getApplicationSubmissionContext().getUnmanagedAM(),
unmanagedApplication); unmanagedApplication);
assertEquals("unmanagedApplication doesn't match",
app.getApplicationSubmissionContext().getNodeLabelExpression(),
appNodeLabelExpression);
assertEquals("unmanagedApplication doesn't match",
app.getAMResourceRequest().getNodeLabelExpression(),
amNodeLabelExpression);
} }
@Test @Test

View File

@ -1379,11 +1379,13 @@ Response Body:
"allocatedVCores" : 0, "allocatedVCores" : 0,
"runningContainers" : 0, "runningContainers" : 0,
"applicationType" : "MAPREDUCE", "applicationType" : "MAPREDUCE",
"applicationTags" : "" "applicationTags" : "",
"memorySeconds" : 151730, "memorySeconds" : 151730,
"vcoreSeconds" : 103, "vcoreSeconds" : 103,
"unmanagedApplication":"false" "unmanagedApplication" : "false",
"applicationPriority":0 "applicationPriority" : 0,
"appNodeLabelExpression" : "",
"amnodeLabelExpression" : ""
}, },
{ {
"finishedTime" : 1326815789546, "finishedTime" : 1326815789546,
@ -1406,11 +1408,13 @@ Response Body:
"allocatedVCores" : 0, "allocatedVCores" : 0,
"runningContainers" : 1, "runningContainers" : 1,
"applicationType" : "YARN", "applicationType" : "YARN",
"applicationTags" : "tag1" "applicationTags" : "tag1",
"memorySeconds" : 640064, "memorySeconds" : 640064,
"vcoreSeconds" : 442 "vcoreSeconds" : 442,
"unmanagedApplication":"false" "unmanagedApplication" : "false",
"applicationPriority":0 "applicationPriority" : 0,
"appNodeLabelExpression" : "",
"amNodeLabelExpression" : ""
} }
] ]
} }
@ -1462,6 +1466,8 @@ Response Body:
<vcoreSeconds>103</vcoreSeconds> <vcoreSeconds>103</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication> <unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority> <applicationPriority>0</applicationPriority>
<appNodeLabelExpression></appNodeLabelExpression>
<amNodeLabelExpression></amNodeLabelExpression>
</app> </app>
<app> <app>
<id>application_1326815542473_0002</id> <id>application_1326815542473_0002</id>
@ -1489,6 +1495,8 @@ Response Body:
<vcoreSeconds>442</vcoreSeconds> <vcoreSeconds>442</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication> <unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority> <applicationPriority>0</applicationPriority>
<appNodeLabelExpression></appNodeLabelExpression>
<amNodeLabelExpression></amNodeLabelExpression>
</app> </app>
</apps> </apps>
``` ```
@ -1650,6 +1658,8 @@ Note that depending on security settings a user might not be able to see all the
| vcoreSeconds | long | The amount of CPU resources the application has allocated (virtual core-seconds) | | vcoreSeconds | long | The amount of CPU resources the application has allocated (virtual core-seconds) |
| unmanagedApplication | boolean | Is the application unmanaged. | | unmanagedApplication | boolean | Is the application unmanaged. |
| applicationPriority | int | priority of the submitted application | | applicationPriority | int | priority of the submitted application |
| appNodeLabelExpression | string | Node Label expression which is used to identify the nodes on which application's containers are expected to run by default.|
| amNodeLabelExpression | string | Node Label expression which is used to identify the node on which application's AM container is expected to run.|
### Response Examples ### Response Examples
@ -1690,8 +1700,10 @@ Response Body:
"queue" : "a1", "queue" : "a1",
"memorySeconds" : 151730, "memorySeconds" : 151730,
"vcoreSeconds" : 103, "vcoreSeconds" : 103,
"unmanagedApplication":"false" "unmanagedApplication" : "false",
"applicationPriority":0 "applicationPriority" : 0,
"appNodeLabelExpression" : "",
"amNodeLabelExpression" : ""
} }
} }
``` ```
@ -1735,6 +1747,8 @@ Response Body:
<vcoreSeconds>103</vcoreSeconds> <vcoreSeconds>103</vcoreSeconds>
<unmanagedApplication>false</unmanagedApplication> <unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority> <applicationPriority>0</applicationPriority>
<appNodeLabelExpression></appNodeLabelExpression>
<amNodeLabelExpression></amNodeLabelExpression>
</app> </app>
``` ```

View File

@ -1089,7 +1089,10 @@ Response Body:
"finishedTime":1430425008861, "finishedTime":1430425008861,
"elapsedTime":7857, "elapsedTime":7857,
"unmanagedApplication":"false", "unmanagedApplication":"false",
"applicationPriority":0}, "applicationPriority":0,
"appNodeLabelExpression":"",
"amNodeLabelExpression":""
},
{ {
"appId":"application_1430424020775_0003", "appId":"application_1430424020775_0003",
"currentAppAttemptId":"appattempt_1430424020775_0003_000001", "currentAppAttemptId":"appattempt_1430424020775_0003_000001",
@ -1110,7 +1113,10 @@ Response Body:
"finishedTime":1430424963907, "finishedTime":1430424963907,
"elapsedTime":7257, "elapsedTime":7257,
"unmanagedApplication":"false", "unmanagedApplication":"false",
"applicationPriority":0}, "applicationPriority":0,
"appNodeLabelExpression":"",
"amNodeLabelExpression":""
},
{ {
"appId":"application_1430424020775_0002", "appId":"application_1430424020775_0002",
"currentAppAttemptId":"appattempt_1430424020775_0002_000001", "currentAppAttemptId":"appattempt_1430424020775_0002_000001",
@ -1131,7 +1137,10 @@ Response Body:
"finishedTime":1430424776594, "finishedTime":1430424776594,
"elapsedTime":7199, "elapsedTime":7199,
"unmanagedApplication":"false", "unmanagedApplication":"false",
"applicationPriority":0}, "applicationPriority":0,
"appNodeLabelExpression":"",
"amNodeLabelExpression":""
},
{ {
"appId":"application_1430424020775_0001", "appId":"application_1430424020775_0001",
"currentAppAttemptId":"appattempt_1430424020775_0001_000001", "currentAppAttemptId":"appattempt_1430424020775_0001_000001",
@ -1153,7 +1162,9 @@ Response Body:
"elapsedTime":18344, "elapsedTime":18344,
"applicationTags":"mrapplication,ta-example", "applicationTags":"mrapplication,ta-example",
"unmanagedApplication":"false", "unmanagedApplication":"false",
"applicationPriority":0 "applicationPriority":0,
"appNodeLabelExpression":"",
"amNodeLabelExpression":""
} }
] ]
} }
@ -1197,6 +1208,8 @@ Response Body:
<elapsedTime>7857</elapsedTime> <elapsedTime>7857</elapsedTime>
<unmanagedApplication>false</unmanagedApplication> <unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority> <applicationPriority>0</applicationPriority>
<appNodeLabelExpression></appNodeLabelExpression>
<amNodeLabelExpression></amNodeLabelExpression>
</app> </app>
<app> <app>
<appId>application_1430424020775_0003</appId> <appId>application_1430424020775_0003</appId>
@ -1219,6 +1232,8 @@ Response Body:
<elapsedTime>7257</elapsedTime> <elapsedTime>7257</elapsedTime>
<unmanagedApplication>false</unmanagedApplication> <unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority> <applicationPriority>0</applicationPriority>
<appNodeLabelExpression></appNodeLabelExpression>
<amNodeLabelExpression></amNodeLabelExpression>
</app> </app>
<app> <app>
<appId>application_1430424020775_0002</appId> <appId>application_1430424020775_0002</appId>
@ -1241,6 +1256,8 @@ Response Body:
<elapsedTime>7199</elapsedTime> <elapsedTime>7199</elapsedTime>
<unmanagedApplication>false</unmanagedApplication> <unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority> <applicationPriority>0</applicationPriority>
<appNodeLabelExpression></appNodeLabelExpression>
<amNodeLabelExpression></amNodeLabelExpression>
</app> </app>
<app> <app>
<appId>application_1430424020775_0001</appId> <appId>application_1430424020775_0001</appId>
@ -1264,6 +1281,8 @@ Response Body:
<applicationTags>mrapplication,ta-example</applicationTags> <applicationTags>mrapplication,ta-example</applicationTags>
<unmanagedApplication>false</unmanagedApplication> <unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority> <applicationPriority>0</applicationPriority>
<appNodeLabelExpression></appNodeLabelExpression>
<amNodeLabelExpression></amNodeLabelExpression>
</app> </app>
</apps> </apps>
@ -1316,7 +1335,8 @@ None
| `applicationTags` | string | The application tags. | | `applicationTags` | string | The application tags. |
| `unmanagedApplication` | boolean | Is the application unmanaged. | | `unmanagedApplication` | boolean | Is the application unmanaged. |
| `applicationPriority` | int | Priority of the submitted application. | | `applicationPriority` | int | Priority of the submitted application. |
| `appNodeLabelExpression` | string |Node Label expression which is used to identify the nodes on which application's containers are expected to run by default.|
| `amNodeLabelExpression` | string | Node Label expression which is used to identify the node on which application's AM container is expected to run.|
### Response Examples: ### Response Examples:
#### JSON response #### JSON response
@ -1353,8 +1373,10 @@ Response Body:
"finishedTime": 1430424072153, "finishedTime": 1430424072153,
"elapsedTime": 18344, "elapsedTime": 18344,
"applicationTags": mrapplication,tag-example, "applicationTags": mrapplication,tag-example,
"unmanagedApplication":"false" "unmanagedApplication": "false",
"applicationPriority":0 "applicationPriority": 0,
"appNodeLabelExpression": "",
"amNodeLabelExpression": ""
} }
#### XML response #### XML response
@ -1395,6 +1417,8 @@ Response Body:
<applicationTags>mrapplication,ta-example</applicationTags> <applicationTags>mrapplication,ta-example</applicationTags>
<unmanagedApplication>false</unmanagedApplication> <unmanagedApplication>false</unmanagedApplication>
<applicationPriority>0</applicationPriority> <applicationPriority>0</applicationPriority>
<appNodeLabelExpression><appNodeLabelExpression>
<amNodeLabelExpression><amNodeLabelExpression>
</app> </app>
## <a name="REST_API_APPLICATION_ATTEMPT_LIST"></a>Application Attempt List ## <a name="REST_API_APPLICATION_ATTEMPT_LIST"></a>Application Attempt List