From ae5308fe1d08479da0f3929cc6a57816411e9121 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Tue, 15 Sep 2015 11:40:50 -0700 Subject: [PATCH] YARN-3717. Expose app/am/queue's node-label-expression to RM web UI / CLI / REST-API. (Naganarasimha G R via wangda) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../yarn/api/records/ApplicationReport.java | 29 +++++++++++++- .../hadoop/yarn/api/records/NodeLabel.java | 16 ++++++++ .../src/main/proto/yarn_protos.proto | 2 + .../yarn/client/cli/ApplicationCLI.java | 6 ++- .../hadoop/yarn/client/cli/QueueCLI.java | 11 ++--- .../hadoop/yarn/client/cli/TestYarnCLI.java | 8 +++- .../impl/pb/ApplicationReportPBImpl.java | 38 ++++++++++++++++++ .../hadoop/yarn/api/TestApplicatonReport.java | 2 +- ...licationHistoryManagerOnTimelineStore.java | 29 +++++++++++--- .../metrics/ApplicationMetricsConstants.java | 6 +++ .../hadoop/yarn/server/webapp/AppBlock.java | 6 +++ .../yarn/server/webapp/dao/AppInfo.java | 12 ++++++ .../metrics/ApplicationCreatedEvent.java | 16 +++++++- .../metrics/SystemMetricsPublisher.java | 14 ++++++- .../server/resourcemanager/rmapp/RMApp.java | 6 +++ .../resourcemanager/rmapp/RMAppImpl.java | 27 +++++++++++++ .../webapp/CapacitySchedulerPage.java | 15 ++++--- .../webapp/NodeLabelsPage.java | 6 +-- .../resourcemanager/webapp/dao/AppInfo.java | 23 +++++++++-- .../dao/CapacitySchedulerLeafQueueInfo.java | 8 +++- .../resourcemanager/TestClientRMService.java | 40 ++++++++++++++++--- .../applicationsmanager/MockAsm.java | 10 +++++ .../metrics/TestSystemMetricsPublisher.java | 20 +++++++++- .../resourcemanager/rmapp/MockRMApp.java | 13 ++++++ .../rmapp/TestRMAppTransitions.java | 7 ++-- .../webapp/TestRMWebServicesApps.java | 32 +++++++++++++-- .../src/site/markdown/ResourceManagerRest.md | 32 ++++++++++----- .../src/site/markdown/TimelineServer.md | 38 ++++++++++++++---- 29 files changed, 412 insertions(+), 63 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c2bee706fec..b4c5c5e5914 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -436,6 +436,9 @@ Release 2.8.0 - UNRELEASED YARN-2005. Blacklisting support for scheduling AMs. (Anubhav Dhoot via kasha) + YARN-3717. Expose app/am/queue's node-label-expression to RM web UI / + CLI / REST-API. (Naganarasimha G R via wangda) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 5de78587b97..33116a49f32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -91,8 +91,9 @@ public abstract class ApplicationReport { YarnApplicationState state, String diagnostics, String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, - float progress, String applicationType, Token amRmToken, - Set tags, boolean unmanagedApplication, Priority priority) { + float progress, String applicationType, Token amRmToken, Set tags, + boolean unmanagedApplication, Priority priority, + String appNodeLabelExpression, String amNodeLabelExpression) { ApplicationReport report = newInstance(applicationId, applicationAttemptId, user, queue, name, host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, @@ -101,6 +102,8 @@ public abstract class ApplicationReport { report.setApplicationTags(tags); report.setUnmanagedApp(unmanagedApplication); report.setPriority(priority); + report.setAppNodeLabelExpression(appNodeLabelExpression); + report.setAmNodeLabelExpression(amNodeLabelExpression); return report; } @@ -422,4 +425,26 @@ public abstract class ApplicationReport { @Private @Unstable public abstract void setPriority(Priority priority); + + /** + * Get the default Node Label expression for all the application's containers + * + * @return Application's NodeLabelExpression + */ + @Unstable + public abstract String getAppNodeLabelExpression(); + + @Unstable + public abstract void setAppNodeLabelExpression(String appNodeLabelExpression); + + /** + * Get the default Node Label expression for all the application's containers + * + * @return Application's NodeLabelExpression + */ + @Unstable + public abstract String getAmNodeLabelExpression(); + + @Unstable + public abstract void setAmNodeLabelExpression(String amNodeLabelExpression); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java index aba6962690a..af914d7c0f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java @@ -27,6 +27,22 @@ import org.apache.hadoop.yarn.util.Records; @Public @Unstable public abstract class NodeLabel implements Comparable { + + /** + * Default node label partition. + */ + @Private + @Unstable + public static final String DEFAULT_NODE_LABEL_PARTITION = + ""; + + /** + * Node Label expression not set . + */ + @Private + @Unstable + public static final String NODE_LABEL_EXPRESSION_NOT_SET = ""; + /** * By default, node label is exclusive or not */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 1bd3ddae35d..0bccfc41725 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -197,6 +197,8 @@ message ApplicationReportProto { optional LogAggregationStatusProto log_aggregation_status = 21; optional bool unmanaged_application = 22 [default = false]; optional PriorityProto priority = 23; + optional string appNodeLabelExpression = 24; + optional string amNodeLabelExpression = 25; } enum LogAggregationStatusProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 9c87eaefed7..55692f1b129 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -554,7 +554,11 @@ public class ApplicationCLI extends YarnCLI { appReportStr.print("\tDiagnostics : "); appReportStr.println(appReport.getDiagnostics()); appReportStr.print("\tUnmanaged Application : "); - appReportStr.print(appReport.isUnmanagedApp()); + appReportStr.println(appReport.isUnmanagedApp()); + appReportStr.print("\tApplication Node Label Expression : "); + appReportStr.println(appReport.getAppNodeLabelExpression()); + appReportStr.print("\tAM container Node Label Expression : "); + appReportStr.print(appReport.getAmNodeLabelExpression()); } else { appReportStr.print("Application with id '" + applicationId + "' doesn't exist in RM."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java index 8a5521dc4b6..b5db536b6fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java @@ -32,6 +32,7 @@ import org.apache.commons.cli.Options; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -135,11 +136,11 @@ public class QueueCLI extends YarnCLI { writer.print("\tMaximum Capacity : "); writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%"); writer.print("\tDefault Node Label expression : "); - if (null != queueInfo.getDefaultNodeLabelExpression()) { - writer.println(queueInfo.getDefaultNodeLabelExpression()); - } else { - writer.println(); - } + String nodeLabelExpression = queueInfo.getDefaultNodeLabelExpression(); + nodeLabelExpression = + (nodeLabelExpression == null || nodeLabelExpression.trim().isEmpty()) + ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : nodeLabelExpression; + writer.println(nodeLabelExpression); Set nodeLabels = queueInfo.getAccessibleNodeLabels(); StringBuilder labelList = new StringBuilder(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index f942a4da5d6..de50467fe59 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; @@ -104,7 +105,7 @@ public class TestYarnCLI { "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", - null, null, false, Priority.newInstance(0)); + null, null, false, Priority.newInstance(0), "high-mem", "high-mem"); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); newApplicationReport.setPriority(Priority.newInstance(0)); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( @@ -134,6 +135,8 @@ public class TestYarnCLI { pw.println("\tLog Aggregation Status : SUCCEEDED"); pw.println("\tDiagnostics : diagnostics"); pw.println("\tUnmanaged Application : false"); + pw.println("\tApplication Node Label Expression : high-mem"); + pw.println("\tAM container Node Label Expression : high-mem"); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); @@ -1311,7 +1314,8 @@ public class TestYarnCLI { pw.println("\tCapacity : " + "40.0%"); pw.println("\tCurrent Capacity : " + "50.0%"); pw.println("\tMaximum Capacity : " + "80.0%"); - pw.println("\tDefault Node Label expression : "); + pw.println("\tDefault Node Label expression : " + + NodeLabel.DEFAULT_NODE_LABEL_PARTITION); pw.println("\tAccessible Node Labels : "); pw.close(); String queueInfoStr = baos.toString("UTF-8"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 2e50e0dd251..1072815afae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -630,4 +630,42 @@ public class ApplicationReportPBImpl extends ApplicationReport { builder.clearPriority(); this.priority = priority; } + + @Override + public String getAppNodeLabelExpression() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasAppNodeLabelExpression()) { + return null; + } + return p.getAppNodeLabelExpression(); + } + + @Override + public void setAppNodeLabelExpression(String appNodeLabelExpression) { + maybeInitBuilder(); + if (appNodeLabelExpression == null) { + builder.clearAppNodeLabelExpression(); + return; + } + builder.setAppNodeLabelExpression((appNodeLabelExpression)); + } + + @Override + public String getAmNodeLabelExpression() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasAmNodeLabelExpression()) { + return null; + } + return p.getAmNodeLabelExpression(); + } + + @Override + public void setAmNodeLabelExpression(String amNodeLabelExpression) { + maybeInitBuilder(); + if (amNodeLabelExpression == null) { + builder.clearAmNodeLabelExpression(); + return; + } + builder.setAmNodeLabelExpression((amNodeLabelExpression)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index d0d1d407b03..46fc4d58d92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -60,7 +60,7 @@ public class TestApplicatonReport { "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, - null, false, Priority.newInstance(0)); + null, false, Priority.newInstance(0),"",""); return appReport; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index f02e83c7855..7dac7163cd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -260,6 +260,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService Set appTags = null; Map appViewACLs = new HashMap(); + String appNodeLabelExpression = null; + String amNodeLabelExpression = null; Map entityInfo = entity.getOtherInfo(); if (entityInfo != null) { if (entityInfo.containsKey(ApplicationMetricsConstants.USER_ENTITY_INFO)) { @@ -280,8 +282,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, null, progress, type, null, appTags, - unmanagedApplication, Priority.newInstance(applicationPriority)), - appViewACLs); + unmanagedApplication, Priority.newInstance(applicationPriority), + appNodeLabelExpression, amNodeLabelExpression), appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -298,6 +300,11 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) .toString(); } + if (entityInfo.containsKey(ApplicationMetricsConstants.TYPE_ENTITY_INFO)) { + type = + entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) + .toString(); + } if (entityInfo .containsKey(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)) { unmanagedApplication = @@ -310,6 +317,18 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService applicationPriority = Integer.parseInt(entityInfo.get( ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO).toString()); } + if (entityInfo + .containsKey(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION)) { + appNodeLabelExpression = entityInfo + .get(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION).toString(); + } + if (entityInfo + .containsKey(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)) { + amNodeLabelExpression = + entityInfo.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION) + .toString(); + } + if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { long vcoreSeconds=Long.parseLong(entityInfo.get( ApplicationMetricsConstants.APP_CPU_METRICS).toString()); @@ -381,9 +400,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, - appResources, null, progress, type, null, appTags, - unmanagedApplication, Priority.newInstance(applicationPriority)), - appViewACLs); + appResources, null, progress, type, null, appTags, unmanagedApplication, + Priority.newInstance(applicationPriority), appNodeLabelExpression, + amNodeLabelExpression), appViewACLs); } private static ApplicationAttemptReport convertToApplicationAttemptReport( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index 481b84c2b54..3cbcc1e330d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -80,4 +80,10 @@ public class ApplicationMetricsConstants { public static final String APPLICATION_PRIORITY_INFO = "YARN_APPLICATION_PRIORITY"; + + public static final String APP_NODE_LABEL_EXPRESSION = + "YARN_APP_NODE_LABEL_EXPRESSION"; + + public static final String AM_NODE_LABEL_EXPRESSION = + "YARN_AM_NODE_LABEL_EXPRESSION"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 31a2c8a9e32..44ed22345da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -208,6 +208,12 @@ public class AppBlock extends HtmlBlock { overviewTable._("Diagnostics:", app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo()); overviewTable._("Unmanaged Application:", app.isUnmanagedApp()); + overviewTable._("Application Node Label expression:", + app.getAppNodeLabelExpression() == null ? "" + : app.getAppNodeLabelExpression()); + overviewTable._("AM container Node Label expression:", + app.getAmNodeLabelExpression() == null ? "" + : app.getAmNodeLabelExpression()); Collection attempts; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index 7efbcb92175..cad3b2e3ac8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -62,6 +62,8 @@ public class AppInfo { private int allocatedCpuVcores; private int allocatedMemoryMB; protected boolean unmanagedApplication; + private String appNodeLabelExpression; + private String amNodeLabelExpression; public AppInfo() { // JAXB needs this @@ -106,6 +108,8 @@ public class AppInfo { this.applicationTags = CSV_JOINER.join(app.getApplicationTags()); } unmanagedApplication = app.isUnmanagedApp(); + appNodeLabelExpression = app.getAppNodeLabelExpression(); + amNodeLabelExpression = app.getAmNodeLabelExpression(); } public String getAppId() { @@ -203,4 +207,12 @@ public class AppInfo { public int getPriority() { return priority; } + + public String getAppNodeLabelExpression() { + return appNodeLabelExpression; + } + + public String getAmNodeLabelExpression() { + return amNodeLabelExpression; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java index 343641300d7..a684dfc4b46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java @@ -35,6 +35,8 @@ public class ApplicationCreatedEvent extends private Set appTags; private boolean unmanagedApplication; private Priority applicationPriority; + private String appNodeLabelsExpression; + private String amNodeLabelsExpression; public ApplicationCreatedEvent(ApplicationId appId, String name, @@ -45,7 +47,9 @@ public class ApplicationCreatedEvent extends long createdTime, Set appTags, boolean unmanagedApplication, - Priority applicationPriority) { + Priority applicationPriority, + String appNodeLabelsExpression, + String amNodeLabelsExpression) { super(SystemMetricsEventType.APP_CREATED, createdTime); this.appId = appId; this.name = name; @@ -56,6 +60,8 @@ public class ApplicationCreatedEvent extends this.appTags = appTags; this.unmanagedApplication = unmanagedApplication; this.applicationPriority = applicationPriority; + this.appNodeLabelsExpression = appNodeLabelsExpression; + this.amNodeLabelsExpression = amNodeLabelsExpression; } @Override @@ -98,4 +104,12 @@ public class ApplicationCreatedEvent extends public Priority getApplicationPriority() { return applicationPriority; } + + public String getAppNodeLabelsExpression() { + return appNodeLabelsExpression; + } + + public String getAmNodeLabelsExpression() { + return amNodeLabelsExpression; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index 3d7ac9f6f21..0852ff4878b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; @@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -98,6 +100,8 @@ public class SystemMetricsPublisher extends CompositeService { @SuppressWarnings("unchecked") public void appCreated(RMApp app, long createdTime) { if (publishSystemMetrics) { + ApplicationSubmissionContext appSubmissionContext = + app.getApplicationSubmissionContext(); dispatcher.getEventHandler().handle( new ApplicationCreatedEvent( app.getApplicationId(), @@ -107,8 +111,10 @@ public class SystemMetricsPublisher extends CompositeService { app.getQueue(), app.getSubmitTime(), createdTime, app.getApplicationTags(), - app.getApplicationSubmissionContext().getUnmanagedAM(), - app.getApplicationSubmissionContext().getPriority())); + appSubmissionContext.getUnmanagedAM(), + appSubmissionContext.getPriority(), + app.getAppNodeLabelExpression(), + app.getAmNodeLabelExpression())); } } @@ -260,6 +266,10 @@ public class SystemMetricsPublisher extends CompositeService { event.isUnmanagedApp()); entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, event.getApplicationPriority().getPriority()); + entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION, + event.getAppNodeLabelsExpression()); + entityInfo.put(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION, + event.getAmNodeLabelsExpression()); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index be9dfaf91ad..720d863e712 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -248,4 +248,10 @@ public interface RMApp extends EventHandler { Map getLogAggregationReportsForApp(); LogAggregationStatus getLogAggregationStatusForAppReport(); + /** + * Return the node label expression of the AM container. + */ + String getAmNodeLabelExpression(); + + String getAppNodeLabelExpression(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 7cf39b88855..ea9aa7030ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; @@ -683,6 +684,8 @@ public class RMAppImpl implements RMApp, Recoverable { this.submissionContext.getPriority()); report.setLogAggregationStatus(logAggregationStatus); report.setUnmanagedApp(submissionContext.getUnmanagedAM()); + report.setAppNodeLabelExpression(getAppNodeLabelExpression()); + report.setAmNodeLabelExpression(getAmNodeLabelExpression()); return report; } finally { this.readLock.unlock(); @@ -1700,4 +1703,28 @@ public class RMAppImpl implements RMApp, Recoverable { this.readLock.unlock(); } } + + @Override + public String getAppNodeLabelExpression() { + String appNodeLabelExpression = + getApplicationSubmissionContext().getNodeLabelExpression(); + appNodeLabelExpression = (appNodeLabelExpression == null) + ? NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET : appNodeLabelExpression; + appNodeLabelExpression = (appNodeLabelExpression.trim().isEmpty()) + ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appNodeLabelExpression; + return appNodeLabelExpression; + } + + @Override + public String getAmNodeLabelExpression() { + String amNodeLabelExpression = null; + if (!getApplicationSubmissionContext().getUnmanagedAM()) { + amNodeLabelExpression = getAMResourceRequest().getNodeLabelExpression(); + amNodeLabelExpression = (amNodeLabelExpression == null) + ? NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET : amNodeLabelExpression; + amNodeLabelExpression = (amNodeLabelExpression.trim().isEmpty()) + ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : amNodeLabelExpression; + } + return amNodeLabelExpression; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index d8971b7534b..9e27627b659 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -91,7 +92,8 @@ class CapacitySchedulerPage extends RmView { } private void renderLeafQueueInfoWithPartition(Block html) { - nodeLabel = nodeLabel.length() == 0 ? "" : nodeLabel; + nodeLabel = nodeLabel.length() == 0 + ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : nodeLabel; // first display the queue's label specific details : ResponseInfo ri = info("\'" + lqinfo.getQueuePath().substring(5) @@ -152,7 +154,11 @@ class CapacitySchedulerPage extends RmView { "%.1f", lqinfo.getUserLimitFactor())). _("Accessible Node Labels:", StringUtils.join(",", lqinfo.getNodeLabels())). _("Ordering Policy: ", lqinfo.getOrderingPolicyInfo()). - _("Preemption:", lqinfo.getPreemptionDisabled() ? "disabled" : "enabled"); + _("Preemption:", lqinfo.getPreemptionDisabled() ? "disabled" : "enabled"). + _("Default Node Label Expression:", + lqinfo.getDefaultNodeLabelExpression() == null + ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION + : lqinfo.getDefaultNodeLabelExpression()); } } @@ -363,9 +369,8 @@ class CapacitySchedulerPage extends RmView { csqinfo.csinfo = sinfo; csqinfo.qinfo = null; csqinfo.label = label.getLabelName(); - String nodeLabel = - csqinfo.label.length() == 0 ? "" - : csqinfo.label; + String nodeLabel = csqinfo.label.length() == 0 + ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : csqinfo.label; QueueCapacities queueCapacities = root.getQueueCapacities(); used = queueCapacities.getUsedCapacity(label.getLabelName()); String partitionUiTag = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java index b0b301a1310..ea85d13ea5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -58,9 +59,8 @@ public class NodeLabelsPage extends RmView { RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager(); for (RMNodeLabel info : nlm.pullRMNodeLabelsInfo()) { TR>> row = - tbody.tr().td( - info.getLabelName().isEmpty() ? "" : info - .getLabelName()); + tbody.tr().td(info.getLabelName().isEmpty() + ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : info.getLabelName()); String type = (info.getIsExclusive()) ? "Exclusive Partition" : "Non Exclusive Partition"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 46f05336a88..7f8031529a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; @@ -98,6 +99,8 @@ public class AppInfo { protected LogAggregationStatus logAggregationStatus; protected boolean unmanagedApplication; + protected String appNodeLabelExpression; + protected String amNodeLabelExpression; public AppInfo() { } // JAXB needs this @@ -132,8 +135,10 @@ public class AppInfo { this.name = app.getName().toString(); this.queue = app.getQueue().toString(); this.priority = 0; - if (app.getApplicationSubmissionContext().getPriority() != null) { - this.priority = app.getApplicationSubmissionContext().getPriority() + ApplicationSubmissionContext appSubmissionContext = + app.getApplicationSubmissionContext(); + if (appSubmissionContext.getPriority() != null) { + this.priority = appSubmissionContext.getPriority() .getPriority(); } this.progress = app.getProgress() * 100; @@ -191,7 +196,11 @@ public class AppInfo { memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds(); unmanagedApplication = - app.getApplicationSubmissionContext().getUnmanagedAM(); + appSubmissionContext.getUnmanagedAM(); + appNodeLabelExpression = + app.getApplicationSubmissionContext().getNodeLabelExpression(); + amNodeLabelExpression = (unmanagedApplication) ? null + : app.getAMResourceRequest().getNodeLabelExpression(); } } @@ -338,4 +347,12 @@ public class AppInfo { public int getPriority() { return this.priority; } + + public String getAppNodeLabelExpression() { + return this.appNodeLabelExpression; + } + + public String getAmNodeLabelExpression() { + return this.amNodeLabelExpression; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index ae8d747d645..f31a2565447 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -40,7 +40,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { protected ResourceInfo usedAMResource; protected ResourceInfo userAMResourceLimit; protected boolean preemptionDisabled; - + protected String defaultNodeLabelExpression; + @XmlTransient protected String orderingPolicyInfo; @@ -62,6 +63,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit()); preemptionDisabled = q.getPreemptionDisabled(); orderingPolicyInfo = q.getOrderingPolicy().getInfo(); + defaultNodeLabelExpression = q.getDefaultNodeLabelExpression(); } public int getNumActiveApplications() { @@ -116,4 +118,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { public String getOrderingPolicyInfo() { return orderingPolicyInfo; } + + public String getDefaultNodeLabelExpression() { + return defaultNodeLabelExpression; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 0be8bc26358..39964da9b8d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -300,6 +300,33 @@ public class TestClientRMService { report.getApplicationResourceUsageReport(); Assert.assertEquals(10, usageReport.getMemorySeconds()); Assert.assertEquals(3, usageReport.getVcoreSeconds()); + Assert.assertEquals("", report.getAmNodeLabelExpression()); + Assert.assertEquals("", report.getAppNodeLabelExpression()); + + // if application has am node label set to blank + ApplicationId appId2 = getApplicationId(2); + when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), + ApplicationAccessType.VIEW_APP, null, appId2)).thenReturn(true); + request.setApplicationId(appId2); + response = rmService.getApplicationReport(request); + report = response.getApplicationReport(); + + Assert.assertEquals(NodeLabel.DEFAULT_NODE_LABEL_PARTITION, + report.getAmNodeLabelExpression()); + Assert.assertEquals(NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET, + report.getAppNodeLabelExpression()); + + // if application has am node label set to blank + ApplicationId appId3 = getApplicationId(3); + when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), + ApplicationAccessType.VIEW_APP, null, appId3)).thenReturn(true); + + request.setApplicationId(appId3); + response = rmService.getApplicationReport(request); + report = response.getApplicationReport(); + + Assert.assertEquals("high-mem", report.getAmNodeLabelExpression()); + Assert.assertEquals("high-mem", report.getAppNodeLabelExpression()); // if application id is null GetApplicationReportRequest invalidRequest = recordFactory @@ -951,11 +978,11 @@ public class TestClientRMService { ApplicationId applicationId3 = getApplicationId(3); YarnConfiguration config = new YarnConfiguration(); apps.put(applicationId1, getRMApp(rmContext, yarnScheduler, applicationId1, - config, "testqueue", 10, 3)); + config, "testqueue", 10, 3,null,null)); apps.put(applicationId2, getRMApp(rmContext, yarnScheduler, applicationId2, - config, "a", 20, 2)); + config, "a", 20, 2,null,"")); apps.put(applicationId3, getRMApp(rmContext, yarnScheduler, applicationId3, - config, "testqueue", 40, 5)); + config, "testqueue", 40, 5,"high-mem","high-mem")); return apps; } @@ -978,10 +1005,11 @@ public class TestClientRMService { private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, ApplicationId applicationId3, YarnConfiguration config, String queueName, - final long memorySeconds, final long vcoreSeconds) { + final long memorySeconds, final long vcoreSeconds, + String appNodeLabelExpression, String amNodeLabelExpression) { ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class); when(asContext.getMaxAppAttempts()).thenReturn(1); - + when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression); RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null, @@ -1002,7 +1030,7 @@ public class TestClientRMService { return report; } }); - + app.getAMResourceRequest().setNodeLabelExpression(amNodeLabelExpression); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index a23c789ed1d..87f96eb620e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -202,6 +202,16 @@ public abstract class MockAsm extends MockApps { public LogAggregationStatus getLogAggregationStatusForAppReport() { throw new UnsupportedOperationException("Not supported yet."); } + + @Override + public String getAmNodeLabelExpression() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public String getAppNodeLabelExpression() { + throw new UnsupportedOperationException("Not supported yet."); + } } public static RMApp newApplication(int i) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index eb48cc74f16..0498a4f5309 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; @@ -47,6 +48,7 @@ import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -146,6 +148,14 @@ public class TestSystemMetricsPublisher { entity.getOtherInfo().get( ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO)); + Assert.assertEquals(app.getAmNodeLabelExpression(), entity.getOtherInfo() + .get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)); + + Assert.assertEquals( + app.getApplicationSubmissionContext().getNodeLabelExpression(), + entity.getOtherInfo() + .get(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION)); + Assert .assertEquals( app.getUser(), @@ -351,7 +361,7 @@ public class TestSystemMetricsPublisher { } private static RMApp createRMApp(ApplicationId appId) { - RMApp app = mock(RMApp.class); + RMApp app = mock(RMAppImpl.class); when(app.getApplicationId()).thenReturn(appId); when(app.getName()).thenReturn("test app"); when(app.getApplicationType()).thenReturn("test app type"); @@ -376,8 +386,14 @@ public class TestSystemMetricsPublisher { when(app.getApplicationTags()).thenReturn(appTags); ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class); when(asc.getUnmanagedAM()).thenReturn(false); - when(asc.getPriority()).thenReturn(Priority.newInstance(0)); + when(asc.getPriority()).thenReturn(Priority.newInstance(10)); + when(asc.getNodeLabelExpression()).thenReturn("high-cpu"); when(app.getApplicationSubmissionContext()).thenReturn(asc); + when(app.getAppNodeLabelExpression()).thenCallRealMethod(); + ResourceRequest amReq = mock(ResourceRequest.class); + when(amReq.getNodeLabelExpression()).thenReturn("high-mem"); + when(app.getAMResourceRequest()).thenReturn(amReq); + when(app.getAmNodeLabelExpression()).thenCallRealMethod(); return app; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index c6ee3baa20c..7d4e6fd699f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -64,6 +65,8 @@ public class MockRMApp implements RMApp { finish = time; id = MockApps.newAppID(newid); state = newState; + amReq = ResourceRequest.newInstance(Priority.UNDEFINED, "0.0.0.0", + Resource.newInstance(0, 0), 1); } public MockRMApp(int newid, long time, RMAppState newState, String userName) { @@ -283,4 +286,14 @@ public class MockRMApp implements RMApp { public LogAggregationStatus getLogAggregationStatusForAppReport() { return null; } + + @Override + public String getAmNodeLabelExpression() { + return null; + } + + @Override + public String getAppNodeLabelExpression() { + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index a5e3308f565..afe95cf1ae2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -261,10 +261,9 @@ public class TestRMAppTransitions { // but applicationId is still set for safety submissionContext.setApplicationId(applicationId); - RMApp application = - new RMAppImpl(applicationId, rmContext, conf, name, user, queue, - submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN", null, null); + RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, + user, queue, submissionContext, scheduler, masterService, + System.currentTimeMillis(), "YARN", null, mock(ResourceRequest.class)); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index a784295c5b4..47b44d289aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1310,14 +1310,29 @@ public class TestRMWebServicesApps extends JerseyTestBase { WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"), WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted"), WebServicesTestUtils.getXmlString(element, "logAggregationStatus"), - WebServicesTestUtils.getXmlBoolean(element, "unmanagedApplication")); + WebServicesTestUtils.getXmlBoolean(element, "unmanagedApplication"), + WebServicesTestUtils.getXmlString(element, "appNodeLabelExpression"), + WebServicesTestUtils.getXmlString(element, "amNodeLabelExpression")); } } public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - assertEquals("incorrect number of elements", 30, info.length()); + int expectedNumberOfElements = 30; + String appNodeLabelExpression = null; + String amNodeLabelExpression = null; + if (app.getApplicationSubmissionContext() + .getNodeLabelExpression() != null) { + expectedNumberOfElements++; + appNodeLabelExpression = info.getString("appNodeLabelExpression"); + } + if (app.getAMResourceRequest().getNodeLabelExpression() != null) { + expectedNumberOfElements++; + amNodeLabelExpression = info.getString("amNodeLabelExpression"); + } + assertEquals("incorrect number of elements", expectedNumberOfElements, + info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), @@ -1334,7 +1349,9 @@ public class TestRMWebServicesApps extends JerseyTestBase { info.getInt("numNonAMContainerPreempted"), info.getInt("numAMContainerPreempted"), info.getString("logAggregationStatus"), - info.getBoolean("unmanagedApplication")); + info.getBoolean("unmanagedApplication"), + appNodeLabelExpression, + amNodeLabelExpression); } public void verifyAppInfoGeneric(RMApp app, String id, String user, @@ -1345,7 +1362,8 @@ public class TestRMWebServicesApps extends JerseyTestBase { int allocatedMB, int allocatedVCores, int numContainers, int preemptedResourceMB, int preemptedResourceVCores, int numNonAMContainerPreempted, int numAMContainerPreempted, - String logAggregationStatus, boolean unmanagedApplication) + String logAggregationStatus, boolean unmanagedApplication, + String appNodeLabelExpression, String amNodeLabelExpression) throws JSONException, Exception { @@ -1400,6 +1418,12 @@ public class TestRMWebServicesApps extends JerseyTestBase { assertEquals("unmanagedApplication doesn't match", app .getApplicationSubmissionContext().getUnmanagedAM(), unmanagedApplication); + assertEquals("unmanagedApplication doesn't match", + app.getApplicationSubmissionContext().getNodeLabelExpression(), + appNodeLabelExpression); + assertEquals("unmanagedApplication doesn't match", + app.getAMResourceRequest().getNodeLabelExpression(), + amNodeLabelExpression); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index f8b806158db..53df195515b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -1379,11 +1379,13 @@ Response Body: "allocatedVCores" : 0, "runningContainers" : 0, "applicationType" : "MAPREDUCE", - "applicationTags" : "" + "applicationTags" : "", "memorySeconds" : 151730, "vcoreSeconds" : 103, - "unmanagedApplication":"false" - "applicationPriority":0 + "unmanagedApplication" : "false", + "applicationPriority" : 0, + "appNodeLabelExpression" : "", + "amnodeLabelExpression" : "" }, { "finishedTime" : 1326815789546, @@ -1406,11 +1408,13 @@ Response Body: "allocatedVCores" : 0, "runningContainers" : 1, "applicationType" : "YARN", - "applicationTags" : "tag1" + "applicationTags" : "tag1", "memorySeconds" : 640064, - "vcoreSeconds" : 442 - "unmanagedApplication":"false" - "applicationPriority":0 + "vcoreSeconds" : 442, + "unmanagedApplication" : "false", + "applicationPriority" : 0, + "appNodeLabelExpression" : "", + "amNodeLabelExpression" : "" } ] } @@ -1462,6 +1466,8 @@ Response Body: 103 false 0 + + application_1326815542473_0002 @@ -1489,6 +1495,8 @@ Response Body: 442 false 0 + + ``` @@ -1650,6 +1658,8 @@ Note that depending on security settings a user might not be able to see all the | vcoreSeconds | long | The amount of CPU resources the application has allocated (virtual core-seconds) | | unmanagedApplication | boolean | Is the application unmanaged. | | applicationPriority | int | priority of the submitted application | +| appNodeLabelExpression | string | Node Label expression which is used to identify the nodes on which application's containers are expected to run by default.| +| amNodeLabelExpression | string | Node Label expression which is used to identify the node on which application's AM container is expected to run.| ### Response Examples @@ -1690,8 +1700,10 @@ Response Body: "queue" : "a1", "memorySeconds" : 151730, "vcoreSeconds" : 103, - "unmanagedApplication":"false" - "applicationPriority":0 + "unmanagedApplication" : "false", + "applicationPriority" : 0, + "appNodeLabelExpression" : "", + "amNodeLabelExpression" : "" } } ``` @@ -1735,6 +1747,8 @@ Response Body: 103 false 0 + + ``` diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md index 72b813a2574..1f0388e9bbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md @@ -1089,7 +1089,10 @@ Response Body: "finishedTime":1430425008861, "elapsedTime":7857, "unmanagedApplication":"false", - "applicationPriority":0}, + "applicationPriority":0, + "appNodeLabelExpression":"", + "amNodeLabelExpression":"" + }, { "appId":"application_1430424020775_0003", "currentAppAttemptId":"appattempt_1430424020775_0003_000001", @@ -1110,7 +1113,10 @@ Response Body: "finishedTime":1430424963907, "elapsedTime":7257, "unmanagedApplication":"false", - "applicationPriority":0}, + "applicationPriority":0, + "appNodeLabelExpression":"", + "amNodeLabelExpression":"" + }, { "appId":"application_1430424020775_0002", "currentAppAttemptId":"appattempt_1430424020775_0002_000001", @@ -1131,7 +1137,10 @@ Response Body: "finishedTime":1430424776594, "elapsedTime":7199, "unmanagedApplication":"false", - "applicationPriority":0}, + "applicationPriority":0, + "appNodeLabelExpression":"", + "amNodeLabelExpression":"" + }, { "appId":"application_1430424020775_0001", "currentAppAttemptId":"appattempt_1430424020775_0001_000001", @@ -1153,7 +1162,9 @@ Response Body: "elapsedTime":18344, "applicationTags":"mrapplication,ta-example", "unmanagedApplication":"false", - "applicationPriority":0 + "applicationPriority":0, + "appNodeLabelExpression":"", + "amNodeLabelExpression":"" } ] } @@ -1197,6 +1208,8 @@ Response Body: 7857 false 0 + + application_1430424020775_0003 @@ -1219,6 +1232,8 @@ Response Body: 7257 false 0 + + application_1430424020775_0002 @@ -1241,6 +1256,8 @@ Response Body: 7199 false 0 + + application_1430424020775_0001 @@ -1264,6 +1281,8 @@ Response Body: mrapplication,ta-example false 0 + + @@ -1316,7 +1335,8 @@ None | `applicationTags` | string | The application tags. | | `unmanagedApplication` | boolean | Is the application unmanaged. | | `applicationPriority` | int | Priority of the submitted application. | - +| `appNodeLabelExpression` | string |Node Label expression which is used to identify the nodes on which application's containers are expected to run by default.| +| `amNodeLabelExpression` | string | Node Label expression which is used to identify the node on which application's AM container is expected to run.| ### Response Examples: #### JSON response @@ -1353,8 +1373,10 @@ Response Body: "finishedTime": 1430424072153, "elapsedTime": 18344, "applicationTags": mrapplication,tag-example, - "unmanagedApplication":"false" - "applicationPriority":0 + "unmanagedApplication": "false", + "applicationPriority": 0, + "appNodeLabelExpression": "", + "amNodeLabelExpression": "" } #### XML response @@ -1395,6 +1417,8 @@ Response Body: mrapplication,ta-example false 0 + + ## Application Attempt List