From 7af5d6b4bae762e3c7fd0bb16f1f04d45e42ea25 Mon Sep 17 00:00:00 2001 From: Xuan Date: Wed, 1 Apr 2015 04:19:18 -0700 Subject: [PATCH] YARN-3248. Display count of nodes blacklisted by apps in the web UI. Contributed by Varun Vasudev (cherry picked from commit 4728bdfa15809db4b8b235faa286c65de4a48cf6) (cherry picked from commit e26b6e55e96b763063dfbd39977096367eafc1e3) --- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/server/webapp/AppAttemptBlock.java | 66 ++++---- .../hadoop/yarn/server/webapp/AppBlock.java | 38 +++-- .../hadoop/yarn/server/webapp/AppsBlock.java | 95 +++++++----- .../scheduler/AppSchedulingInfo.java | 4 + .../SchedulerApplicationAttempt.java | 4 + .../webapp/AppsBlockWithMetrics.java | 2 +- .../webapp/CapacitySchedulerPage.java | 2 +- .../webapp/RMAppAttemptBlock.java | 68 ++++++++ .../resourcemanager/webapp/RMAppBlock.java | 110 +++++++++++++ .../resourcemanager/webapp/RMAppsBlock.java | 146 ++++++++++++++++++ .../resourcemanager/webapp/RMWebServices.java | 3 +- .../webapp/dao/AppAttemptInfo.java | 19 ++- .../webapp/TestRMWebServicesApps.java | 2 +- 14 files changed, 476 insertions(+), 86 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e7d847f635c..c2c2bfa1c06 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -217,6 +217,9 @@ Release 2.6.1 - 2015-09-09 YARN-3379. Fixed missing data in localityTable and ResourceRequests table in RM WebUI. (Xuan Gong via jianhe) + YARN-3248. Display count of nodes blacklisted by apps in the web UI. + (Varun Vasudev via xgong) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index abf7c866f3d..50d4f72e813 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; + import com.google.inject.Inject; public class AppAttemptBlock extends HtmlBlock { @@ -73,7 +74,7 @@ public class AppAttemptBlock extends HtmlBlock { } UserGroupInformation callerUGI = getCallerUGI(); - ApplicationAttemptReport appAttemptReport = null; + ApplicationAttemptReport appAttemptReport; try { final GetApplicationAttemptReportRequest request = GetApplicationAttemptReportRequest.newInstance(appAttemptId); @@ -136,33 +137,7 @@ public class AppAttemptBlock extends HtmlBlock { && appAttempt.getRpcPort() < 65536) { node = appAttempt.getHost() + ":" + appAttempt.getRpcPort(); } - info("Application Attempt Overview") - ._( - "Application Attempt State:", - appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt - .getAppAttemptState()) - ._( - "AM Container:", - appAttempt.getAmContainerId() == null || containers == null - || !hasAMContainer(appAttemptReport.getAMContainerId(), containers) - ? null : root_url("container", appAttempt.getAmContainerId()), - String.valueOf(appAttempt.getAmContainerId())) - ._("Node:", node) - ._( - "Tracking URL:", - appAttempt.getTrackingUrl() == null - || appAttempt.getTrackingUrl() == UNAVAILABLE ? null - : root_url(appAttempt.getTrackingUrl()), - appAttempt.getTrackingUrl() == null - || appAttempt.getTrackingUrl() == UNAVAILABLE - ? "Unassigned" - : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED - || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED - || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED - ? "History" : "ApplicationMaster") - ._("Diagnostics Info:", appAttempt.getDiagnosticsInfo() == null ? - "" : appAttempt.getDiagnosticsInfo()); - + generateOverview(appAttemptReport, containers, appAttempt, node); html._(InfoBlock.class); if (exceptionWhenGetContainerReports) { @@ -216,7 +191,40 @@ public class AppAttemptBlock extends HtmlBlock { tbody._()._(); } - private boolean hasAMContainer(ContainerId containerId, + protected void generateOverview(ApplicationAttemptReport appAttemptReport, + Collection containers, AppAttemptInfo appAttempt, + String node) { + info("Application Attempt Overview") + ._( + "Application Attempt State:", + appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt + .getAppAttemptState()) + ._( + "AM Container:", + appAttempt.getAmContainerId() == null || containers == null + || !hasAMContainer(appAttemptReport.getAMContainerId(), containers) + ? null : root_url("container", appAttempt.getAmContainerId()), + String.valueOf(appAttempt.getAmContainerId())) + ._("Node:", node) + ._( + "Tracking URL:", + appAttempt.getTrackingUrl() == null + || appAttempt.getTrackingUrl().equals(UNAVAILABLE) ? null + : root_url(appAttempt.getTrackingUrl()), + appAttempt.getTrackingUrl() == null + || appAttempt.getTrackingUrl().equals(UNAVAILABLE) + ? "Unassigned" + : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED + || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED + || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED + ? "History" : "ApplicationMaster") + ._( + "Diagnostics Info:", + appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt + .getDiagnosticsInfo()); + } + + protected boolean hasAMContainer(ContainerId containerId, Collection containers) { for (ContainerReport container : containers) { if (containerId.equals(container.getContainerId())) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 93b24a30b0d..c310f929553 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -24,6 +24,7 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE; import java.security.PrivilegedExceptionAction; import java.util.Collection; + import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -89,7 +90,7 @@ public class AppBlock extends HtmlBlock { } UserGroupInformation callerUGI = getCallerUGI(); - ApplicationReport appReport = null; + ApplicationReport appReport; try { final GetApplicationReportRequest request = GetApplicationReportRequest.newInstance(appID); @@ -161,7 +162,8 @@ public class AppBlock extends HtmlBlock { ._("Application Type:", app.getType()) ._("Application Tags:", app.getApplicationTags() == null ? "" : app.getApplicationTags()) - ._("YarnApplicationState:", + ._( + "YarnApplicationState:", app.getAppState() == null ? UNAVAILABLE : clarifyAppState(app .getAppState())) ._("FinalStatus Reported by AM:", @@ -171,16 +173,19 @@ public class AppBlock extends HtmlBlock { "Elapsed:", StringUtils.formatTime(Times.elapsed(app.getStartedTime(), app.getFinishedTime()))) - ._("Tracking URL:", - app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE - ? null : root_url(app.getTrackingUrl()), - app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE - ? "Unassigned" : app.getAppState() == YarnApplicationState.FINISHED - || app.getAppState() == YarnApplicationState.FAILED - || app.getAppState() == YarnApplicationState.KILLED ? "History" - : "ApplicationMaster") + ._( + "Tracking URL:", + app.getTrackingUrl() == null + || app.getTrackingUrl().equals(UNAVAILABLE) ? null : root_url(app + .getTrackingUrl()), + app.getTrackingUrl() == null + || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app + .getAppState() == YarnApplicationState.FINISHED + || app.getAppState() == YarnApplicationState.FAILED + || app.getAppState() == YarnApplicationState.KILLED ? "History" + : "ApplicationMaster") ._("Diagnostics:", - app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo()); + app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo()); Collection attempts; try { @@ -211,6 +216,13 @@ public class AppBlock extends HtmlBlock { html._(InfoBlock.class); + generateApplicationTable(html, callerUGI, attempts); + + } + + protected void generateApplicationTable(Block html, + UserGroupInformation callerUGI, + Collection attempts) { // Application Attempt Table TBODY> tbody = html.table("#attempts").thead().tr().th(".id", "Attempt ID") @@ -220,7 +232,7 @@ public class AppBlock extends HtmlBlock { StringBuilder attemptsTableData = new StringBuilder("[\n"); for (final ApplicationAttemptReport appAttemptReport : attempts) { AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport); - ContainerReport containerReport = null; + ContainerReport containerReport; try { // AM container is always the first container of the attempt final GetContainerReportRequest request = @@ -231,7 +243,7 @@ public class AppBlock extends HtmlBlock { appBaseProt.getContainerReport(request).getContainerReport(); } else { containerReport = callerUGI.doAs( - new PrivilegedExceptionAction () { + new PrivilegedExceptionAction() { @Override public ContainerReport run() throws Exception { ContainerReport report = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index 583a0ae5441..6f8e10baf88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -24,6 +24,7 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; +import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.Collection; import java.util.EnumSet; @@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; @@ -48,26 +50,19 @@ public class AppsBlock extends HtmlBlock { private static final Log LOG = LogFactory.getLog(AppsBlock.class); protected ApplicationBaseProtocol appBaseProt; + protected EnumSet reqAppStates; + protected UserGroupInformation callerUGI; + protected Collection appReports; @Inject - AppsBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) { + protected AppsBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) { super(ctx); this.appBaseProt = appBaseProt; } - @Override - public void render(Block html) { - setTitle("Applications"); - - TBODY> tbody = - html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") - .th(".name", "Name").th(".type", "Application Type") - .th(".queue", "Queue").th(".starttime", "StartTime") - .th(".finishtime", "FinishTime").th(".state", "State") - .th(".finalstatus", "FinalStatus").th(".progress", "Progress") - .th(".ui", "Tracking UI")._()._().tbody(); - EnumSet reqAppStates = - EnumSet.noneOf(YarnApplicationState.class); + protected void fetchData() throws YarnException, IOException, + InterruptedException { + reqAppStates = EnumSet.noneOf(YarnApplicationState.class); String reqStateString = $(APP_STATE); if (reqStateString != null && !reqStateString.isEmpty()) { String[] appStateStrings = reqStateString.split(","); @@ -76,33 +71,54 @@ public class AppsBlock extends HtmlBlock { } } - UserGroupInformation callerUGI = getCallerUGI(); - Collection appReports = null; + callerUGI = getCallerUGI(); + final GetApplicationsRequest request = + GetApplicationsRequest.newInstance(reqAppStates); + String appsNumStr = $(APPS_NUM); + if (appsNumStr != null && !appsNumStr.isEmpty()) { + long appsNum = Long.parseLong(appsNumStr); + request.setLimit(appsNum); + } + if (callerUGI == null) { + appReports = appBaseProt.getApplications(request).getApplicationList(); + } else { + appReports = + callerUGI + .doAs(new PrivilegedExceptionAction>() { + @Override + public Collection run() throws Exception { + return appBaseProt.getApplications(request) + .getApplicationList(); + } + }); + } + } + + @Override + public void render(Block html) { + setTitle("Applications"); + try { - final GetApplicationsRequest request = - GetApplicationsRequest.newInstance(reqAppStates); - String appsNumStr = $(APPS_NUM); - if (appsNumStr != null && !appsNumStr.isEmpty()) { - long appsNum = Long.parseLong(appsNumStr); - request.setLimit(appsNum); - } - if (callerUGI == null) { - appReports = appBaseProt.getApplications(request).getApplicationList(); - } else { - appReports = callerUGI.doAs( - new PrivilegedExceptionAction> () { - @Override - public Collection run() throws Exception { - return appBaseProt.getApplications(request).getApplicationList(); - } - }); - } - } catch (Exception e) { + fetchData(); + } + catch( Exception e) { String message = "Failed to read the applications."; LOG.error(message, e); html.p()._(message)._(); return; } + renderData(html); + } + + protected void renderData(Block html) { + TBODY> tbody = + html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") + .th(".name", "Name").th(".type", "Application Type") + .th(".queue", "Queue").th(".starttime", "StartTime") + .th(".finishtime", "FinishTime").th(".state", "State") + .th(".finalstatus", "FinalStatus").th(".progress", "Progress") + .th(".ui", "Tracking UI")._()._().tbody(); + StringBuilder appsTableData = new StringBuilder("[\n"); for (ApplicationReport appReport : appReports) { // TODO: remove the following condition. It is still here because @@ -123,7 +139,7 @@ public class AppsBlock extends HtmlBlock { .append("\",\"") .append( StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app - .getUser()))) + .getUser()))) .append("\",\"") .append( StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app @@ -150,11 +166,12 @@ public class AppsBlock extends HtmlBlock { .append("'> ").append("\",\" getBlackListCopy() { + return new HashSet(this.blacklist); + } + public synchronized void transferStateFromPreviousAppSchedulingInfo( AppSchedulingInfo appInfo) { // this.priorities = appInfo.getPriorities(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 84975b6b84e..510276cbb33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -615,4 +615,8 @@ public class SchedulerApplicationAttempt { // schedulingOpportunities // lastScheduledContainer } + + public Set getBlacklistedNodes() { + return this.appSchedulingInfo.getBlackListCopy(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java index cb0836a7e5f..29889ecc6e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java @@ -27,6 +27,6 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock; class AppsBlockWithMetrics extends HtmlBlock { @Override public void render(Block html) { html._(MetricsOverviewTable.class); - html._(AppsBlock.class); + html._(RMAppsBlock.class); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 0be361a0be7..b1d0ebad063 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -230,7 +230,7 @@ class CapacitySchedulerPage extends RmView { ul._()._(). script().$type("text/javascript"). _("$('#cs').hide();")._()._(). - _(AppsBlock.class); + _(RMAppsBlock.class); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java index 7dc70301c56..c94d59a5a69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -18,12 +18,23 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import java.util.Collection; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock; +import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import com.google.inject.Inject; @@ -53,4 +64,61 @@ public class RMAppAttemptBlock extends AppAttemptBlock{ } return attempt; } + + protected void generateOverview(ApplicationAttemptReport appAttemptReport, + Collection containers, AppAttemptInfo appAttempt, + String node) { + + String blacklistedNodes = "-"; + Set nodes = + getBlacklistedNodes(rm, getRMAppAttempt().getAppAttemptId()); + if (nodes != null) { + if (!nodes.isEmpty()) { + blacklistedNodes = StringUtils.join(nodes, ", "); + } + } + + info("Application Attempt Overview") + ._( + "Application Attempt State:", + appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt + .getAppAttemptState()) + ._( + "AM Container:", + appAttempt.getAmContainerId() == null || containers == null + || !hasAMContainer(appAttemptReport.getAMContainerId(), containers) + ? null : root_url("container", appAttempt.getAmContainerId()), + String.valueOf(appAttempt.getAmContainerId())) + ._("Node:", node) + ._( + "Tracking URL:", + appAttempt.getTrackingUrl() == null + || appAttempt.getTrackingUrl().equals(UNAVAILABLE) ? null + : root_url(appAttempt.getTrackingUrl()), + appAttempt.getTrackingUrl() == null + || appAttempt.getTrackingUrl().equals(UNAVAILABLE) + ? "Unassigned" + : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED + || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED + || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED + ? "History" : "ApplicationMaster") + ._( + "Diagnostics Info:", + appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt + .getDiagnosticsInfo())._("Blacklisted Nodes:", blacklistedNodes); + } + + public static Set getBlacklistedNodes(ResourceManager rm, + ApplicationAttemptId appid) { + if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) { + AbstractYarnScheduler ayScheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + SchedulerApplicationAttempt attempt = + ayScheduler.getApplicationAttempt(appid); + if (attempt != null) { + return attempt.getBlacklistedNodes(); + } + } + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java index 64c57476f79..171c6c9ea22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java @@ -20,13 +20,29 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP; +import java.security.PrivilegedExceptionAction; +import java.util.Collection; +import java.util.Set; + +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; import org.apache.hadoop.yarn.server.webapp.AppBlock; +import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; @@ -36,8 +52,10 @@ import com.google.inject.Inject; public class RMAppBlock extends AppBlock{ + private static final Log LOG = LogFactory.getLog(RMAppBlock.class); private final ResourceManager rm; + @Inject RMAppBlock(ViewContext ctx, Configuration conf, ResourceManager rm) { super(rm.getClientRMService(), ctx, conf); @@ -91,4 +109,96 @@ public class RMAppBlock extends AppBlock{ appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds())); pdiv._(); } + + @Override + protected void generateApplicationTable(Block html, + UserGroupInformation callerUGI, + Collection attempts) { + // Application Attempt Table + Hamlet.TBODY> tbody = + html.table("#attempts").thead().tr().th(".id", "Attempt ID") + .th(".started", "Started").th(".node", "Node").th(".logs", "Logs") + .th(".blacklistednodes", "Blacklisted Nodes")._()._().tbody(); + + StringBuilder attemptsTableData = new StringBuilder("[\n"); + for (final ApplicationAttemptReport appAttemptReport : attempts) { + AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport); + ContainerReport containerReport = null; + try { + // AM container is always the first container of the attempt + final GetContainerReportRequest request = + GetContainerReportRequest.newInstance(ContainerId.newContainerId( + appAttemptReport.getApplicationAttemptId(), 1)); + if (callerUGI == null) { + containerReport = + appBaseProt.getContainerReport(request).getContainerReport(); + } else { + containerReport = callerUGI.doAs( + new PrivilegedExceptionAction() { + @Override + public ContainerReport run() throws Exception { + ContainerReport report = null; + try { + report = appBaseProt.getContainerReport(request) + .getContainerReport(); + } catch (ContainerNotFoundException ex) { + LOG.warn(ex.getMessage()); + } + return report; + } + }); + } + } catch (Exception e) { + String message = + "Failed to read the AM container of the application attempt " + + appAttemptReport.getApplicationAttemptId() + "."; + LOG.error(message, e); + html.p()._(message)._(); + return; + } + long startTime = 0L; + String logsLink = null; + String nodeLink = null; + if (containerReport != null) { + ContainerInfo container = new ContainerInfo(containerReport); + startTime = container.getStartedTime(); + logsLink = containerReport.getLogUrl(); + nodeLink = containerReport.getNodeHttpAddress(); + } + String blacklistedNodesCount = "N/A"; + Set nodes = RMAppAttemptBlock.getBlacklistedNodes(rm, + ConverterUtils.toApplicationAttemptId(appAttempt.getAppAttemptId())); + if(nodes != null) { + blacklistedNodesCount = String.valueOf(nodes.size()); + } + + // AppAttemptID numerical value parsed by parseHadoopID in + // yarn.dt.plugins.js + attemptsTableData + .append("[\"") + .append(appAttempt.getAppAttemptId()) + .append("\",\"") + .append(startTime) + .append("\",\"") + .append(nodeLink == null ? "N/A" : StringEscapeUtils + .escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink))) + .append("\",\"") + .append(logsLink == null ? "N/A" : "Logs").append("\",").append( + "\"").append(blacklistedNodesCount).append("\"],\n"); + } + if (attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') { + attemptsTableData.delete(attemptsTableData.length() - 2, + attemptsTableData.length() - 1); + } + attemptsTableData.append("]"); + html.script().$type("text/javascript") + ._("var attemptsTableData=" + attemptsTableData)._(); + + tbody._()._(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java new file mode 100644 index 00000000000..94f3afac2ac --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; + +import java.util.Set; + +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.webapp.AppsBlock; +import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.View; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; + +import com.google.inject.Inject; + +public class RMAppsBlock extends AppsBlock { + + private ResourceManager rm; + + @Inject + RMAppsBlock(ResourceManager rm, ApplicationBaseProtocol appBaseProt, + View.ViewContext ctx) { + super(appBaseProt, ctx); + this.rm = rm; + } + + @Override + protected void renderData(Block html) { + TBODY> tbody = + html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") + .th(".name", "Name").th(".type", "Application Type") + .th(".queue", "Queue").th(".starttime", "StartTime") + .th(".finishtime", "FinishTime").th(".state", "State") + .th(".finalstatus", "FinalStatus").th(".progress", "Progress") + .th(".ui", "Tracking UI").th(".blacklisted", "Blacklisted Nodes")._() + ._().tbody(); + + StringBuilder appsTableData = new StringBuilder("[\n"); + for (ApplicationReport appReport : appReports) { + // TODO: remove the following condition. It is still here because + // the history side implementation of ApplicationBaseProtocol + // hasn't filtering capability (YARN-1819). + if (!reqAppStates.isEmpty() + && !reqAppStates.contains(appReport.getYarnApplicationState())) { + continue; + } + + AppInfo app = new AppInfo(appReport); + String blacklistedNodesCount = "N/A"; + Set nodes = + RMAppAttemptBlock + .getBlacklistedNodes(rm, ConverterUtils.toApplicationAttemptId(app + .getCurrentAppAttemptId())); + if (nodes != null) { + blacklistedNodesCount = String.valueOf(nodes.size()); + } + String percent = String.format("%.1f", app.getProgress()); + // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js + appsTableData + .append("[\"") + .append(app.getAppId()) + .append("\",\"") + .append( + StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app + .getUser()))) + .append("\",\"") + .append( + StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app + .getName()))) + .append("\",\"") + .append( + StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app + .getType()))) + .append("\",\"") + .append( + StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app + .getQueue()))).append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getFinishedTime()) + .append("\",\"") + .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) + .append("\",\"") + .append(app.getFinalAppStatus()) + .append("\",\"") + // Progress bar + .append("
").append("
").append("\",\"").append(trackingUI).append("\",").append("\"") + .append(blacklistedNodesCount).append("\"],\n"); + + } + if (appsTableData.charAt(appsTableData.length() - 2) == ',') { + appsTableData.delete(appsTableData.length() - 2, + appsTableData.length() - 1); + } + appsTableData.append("]"); + html.script().$type("text/javascript") + ._("var appsTableData=" + appsTableData)._(); + + tbody._()._(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 27417a96633..774692209d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -631,7 +631,8 @@ public class RMWebServices { AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo(); for (RMAppAttempt attempt : app.getAppAttempts().values()) { - AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser()); + AppAttemptInfo attemptInfo = + new AppAttemptInfo(rm, attempt, app.getUser()); appAttemptsInfo.add(attemptInfo); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java index 0bb9cf7309a..2c7bbd037bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java @@ -21,8 +21,13 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMAppAttemptBlock; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -36,16 +41,18 @@ public class AppAttemptInfo { protected String nodeHttpAddress; protected String nodeId; protected String logsLink; + protected String blacklistedNodes; public AppAttemptInfo() { } - public AppAttemptInfo(RMAppAttempt attempt, String user) { + public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, String user) { this.startTime = 0; this.containerId = ""; this.nodeHttpAddress = ""; this.nodeId = ""; this.logsLink = ""; + this.blacklistedNodes = ""; if (attempt != null) { this.id = attempt.getAppAttemptId().getAttemptId(); this.startTime = attempt.getStartTime(); @@ -57,6 +64,16 @@ public class AppAttemptInfo { this.logsLink = WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(), ConverterUtils.toString(masterContainer.getId()), user); + if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) { + AbstractYarnScheduler ayScheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + SchedulerApplicationAttempt sattempt = + ayScheduler.getApplicationAttempt(attempt.getAppAttemptId()); + if (sattempt != null) { + blacklistedNodes = + StringUtils.join(sattempt.getBlacklistedNodes(), ", "); + } + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 23ea22e37fb..3258d7ef103 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1599,7 +1599,7 @@ public class TestRMWebServicesApps extends JerseyTest { String user) throws JSONException, Exception { - assertEquals("incorrect number of elements", 6, info.length()); + assertEquals("incorrect number of elements", 7, info.length()); verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"), info.getLong("startTime"), info.getString("containerId"),