diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0a1d8835fbe..0dd09062dc3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1058,6 +1058,9 @@ Release 2.6.1 - 2015-09-09 YARN-4047. ClientRMService getApplications has high scheduler lock contention. (Jason Lowe via jianhe) + YARN-3248. Display count of nodes blacklisted by apps in the web UI. + (Varun Vasudev via xgong) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index 42e20341ebd..4c4033771db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -72,7 +72,7 @@ public class AppAttemptBlock extends HtmlBlock { } UserGroupInformation callerUGI = getCallerUGI(); - ApplicationAttemptReport appAttemptReport = null; + ApplicationAttemptReport appAttemptReport; try { final GetApplicationAttemptReportRequest request = GetApplicationAttemptReportRequest.newInstance(appAttemptId); @@ -135,32 +135,7 @@ public class AppAttemptBlock extends HtmlBlock { && appAttempt.getRpcPort() < 65536) { node = appAttempt.getHost() + ":" + appAttempt.getRpcPort(); } - info("Application Attempt Overview") - ._( - "Application Attempt State:", - appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt - .getAppAttemptState()) - ._( - "AM Container:", - appAttempt.getAmContainerId() == null || containers == null - || !hasAMContainer(appAttemptReport.getAMContainerId(), containers) - ? null : root_url("container", appAttempt.getAmContainerId()), - String.valueOf(appAttempt.getAmContainerId())) - ._("Node:", node) - ._( - "Tracking URL:", - appAttempt.getTrackingUrl() == null - || appAttempt.getTrackingUrl() == UNAVAILABLE ? null - : root_url(appAttempt.getTrackingUrl()), - appAttempt.getTrackingUrl() == null - || appAttempt.getTrackingUrl() == UNAVAILABLE - ? "Unassigned" - : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED - || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED - || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED - ? "History" : "ApplicationMaster") - ._("Diagnostics Info:", appAttempt.getDiagnosticsInfo() == null ? - "" : appAttempt.getDiagnosticsInfo()); + generateOverview(appAttemptReport, containers, appAttempt, node); if (exceptionWhenGetContainerReports) { html @@ -216,7 +191,40 @@ public class AppAttemptBlock extends HtmlBlock { tbody._()._(); } - private boolean hasAMContainer(ContainerId containerId, + protected void generateOverview(ApplicationAttemptReport appAttemptReport, + Collection containers, AppAttemptInfo appAttempt, + String node) { + info("Application Attempt Overview") + ._( + "Application Attempt State:", + appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt + .getAppAttemptState()) + ._( + "AM Container:", + appAttempt.getAmContainerId() == null || containers == null + || !hasAMContainer(appAttemptReport.getAMContainerId(), containers) + ? null : root_url("container", appAttempt.getAmContainerId()), + String.valueOf(appAttempt.getAmContainerId())) + ._("Node:", node) + ._( + "Tracking URL:", + appAttempt.getTrackingUrl() == null + || appAttempt.getTrackingUrl().equals(UNAVAILABLE) ? null + : root_url(appAttempt.getTrackingUrl()), + appAttempt.getTrackingUrl() == null + || appAttempt.getTrackingUrl().equals(UNAVAILABLE) + ? "Unassigned" + : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED + || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED + || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED + ? "History" : "ApplicationMaster") + ._( + "Diagnostics Info:", + appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt + .getDiagnosticsInfo()); + } + + protected boolean hasAMContainer(ContainerId containerId, Collection containers) { for (ContainerReport container : containers) { if (containerId.equals(container.getContainerId())) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index c620f8eb0d5..2497ff3fc1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -88,7 +88,7 @@ public class AppBlock extends HtmlBlock { } callerUGI = getCallerUGI(); - ApplicationReport appReport = null; + ApplicationReport appReport; try { final GetApplicationReportRequest request = GetApplicationReportRequest.newInstance(appID); @@ -160,7 +160,8 @@ public class AppBlock extends HtmlBlock { ._("Application Type:", app.getType()) ._("Application Tags:", app.getApplicationTags() == null ? "" : app.getApplicationTags()) - ._("YarnApplicationState:", + ._( + "YarnApplicationState:", app.getAppState() == null ? UNAVAILABLE : clarifyAppState(app .getAppState())) ._("FinalStatus Reported by AM:", @@ -170,16 +171,19 @@ public class AppBlock extends HtmlBlock { "Elapsed:", StringUtils.formatTime(Times.elapsed(app.getStartedTime(), app.getFinishedTime()))) - ._("Tracking URL:", - app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE - ? null : root_url(app.getTrackingUrl()), - app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE - ? "Unassigned" : app.getAppState() == YarnApplicationState.FINISHED - || app.getAppState() == YarnApplicationState.FAILED - || app.getAppState() == YarnApplicationState.KILLED ? "History" - : "ApplicationMaster") + ._( + "Tracking URL:", + app.getTrackingUrl() == null + || app.getTrackingUrl().equals(UNAVAILABLE) ? null : root_url(app + .getTrackingUrl()), + app.getTrackingUrl() == null + || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app + .getAppState() == YarnApplicationState.FINISHED + || app.getAppState() == YarnApplicationState.FAILED + || app.getAppState() == YarnApplicationState.KILLED ? "History" + : "ApplicationMaster") ._("Diagnostics:", - app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo()); + app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo()); Collection attempts; try { @@ -210,6 +214,13 @@ public class AppBlock extends HtmlBlock { html._(InfoBlock.class); + generateApplicationTable(html, callerUGI, attempts); + + } + + protected void generateApplicationTable(Block html, + UserGroupInformation callerUGI, + Collection attempts) { // Application Attempt Table createApplicationAttemptTable(html, attempts); } @@ -224,7 +235,7 @@ public class AppBlock extends HtmlBlock { StringBuilder attemptsTableData = new StringBuilder("[\n"); for (final ApplicationAttemptReport appAttemptReport : attempts) { AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport); - ContainerReport containerReport = null; + ContainerReport containerReport; try { final GetContainerReportRequest request = GetContainerReportRequest.newInstance( @@ -234,7 +245,7 @@ public class AppBlock extends HtmlBlock { appBaseProt.getContainerReport(request).getContainerReport(); } else { containerReport = callerUGI.doAs( - new PrivilegedExceptionAction () { + new PrivilegedExceptionAction() { @Override public ContainerReport run() throws Exception { ContainerReport report = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index 315216581b1..f5b4d62395b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -24,6 +24,7 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; +import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.Collection; import java.util.EnumSet; @@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; @@ -48,26 +50,19 @@ public class AppsBlock extends HtmlBlock { private static final Log LOG = LogFactory.getLog(AppsBlock.class); protected ApplicationBaseProtocol appBaseProt; + protected EnumSet reqAppStates; + protected UserGroupInformation callerUGI; + protected Collection appReports; @Inject - AppsBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) { + protected AppsBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) { super(ctx); this.appBaseProt = appBaseProt; } - @Override - public void render(Block html) { - setTitle("Applications"); - - TBODY> tbody = - html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") - .th(".name", "Name").th(".type", "Application Type") - .th(".queue", "Queue").th(".starttime", "StartTime") - .th(".finishtime", "FinishTime").th(".state", "State") - .th(".finalstatus", "FinalStatus").th(".progress", "Progress") - .th(".ui", "Tracking UI")._()._().tbody(); - EnumSet reqAppStates = - EnumSet.noneOf(YarnApplicationState.class); + protected void fetchData() throws YarnException, IOException, + InterruptedException { + reqAppStates = EnumSet.noneOf(YarnApplicationState.class); String reqStateString = $(APP_STATE); if (reqStateString != null && !reqStateString.isEmpty()) { String[] appStateStrings = reqStateString.split(","); @@ -76,34 +71,54 @@ public class AppsBlock extends HtmlBlock { } } - UserGroupInformation callerUGI = getCallerUGI(); - Collection appReports = null; + callerUGI = getCallerUGI(); + final GetApplicationsRequest request = + GetApplicationsRequest.newInstance(reqAppStates); + String appsNumStr = $(APPS_NUM); + if (appsNumStr != null && !appsNumStr.isEmpty()) { + long appsNum = Long.parseLong(appsNumStr); + request.setLimit(appsNum); + } + if (callerUGI == null) { + appReports = appBaseProt.getApplications(request).getApplicationList(); + } else { + appReports = + callerUGI + .doAs(new PrivilegedExceptionAction>() { + @Override + public Collection run() throws Exception { + return appBaseProt.getApplications(request) + .getApplicationList(); + } + }); + } + } + + @Override + public void render(Block html) { + setTitle("Applications"); try { - final GetApplicationsRequest request = - GetApplicationsRequest.newInstance(reqAppStates); - String appsNumStr = $(APPS_NUM); - if (appsNumStr != null && !appsNumStr.isEmpty()) { - long appsNum = Long.parseLong(appsNumStr); - request.setLimit(appsNum); - } - if (callerUGI == null) { - appReports = appBaseProt.getApplications(request).getApplicationList(); - } else { - appReports = callerUGI.doAs( - new PrivilegedExceptionAction> () { - @Override - public Collection run() throws Exception { - return appBaseProt.getApplications(request).getApplicationList(); - } - }); - } - } catch (Exception e) { + fetchData(); + } + catch( Exception e) { String message = "Failed to read the applications."; LOG.error(message, e); html.p()._(message)._(); return; } + renderData(html); + } + + protected void renderData(Block html) { + TBODY> tbody = + html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") + .th(".name", "Name").th(".type", "Application Type") + .th(".queue", "Queue").th(".starttime", "StartTime") + .th(".finishtime", "FinishTime").th(".state", "State") + .th(".finalstatus", "FinalStatus").th(".progress", "Progress") + .th(".ui", "Tracking UI")._()._().tbody(); + StringBuilder appsTableData = new StringBuilder("[\n"); for (ApplicationReport appReport : appReports) { // TODO: remove the following condition. It is still here because @@ -123,7 +138,7 @@ public class AppsBlock extends HtmlBlock { .append("\",\"") .append( StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app - .getUser()))) + .getUser()))) .append("\",\"") .append( StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app @@ -150,11 +165,12 @@ public class AppsBlock extends HtmlBlock { .append("'> ").append("\",\" getBlackListCopy() { + return new HashSet<>(this.blacklist); + } + public synchronized void transferStateFromPreviousAppSchedulingInfo( AppSchedulingInfo appInfo) { // this.priorities = appInfo.getPriorities(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 9816699851c..ce7f9d1d0c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -642,4 +642,8 @@ public class SchedulerApplicationAttempt { Resources.clone(headroom)); } } + + public Set getBlacklistedNodes() { + return this.appSchedulingInfo.getBlackListCopy(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java index cb0836a7e5f..29889ecc6e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java @@ -27,6 +27,6 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock; class AppsBlockWithMetrics extends HtmlBlock { @Override public void render(Block html) { html._(MetricsOverviewTable.class); - html._(AppsBlock.class); + html._(RMAppsBlock.class); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index c6c410c3ec6..e62fd706788 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -239,7 +239,7 @@ class CapacitySchedulerPage extends RmView { ul._()._(). script().$type("text/javascript"). _("$('#cs').hide();")._()._(). - _(AppsBlock.class); + _(RMAppsBlock.class); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java index 279c3ea1b67..b93f552fc8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -23,8 +23,12 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._ODD; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; @@ -32,8 +36,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock; +import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; @@ -43,6 +50,9 @@ import org.apache.hadoop.yarn.webapp.view.InfoBlock; import com.google.inject.Inject; import java.util.List; +import java.util.Collection; +import java.util.Set; + public class RMAppAttemptBlock extends AppAttemptBlock{ private final ResourceManager rm; @@ -197,4 +207,61 @@ public class RMAppAttemptBlock extends AppAttemptBlock{ createContainerLocalityTable(html); createResourceRequestsTable(html); } + + protected void generateOverview(ApplicationAttemptReport appAttemptReport, + Collection containers, AppAttemptInfo appAttempt, + String node) { + + String blacklistedNodes = "-"; + Set nodes = + getBlacklistedNodes(rm, getRMAppAttempt().getAppAttemptId()); + if (nodes != null) { + if (!nodes.isEmpty()) { + blacklistedNodes = StringUtils.join(nodes, ", "); + } + } + + info("Application Attempt Overview") + ._( + "Application Attempt State:", + appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt + .getAppAttemptState()) + ._( + "AM Container:", + appAttempt.getAmContainerId() == null || containers == null + || !hasAMContainer(appAttemptReport.getAMContainerId(), containers) + ? null : root_url("container", appAttempt.getAmContainerId()), + String.valueOf(appAttempt.getAmContainerId())) + ._("Node:", node) + ._( + "Tracking URL:", + appAttempt.getTrackingUrl() == null + || appAttempt.getTrackingUrl().equals(UNAVAILABLE) ? null + : root_url(appAttempt.getTrackingUrl()), + appAttempt.getTrackingUrl() == null + || appAttempt.getTrackingUrl().equals(UNAVAILABLE) + ? "Unassigned" + : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED + || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED + || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED + ? "History" : "ApplicationMaster") + ._( + "Diagnostics Info:", + appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt + .getDiagnosticsInfo())._("Blacklisted Nodes:", blacklistedNodes); + } + + public static Set getBlacklistedNodes(ResourceManager rm, + ApplicationAttemptId appid) { + if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) { + AbstractYarnScheduler ayScheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + SchedulerApplicationAttempt attempt = + ayScheduler.getApplicationAttempt(appid); + if (attempt != null) { + return attempt.getBlacklistedNodes(); + } + } + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java index bf5e4732306..817195c240a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java @@ -20,12 +20,21 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP; +import java.security.PrivilegedExceptionAction; import java.util.Collection; +import java.util.Set; import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; @@ -33,6 +42,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.AppBlock; +//import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; @@ -45,9 +57,11 @@ import com.google.inject.Inject; public class RMAppBlock extends AppBlock{ + private static final Log LOG = LogFactory.getLog(RMAppBlock.class); private final ResourceManager rm; private final Configuration conf; + @Inject RMAppBlock(ViewContext ctx, Configuration conf, ResourceManager rm) { super(rm.getClientRMService(), ctx, conf); @@ -122,7 +136,7 @@ public class RMAppBlock extends AppBlock{ continue; } AppAttemptInfo attemptInfo = - new AppAttemptInfo(rmAppAttempt, rmApp.getUser(), + new AppAttemptInfo(rm, rmAppAttempt, rmApp.getUser(), WebAppUtils.getHttpSchemePrefix(conf)); String nodeLink = attemptInfo.getNodeHttpAddress(); if (nodeLink != null) { @@ -153,6 +167,99 @@ public class RMAppBlock extends AppBlock{ attemptsTableData.append("]"); html.script().$type("text/javascript") ._("var attemptsTableData=" + attemptsTableData)._(); + tbody._()._(); + + } + + protected void generateApplicationTable(Block html, + UserGroupInformation callerUGI, + Collection attempts) { + // Application Attempt Table + Hamlet.TBODY> tbody = + html.table("#attempts").thead().tr().th(".id", "Attempt ID") + .th(".started", "Started").th(".node", "Node").th(".logs", "Logs") + .th(".blacklistednodes", "Blacklisted Nodes")._()._().tbody(); + + StringBuilder attemptsTableData = new StringBuilder("[\n"); + for (final ApplicationAttemptReport appAttemptReport : attempts) { + org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo appAttempt = + new org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo( + appAttemptReport); + ContainerReport containerReport = null; + try { + // AM container is always the first container of the attempt + final GetContainerReportRequest request = + GetContainerReportRequest.newInstance(ContainerId.newContainerId( + appAttemptReport.getApplicationAttemptId(), 1)); + if (callerUGI == null) { + containerReport = + appBaseProt.getContainerReport(request).getContainerReport(); + } else { + containerReport = callerUGI.doAs( + new PrivilegedExceptionAction() { + @Override + public ContainerReport run() throws Exception { + ContainerReport report = null; + try { + report = appBaseProt.getContainerReport(request) + .getContainerReport(); + } catch (ContainerNotFoundException ex) { + LOG.warn(ex.getMessage()); + } + return report; + } + }); + } + } catch (Exception e) { + String message = + "Failed to read the AM container of the application attempt " + + appAttemptReport.getApplicationAttemptId() + "."; + LOG.error(message, e); + html.p()._(message)._(); + return; + } + long startTime = 0L; + String logsLink = null; + String nodeLink = null; + if (containerReport != null) { + ContainerInfo container = new ContainerInfo(containerReport); + startTime = container.getStartedTime(); + logsLink = containerReport.getLogUrl(); + nodeLink = containerReport.getNodeHttpAddress(); + } + String blacklistedNodesCount = "N/A"; + Set nodes = RMAppAttemptBlock.getBlacklistedNodes(rm, + ConverterUtils.toApplicationAttemptId(appAttempt.getAppAttemptId())); + if(nodes != null) { + blacklistedNodesCount = String.valueOf(nodes.size()); + } + + // AppAttemptID numerical value parsed by parseHadoopID in + // yarn.dt.plugins.js + attemptsTableData + .append("[\"") + .append(appAttempt.getAppAttemptId()) + .append("\",\"") + .append(startTime) + .append("\",\"") + .append(nodeLink == null ? "N/A" : StringEscapeUtils + .escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink))) + .append("\",\"") + .append(logsLink == null ? "N/A" : "Logs").append("\",").append( + "\"").append(blacklistedNodesCount).append("\"],\n"); + } + if (attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') { + attemptsTableData.delete(attemptsTableData.length() - 2, + attemptsTableData.length() - 1); + } + attemptsTableData.append("]"); + html.script().$type("text/javascript") + ._("var attemptsTableData=" + attemptsTableData)._(); tbody._()._(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java new file mode 100644 index 00000000000..94f3afac2ac --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; + +import java.util.Set; + +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.webapp.AppsBlock; +import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.View; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; + +import com.google.inject.Inject; + +public class RMAppsBlock extends AppsBlock { + + private ResourceManager rm; + + @Inject + RMAppsBlock(ResourceManager rm, ApplicationBaseProtocol appBaseProt, + View.ViewContext ctx) { + super(appBaseProt, ctx); + this.rm = rm; + } + + @Override + protected void renderData(Block html) { + TBODY> tbody = + html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") + .th(".name", "Name").th(".type", "Application Type") + .th(".queue", "Queue").th(".starttime", "StartTime") + .th(".finishtime", "FinishTime").th(".state", "State") + .th(".finalstatus", "FinalStatus").th(".progress", "Progress") + .th(".ui", "Tracking UI").th(".blacklisted", "Blacklisted Nodes")._() + ._().tbody(); + + StringBuilder appsTableData = new StringBuilder("[\n"); + for (ApplicationReport appReport : appReports) { + // TODO: remove the following condition. It is still here because + // the history side implementation of ApplicationBaseProtocol + // hasn't filtering capability (YARN-1819). + if (!reqAppStates.isEmpty() + && !reqAppStates.contains(appReport.getYarnApplicationState())) { + continue; + } + + AppInfo app = new AppInfo(appReport); + String blacklistedNodesCount = "N/A"; + Set nodes = + RMAppAttemptBlock + .getBlacklistedNodes(rm, ConverterUtils.toApplicationAttemptId(app + .getCurrentAppAttemptId())); + if (nodes != null) { + blacklistedNodesCount = String.valueOf(nodes.size()); + } + String percent = String.format("%.1f", app.getProgress()); + // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js + appsTableData + .append("[\"") + .append(app.getAppId()) + .append("\",\"") + .append( + StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app + .getUser()))) + .append("\",\"") + .append( + StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app + .getName()))) + .append("\",\"") + .append( + StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app + .getType()))) + .append("\",\"") + .append( + StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app + .getQueue()))).append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getFinishedTime()) + .append("\",\"") + .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) + .append("\",\"") + .append(app.getFinalAppStatus()) + .append("\",\"") + // Progress bar + .append("
").append("
").append("\",\"").append(trackingUI).append("\",").append("\"") + .append(blacklistedNodesCount).append("\"],\n"); + + } + if (appsTableData.charAt(appsTableData.length() - 2) == ',') { + appsTableData.delete(appsTableData.length() - 2, + appsTableData.length() - 1); + } + appsTableData.append("]"); + html.script().$type("text/javascript") + ._("var appsTableData=" + appsTableData)._(); + + tbody._()._(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 7c3dd551873..474630a7bbb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -648,8 +648,9 @@ public class RMWebServices { AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo(); for (RMAppAttempt attempt : app.getAppAttempts().values()) { - AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser(), - hsr.getScheme() + "://"); + AppAttemptInfo attemptInfo = + new AppAttemptInfo(rm, attempt, app.getUser(), hsr.getScheme() + + "://"); appAttemptsInfo.add(attemptInfo); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java index e722fee424f..b6e95a6bb35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java @@ -21,8 +21,12 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -36,17 +40,19 @@ public class AppAttemptInfo { protected String nodeHttpAddress; protected String nodeId; protected String logsLink; + protected String blacklistedNodes; public AppAttemptInfo() { } - public AppAttemptInfo(RMAppAttempt attempt, String user, + public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, String user, String schemePrefix) { this.startTime = 0; this.containerId = ""; this.nodeHttpAddress = ""; this.nodeId = ""; this.logsLink = ""; + this.blacklistedNodes = ""; if (attempt != null) { this.id = attempt.getAppAttemptId().getAttemptId(); this.startTime = attempt.getStartTime(); @@ -58,6 +64,16 @@ public class AppAttemptInfo { this.logsLink = WebAppUtils.getRunningLogURL(schemePrefix + masterContainer.getNodeHttpAddress(), ConverterUtils.toString(masterContainer.getId()), user); + if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) { + AbstractYarnScheduler ayScheduler = + (AbstractYarnScheduler) rm.getResourceScheduler(); + SchedulerApplicationAttempt sattempt = + ayScheduler.getApplicationAttempt(attempt.getAppAttemptId()); + if (sattempt != null) { + blacklistedNodes = + StringUtils.join(sattempt.getBlacklistedNodes(), ", "); + } + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 3ae8601e670..ea8fd57a645 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1598,7 +1598,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { String user) throws JSONException, Exception { - assertEquals("incorrect number of elements", 6, info.length()); + assertEquals("incorrect number of elements", 7, info.length()); verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"), info.getLong("startTime"), info.getString("containerId"),