From 647457e6ab4395398f054d4668adb7c7b1ef7104 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Fri, 14 Oct 2022 05:05:30 +0800 Subject: [PATCH] YARN-11327. [Federation] Refactoring Yarn Router's Node Web Page. (#5009) --- .../hadoop/yarn/webapp/YarnWebParams.java | 1 + .../resourcemanager/webapp/dao/NodeInfo.java | 9 ++ .../resourcemanager/webapp/dao/NodesInfo.java | 5 + .../server/router/webapp/FederationBlock.java | 3 - .../router/webapp/MetricsOverviewTable.java | 41 ++++- .../yarn/server/router/webapp/NavBlock.java | 60 ++++++-- .../yarn/server/router/webapp/NodesBlock.java | 143 +++++++++++++----- .../yarn/server/router/webapp/NodesPage.java | 4 +- .../server/router/webapp/RouterBlock.java | 84 +++++++++- .../server/router/webapp/RouterWebApp.java | 4 +- .../webapp/dao/RouterClusterMetrics.java | 13 ++ .../router/webapp/TestFederationWebApp.java | 18 +++ 12 files changed, 314 insertions(+), 71 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java index ee9100f8e78..67d9b8512f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java @@ -39,6 +39,7 @@ public interface YarnWebParams { String QUEUE_NAME = "queue.name"; String NODE_STATE = "node.state"; String NODE_LABEL = "node.label"; + String NODE_SC = "node.subcluster"; String WEB_UI_TYPE = "web.ui.type"; String NEXT_REFRESH_INTERVAL = "next.refresh.interval"; String ERROR_MESSAGE = "error.message"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index ad7de729f03..6f36b52627b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -68,6 +68,7 @@ public class NodeInfo { protected ResourceInfo availableResource; protected NodeAttributesInfo nodeAttributesInfo; private ResourceInfo totalResource; + private String subClusterId; public NodeInfo() { } // JAXB needs this @@ -287,4 +288,12 @@ public class NodeInfo { public ResourceInfo getTotalResource() { return this.totalResource; } + + public String getSubClusterId() { + return subClusterId; + } + + public void setSubClusterId(String subClusterId) { + this.subClusterId = subClusterId; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodesInfo.java index 8174be0ad0f..6b66e0e1d70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodesInfo.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import java.util.ArrayList; +import java.util.Collection; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -43,4 +44,8 @@ public class NodesInfo { public void addAll(ArrayList nodesInfo) { node.addAll(nodesInfo); } + + public void addAll(Collection nodesInfo) { + node.addAll(nodesInfo); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java index 9e449a46e29..f80442714f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java @@ -112,9 +112,6 @@ class FederationBlock extends RouterBlock { private void initHtmlPageFederation(Block html, boolean isEnabled) { List> lists = new ArrayList<>(); - // If Yarn Federation is not enabled, the user needs to be prompted. - initUserHelpInformationDiv(html, isEnabled); - // Table header TBODY> tbody = html.table("#rms").$class("cell-border").$style("width:100%").thead().tr() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java index ba17fa27ff4..1a157a10ce0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import java.io.IOException; +import java.util.Collection; import java.util.List; public class MetricsOverviewTable extends RouterBlock { @@ -58,7 +59,31 @@ public class MetricsOverviewTable extends RouterBlock { try { initFederationClusterAppsMetrics(div, routerClusterMetrics); initFederationClusterNodesMetrics(div, routerClusterMetrics); - initFederationClusterSchedulersMetrics(div, routerClusterMetrics); + List subClusters = getSubClusterInfoList(); + initFederationClusterSchedulersMetrics(div, routerClusterMetrics, subClusters); + } catch (Exception e) { + LOG.error("MetricsOverviewTable init error.", e); + } + div.__(); + } + + protected void render(Block html, String subClusterId) { + // Initialize page styles + html.style(".metrics {margin-bottom:5px}"); + + // get subClusterId ClusterMetrics Info + ClusterMetricsInfo clusterMetricsInfo = + getClusterMetricsInfoBySubClusterId(subClusterId); + RouterClusterMetrics routerClusterMetrics = + new RouterClusterMetrics(clusterMetricsInfo, subClusterId); + + // metrics div + Hamlet.DIV div = html.div().$class("metrics"); + try { + initFederationClusterAppsMetrics(div, routerClusterMetrics); + initFederationClusterNodesMetrics(div, routerClusterMetrics); + Collection subClusters = getSubClusterInfoList(subClusterId); + initFederationClusterSchedulersMetrics(div, routerClusterMetrics, subClusters); } catch (Exception e) { LOG.error("MetricsOverviewTable init error.", e); } @@ -74,7 +99,7 @@ public class MetricsOverviewTable extends RouterBlock { */ private void initFederationClusterAppsMetrics(Hamlet.DIV div, RouterClusterMetrics metrics) { - div.h3("Federation Cluster Metrics"). + div.h3(metrics.getWebPageTitlePrefix() + " Cluster Metrics"). table("#metricsoverview"). thead().$class("ui-widget-header"). // Initialize table header information @@ -116,7 +141,7 @@ public class MetricsOverviewTable extends RouterBlock { */ private void initFederationClusterNodesMetrics(Hamlet.DIV div, RouterClusterMetrics metrics) { - div.h3("Federation Cluster Nodes Metrics"). + div.h3(metrics.getWebPageTitlePrefix() + " Cluster Nodes Metrics"). table("#nodemetricsoverview"). thead().$class("ui-widget-header"). // Initialize table header information @@ -149,17 +174,17 @@ public class MetricsOverviewTable extends RouterBlock { * * @param div data display div. * @param metrics data metric information. + * @param subclusters active subcluster List. * @throws YarnException yarn error. * @throws IOException io error. * @throws InterruptedException interrupt error. */ private void initFederationClusterSchedulersMetrics(Hamlet.DIV div, - RouterClusterMetrics metrics) throws YarnException, IOException, InterruptedException { - // Sort the SubClusters. - List subclusters = getSubClusterInfoList(); + RouterClusterMetrics metrics, Collection subclusters) + throws YarnException, IOException, InterruptedException { Hamlet.TBODY>> fsMetricsScheduleTr = - div.h3("Federation Scheduler Metrics"). + div.h3(metrics.getWebPageTitlePrefix() + " Scheduler Metrics"). table("#schedulermetricsoverview"). thead().$class("ui-widget-header"). tr(). @@ -202,7 +227,7 @@ public class MetricsOverviewTable extends RouterBlock { private void initSubClusterOverViewTable(RouterClusterMetrics metrics, Hamlet.TBODY>> fsMetricsScheduleTr, - List subclusters) { + Collection subclusters) { // configuration Configuration config = this.router.getConfig(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java index 9c39bb7b7a2..44a9ab6a512 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java @@ -18,29 +18,59 @@ package org.apache.hadoop.yarn.server.router.webapp; -import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import com.google.inject.Inject; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.server.webapp.WebPageUtils; +import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; + +import java.util.List; /** * Navigation block for the Router Web UI. */ -public class NavBlock extends HtmlBlock { +public class NavBlock extends RouterBlock { + + private Router router; + + @Inject + public NavBlock(Router router, ViewContext ctx) { + super(router, ctx); + this.router = router; + } @Override public void render(Block html) { - html. - div("#nav"). + Hamlet.UL> mainList = html.div("#nav"). h3("Cluster"). ul(). - li().a(url(""), "About").__(). - li().a(url("federation"), "Federation").__(). - li().a(url("nodes"), "Nodes").__(). - li().a(url("apps"), "Applications").__(). - __(). - h3("Tools"). - ul(). - li().a("/conf", "Configuration").__(). - li().a("/logs", "Local logs").__(). - li().a("/stacks", "Server stacks").__(). - li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__(); + li().a(url(""), "About").__(). + li().a(url("federation"), "Federation").__(); + + List subClusterIds = getActiveSubClusterIds(); + + Hamlet.UL>>> subAppsList1 = + mainList.li().a(url("nodes"), "Nodes").ul().$style("padding:0.3em 1em 0.1em 2em"); + + // ### nodes info + subAppsList1.li().__(); + for (String subClusterId : subClusterIds) { + subAppsList1.li().a(url("nodes", subClusterId), subClusterId).__(); + } + subAppsList1.__().__(); + + // ### applications info + mainList.li().a(url("apps"), "Applications").__(); + + // ### tools + Hamlet.DIV sectionBefore = mainList.__(); + Configuration conf = new Configuration(); + Hamlet.UL> tools = WebPageUtils.appendToolSection(sectionBefore, conf); + + if (tools == null) { + return; + } + + tools.__().__(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java index 4734cf6bbf3..61f72fb2b2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java @@ -19,50 +19,107 @@ package org.apache.hadoop.yarn.server.router.webapp; import com.sun.jersey.api.client.Client; +import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; import org.apache.hadoop.yarn.server.router.Router; -import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TR; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; -import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import com.google.inject.Inject; +import java.util.Date; + +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC; + /** * Nodes block for the Router Web UI. */ -public class NodesBlock extends HtmlBlock { - - private static final long BYTES_IN_MB = 1024 * 1024; +public class NodesBlock extends RouterBlock { private final Router router; @Inject NodesBlock(Router router, ViewContext ctx) { - super(ctx); + super(router, ctx); this.router = router; } @Override protected void render(Block html) { - // Get the node info from the federation + + boolean isEnabled = isYarnFederationEnabled(); + + // Get subClusterName + String subClusterName = $(NODE_SC); + + // We will try to get the subClusterName. + // If the subClusterName is not empty, + // it means that we need to get the Node list of a subCluster. + NodesInfo nodesInfo = null; + if (subClusterName != null && !subClusterName.isEmpty()) { + initSubClusterMetricsOverviewTable(html, subClusterName); + nodesInfo = getSubClusterNodesInfo(subClusterName); + } else { + // Metrics Overview Table + html.__(MetricsOverviewTable.class); + nodesInfo = getYarnFederationNodesInfo(isEnabled); + } + + // Initialize NodeInfo List + initYarnFederationNodesOfCluster(nodesInfo, html); + } + + private NodesInfo getYarnFederationNodesInfo(boolean isEnabled) { + if (isEnabled) { + String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.router.getConfig()); + return getSubClusterNodesInfoByWebAddress(webAddress); + } + return null; + } + + private NodesInfo getSubClusterNodesInfo(String subCluster) { + try { + SubClusterId subClusterId = SubClusterId.newInstance(subCluster); + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId); + + if (subClusterInfo != null) { + // Prepare webAddress + String webAddress = subClusterInfo.getRMWebServiceAddress(); + String herfWebAppAddress = ""; + if (webAddress != null && !webAddress.isEmpty()) { + herfWebAppAddress = + WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + webAddress; + return getSubClusterNodesInfoByWebAddress(herfWebAppAddress); + } + } + } catch (Exception e) { + LOG.error("get NodesInfo From SubCluster = {} error.", subCluster, e); + } + return null; + } + + private NodesInfo getSubClusterNodesInfoByWebAddress(String webAddress) { Configuration conf = this.router.getConfig(); Client client = RouterWebServiceUtil.createJerseyClient(conf); - String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); NodesInfo nodes = RouterWebServiceUtil - .genericForward(webAppAddress, null, NodesInfo.class, HTTPMethods.GET, - RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf, - client); - - setTitle("Nodes"); + .genericForward(webAddress, null, NodesInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf, + client); + return nodes; + } + private void initYarnFederationNodesOfCluster(NodesInfo nodesInfo, Block html) { TBODY> tbody = html.table("#nodes").thead().tr() .th(".nodelabels", "Node Labels") .th(".rack", "Rack") @@ -79,34 +136,42 @@ public class NodesBlock extends HtmlBlock { .th(".nodeManagerVersion", "Version") .__().__().tbody(); - // Add nodes to the web UI - for (NodeInfo info : nodes.getNodes()) { - int usedMemory = (int) info.getUsedMemory(); - int availableMemory = (int) info.getAvailableMemory(); - TR>> row = tbody.tr(); - row.td().__(StringUtils.join(",", info.getNodeLabels())).__(); - row.td().__(info.getRack()).__(); - row.td().__(info.getState()).__(); - row.td().__(info.getNodeId()).__(); - boolean isInactive = false; - if (isInactive) { - row.td().__("N/A").__(); - } else { - String httpAddress = info.getNodeHTTPAddress(); - row.td().a("//" + httpAddress, httpAddress).__(); + if (nodesInfo != null && CollectionUtils.isNotEmpty(nodesInfo.getNodes())) { + for (NodeInfo info : nodesInfo.getNodes()) { + int usedMemory = (int) info.getUsedMemory(); + int availableMemory = (int) info.getAvailableMemory(); + TR>> row = tbody.tr(); + row.td().__(StringUtils.join(",", info.getNodeLabels())).__(); + row.td().__(info.getRack()).__(); + row.td().__(info.getState()).__(); + row.td().__(info.getNodeId()).__(); + boolean isInactive = false; + if (isInactive) { + row.td().__(UNAVAILABLE).__(); + } else { + String httpAddress = info.getNodeHTTPAddress(); + String herfWebAppAddress = ""; + if (httpAddress != null && !httpAddress.isEmpty()) { + herfWebAppAddress = + WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + httpAddress; + } + row.td().a(herfWebAppAddress, httpAddress).__(); + } + + row.td().br().$title(String.valueOf(info.getLastHealthUpdate())).__() + .__(new Date(info.getLastHealthUpdate())).__() + .td(info.getHealthReport()) + .td(String.valueOf(info.getNumContainers())).td().br() + .$title(String.valueOf(usedMemory)).__() + .__(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)).__().td().br() + .$title(String.valueOf(availableMemory)).__() + .__(StringUtils.byteDesc(availableMemory * BYTES_IN_MB)).__() + .td(String.valueOf(info.getUsedVirtualCores())) + .td(String.valueOf(info.getAvailableVirtualCores())) + .td(info.getVersion()).__(); } - row.td().br().$title(String.valueOf(info.getLastHealthUpdate())).__() - .__(Times.format(info.getLastHealthUpdate())).__() - .td(info.getHealthReport()) - .td(String.valueOf(info.getNumContainers())).td().br() - .$title(String.valueOf(usedMemory)).__() - .__(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)).__().td().br() - .$title(String.valueOf(availableMemory)).__() - .__(StringUtils.byteDesc(availableMemory * BYTES_IN_MB)).__() - .td(String.valueOf(info.getUsedVirtualCores())) - .td(String.valueOf(info.getAvailableVirtualCores())) - .td(info.getVersion()).__(); } + tbody.__().__(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java index 7b2a3da7650..0723cff792d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java @@ -18,7 +18,7 @@ package org.apache.hadoop.yarn.server.router.webapp; -import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; @@ -31,7 +31,7 @@ class NodesPage extends RouterView { @Override protected void preHead(Page.HTML<__> html) { commonPreHead(html); - String type = $(NODE_STATE); + String type = $(NODE_SC); String title = "Nodes of the cluster"; if (type != null && !type.isEmpty()) { title = title + " (" + type + ")"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java index de5d62edf1f..0a03b25d79d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java @@ -33,16 +33,23 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import java.util.List; import java.util.ArrayList; import java.util.Map; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; public abstract class RouterBlock extends HtmlBlock { private final Router router; + private final ViewContext ctx; + private final FederationStateStoreFacade facade; + private final Configuration conf; public RouterBlock(Router router, ViewContext ctx) { super(ctx); + this.ctx = ctx; this.router = router; + this.facade = FederationStateStoreFacade.getInstance(); + this.conf = this.router.getConfig(); } /** @@ -51,7 +58,6 @@ public abstract class RouterBlock extends HtmlBlock { * @return Router ClusterMetricsInfo. */ protected ClusterMetricsInfo getRouterClusterMetricsInfo() { - Configuration conf = this.router.getConfig(); boolean isEnabled = isYarnFederationEnabled(); if(isEnabled) { String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf); @@ -60,6 +66,7 @@ public abstract class RouterBlock extends HtmlBlock { .genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, conf, client); + client.destroy(); return metrics; } return null; @@ -72,7 +79,7 @@ public abstract class RouterBlock extends HtmlBlock { * @throws YarnException if the call to the getSubClusters is unsuccessful. */ protected List getSubClusterInfoList() throws YarnException { - FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + Map subClustersInfo = facade.getSubClusters(true); // Sort the SubClusters. @@ -90,10 +97,81 @@ public abstract class RouterBlock extends HtmlBlock { * @return true, enable yarn federation; false, not enable yarn federation; */ protected boolean isYarnFederationEnabled() { - Configuration conf = this.router.getConfig(); boolean isEnabled = conf.getBoolean( YarnConfiguration.FEDERATION_ENABLED, YarnConfiguration.DEFAULT_FEDERATION_ENABLED); return isEnabled; } + + /** + * Get a list of SubClusterIds for ActiveSubClusters. + * + * @return list of SubClusterIds. + */ + protected List getActiveSubClusterIds() { + List result = new ArrayList<>(); + try { + Map subClustersInfo = facade.getSubClusters(true); + subClustersInfo.values().stream().forEach(subClusterInfo -> { + result.add(subClusterInfo.getSubClusterId().getId()); + }); + } catch (Exception e) { + LOG.error("getActiveSubClusters error.", e); + } + return result; + } + + /** + * init SubCluster MetricsOverviewTable. + * + * @param html HTML Object. + * @param subclusterId subClusterId + */ + protected void initSubClusterMetricsOverviewTable(Block html, String subclusterId) { + MetricsOverviewTable metricsOverviewTable = new MetricsOverviewTable(this.router, this.ctx); + metricsOverviewTable.render(html, subclusterId); + } + + /** + * Get ClusterMetricsInfo By SubClusterId. + * + * @param subclusterId subClusterId + * @return SubCluster RM ClusterMetricsInfo + */ + protected ClusterMetricsInfo getClusterMetricsInfoBySubClusterId(String subclusterId) { + try { + SubClusterId subClusterId = SubClusterId.newInstance(subclusterId); + SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId); + if (subClusterInfo != null) { + Client client = RouterWebServiceUtil.createJerseyClient(this.conf); + // Call the RM interface to obtain schedule information + String webAppAddress = WebAppUtils.getHttpSchemePrefix(this.conf) + + subClusterInfo.getRMWebServiceAddress(); + ClusterMetricsInfo metrics = RouterWebServiceUtil + .genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null, + conf, client); + client.destroy(); + return metrics; + } + } catch (Exception e) { + LOG.error("getClusterMetricsInfoBySubClusterId subClusterId = {} error.", subclusterId, e); + } + return null; + } + + protected Collection getSubClusterInfoList(String subclusterId) { + try { + SubClusterId subClusterId = SubClusterId.newInstance(subclusterId); + SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId); + return Collections.singletonList(subClusterInfo); + } catch (Exception e) { + LOG.error("getSubClusterInfoList subClusterId = {} error.", subclusterId, e); + } + return null; + } + + public FederationStateStoreFacade getFacade() { + return facade; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java index ba07a1afba4..d9a9a5896ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java @@ -24,6 +24,8 @@ import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.YarnWebParams; +import static org.apache.hadoop.yarn.util.StringHelper.pajoin; + /** * The Router webapp. */ @@ -48,7 +50,7 @@ public class RouterWebApp extends WebApp implements YarnWebParams { route("/cluster", RouterController.class, "about"); route("/about", RouterController.class, "about"); route("/apps", RouterController.class, "apps"); - route("/nodes", RouterController.class, "nodes"); + route(pajoin("/nodes", NODE_SC), RouterController.class, "nodes"); route("/federation", RouterController.class, "federation"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterClusterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterClusterMetrics.java index 46e9e89ac18..f06f85574db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterClusterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterClusterMetrics.java @@ -35,6 +35,9 @@ public class RouterClusterMetrics { protected static final long BYTES_IN_MB = 1024 * 1024; private static final Logger LOG = LoggerFactory.getLogger(RouterClusterMetrics.class); + // webPageTitlePrefix + private String webPageTitlePrefix = "Federation"; + // Application Information. private String appsSubmitted = "N/A"; private String appsCompleted = "N/A"; @@ -99,6 +102,12 @@ public class RouterClusterMetrics { } } + public RouterClusterMetrics(ClusterMetricsInfo metrics, + String webPageTitlePrefix) { + this(metrics); + this.webPageTitlePrefix = webPageTitlePrefix; + } + // Get Key Metric Information public String getAppsSubmitted() { return appsSubmitted; @@ -307,4 +316,8 @@ public class RouterClusterMetrics { LOG.error("conversionNodeInformation error.", e); } } + + public String getWebPageTitlePrefix() { + return webPageTitlePrefix; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java index 55f23db72b2..860e266ced1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java @@ -63,4 +63,22 @@ public class TestFederationWebApp extends TestRouterWebServicesREST { config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false); WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config)); } + + @Test + public void testFederationNodeViewEnable() + throws InterruptedException, YarnException, IOException { + // Test Federation Enabled + Configuration config = new YarnConfiguration(); + config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + WebAppTests.testPage(NodesPage.class, Router.class, new MockRouter(config)); + } + + @Test + public void testFederationNodeViewNotEnable() + throws InterruptedException, YarnException, IOException { + // Test Federation Not Enabled + Configuration config = new YarnConfiguration(); + config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false); + WebAppTests.testPage(NodesPage.class, Router.class, new MockRouter(config)); + } }