YARN-11317. [Federation] Refactoring Yarn Router's About Web Page. (#4946)

This commit is contained in:
slfan1989 2022-10-12 04:30:48 +08:00 committed by GitHub
parent 82a88a8ae6
commit 9e16f1f883
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 1191 additions and 88 deletions

View File

@ -97,4 +97,28 @@ public abstract class HtmlBlock extends TextView implements SubView {
return callerUGI; return callerUGI;
} }
/**
* Initialize User Help Information Div.
* When the user does not configure the Yarn Federation function, prompt the user.
*
* @param html HTML page.
* @param isEnabled If federation is enabled.
*/
protected void initUserHelpInformationDiv(Block html, boolean isEnabled) {
if (!isEnabled) {
html.style(".alert {padding: 15px; margin-bottom: 20px; " +
" border: 1px solid transparent; border-radius: 4px;}");
html.style(".alert-dismissable {padding-right: 35px;}");
html.style(".alert-info {color: #856404;background-color: #fff3cd;border-color: #ffeeba;}");
Hamlet.DIV<Hamlet> div = html.div("#div_id").$class("alert alert-dismissable alert-info");
div.p().$style("color:red").__("Federation is not Enabled.").__()
.p().__()
.p().__("We can refer to the following documents to configure Yarn Federation. ").__()
.p().__()
.a("https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/Federation.html",
"Hadoop: YARN Federation").
__();
}
}
} }

View File

@ -57,6 +57,9 @@ public final class RMWSConsts {
/** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */ /** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */
public static final String SCHEDULER_LOGS = "/scheduler/logs"; public static final String SCHEDULER_LOGS = "/scheduler/logs";
/** Path for {@code RMWebServiceProtocol#getSchedulerOverview}. */
public static final String SCHEDULER_OVERVIEW = "/scheduler-overview";
/** /**
* Path for {@code RMWebServiceProtocol#validateAndGetSchedulerConfiguration}. * Path for {@code RMWebServiceProtocol#validateAndGetSchedulerConfiguration}.
*/ */

View File

@ -202,6 +202,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesIn
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webapp.WebServices; import org.apache.hadoop.yarn.server.webapp.WebServices;
@ -2942,4 +2943,14 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
} }
return Response.status(Status.OK).build(); return Response.status(Status.OK).build();
} }
@GET
@Path(RMWSConsts.SCHEDULER_OVERVIEW)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public SchedulerOverviewInfo getSchedulerOverview() {
initForReadableEndpoints();
ResourceScheduler rs = rm.getResourceScheduler();
return new SchedulerOverviewInfo(rs);
}
} }

View File

@ -0,0 +1,118 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.List;
@XmlRootElement(name = "scheduler")
@XmlAccessorType(XmlAccessType.FIELD)
public class SchedulerOverviewInfo {
private String schedulerType;
private String schedulingResourceType;
private ResourceInfo minimumAllocation;
private ResourceInfo maximumAllocation;
private int applicationPriority;
private int schedulerBusy;
private int rmDispatcherEventQueueSize;
private int schedulerDispatcherEventQueueSize;
// JAXB needs this
public SchedulerOverviewInfo() {
}
public SchedulerOverviewInfo(ResourceScheduler rs) {
// Parse the schedule type
this.schedulerType = getSchedulerName(rs);
// Parse and allocate resource information
this.minimumAllocation = new ResourceInfo(rs.getMinimumResourceCapability());
this.maximumAllocation = new ResourceInfo(rs.getMaximumResourceCapability());
// Parse App Priority
this.applicationPriority = rs.getMaxClusterLevelAppPriority().getPriority();
// Resolving resource types
List<ResourceTypeInfo> resourceTypeInfos = ResourceUtils.getResourcesTypeInfo();
resourceTypeInfos.sort((o1, o2) -> o1.getName().compareToIgnoreCase(o2.getName()));
this.schedulingResourceType = StringUtils.join(resourceTypeInfos, ",");
// clusterMetrics
ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(rs);
this.schedulerBusy = clusterMetrics.getRmSchedulerBusyPercent();
this.rmDispatcherEventQueueSize = clusterMetrics.getRmEventQueueSize();
this.schedulerDispatcherEventQueueSize = clusterMetrics.getSchedulerEventQueueSize();
}
private static String getSchedulerName(ResourceScheduler rs) {
if (rs instanceof CapacityScheduler) {
return "Capacity Scheduler";
}
if (rs instanceof FairScheduler) {
return "Fair Scheduler";
}
if (rs instanceof FifoScheduler) {
return "Fifo Scheduler";
}
return rs.getClass().getSimpleName();
}
public String getSchedulerType() {
return schedulerType;
}
public String getSchedulingResourceType() {
return schedulingResourceType;
}
public ResourceInfo getMinimumAllocation() {
return minimumAllocation;
}
public ResourceInfo getMaximumAllocation() {
return maximumAllocation;
}
public int getApplicationPriority() {
return applicationPriority;
}
public int getSchedulerBusy() {
return schedulerBusy;
}
public int getRmDispatcherEventQueueSize() {
return rmDispatcherEventQueueSize;
}
public int getSchedulerDispatcherEventQueueSize() {
return schedulerDispatcherEventQueueSize;
}
}

View File

@ -1099,4 +1099,61 @@ public class TestRMWebServices extends JerseyTestBase {
return webService; return webService;
} }
@Test
public void testClusterSchedulerOverviewFifo() throws JSONException, Exception {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
verifyClusterSchedulerOverView(json, "Fifo Scheduler");
}
public static void verifyClusterSchedulerOverView(
JSONObject json, String expectedSchedulerType) throws Exception {
// why json contains 8 elements because we defined 8 fields
assertEquals("incorrect number of elements in: " + json, 8, json.length());
// 1.Verify that the schedulerType is as expected
String schedulerType = json.getString("schedulerType");
assertEquals(expectedSchedulerType, schedulerType);
// 2.Verify that schedulingResourceType is as expected
String schedulingResourceType = json.getString("schedulingResourceType");
assertEquals("memory-mb (unit=Mi),vcores", schedulingResourceType);
// 3.Verify that minimumAllocation is as expected
JSONObject minimumAllocation = json.getJSONObject("minimumAllocation");
String minMemory = minimumAllocation.getString("memory");
String minVCores = minimumAllocation.getString("vCores");
assertEquals("1024", minMemory);
assertEquals("1", minVCores);
// 4.Verify that maximumAllocation is as expected
JSONObject maximumAllocation = json.getJSONObject("maximumAllocation");
String maxMemory = maximumAllocation.getString("memory");
String maxVCores = maximumAllocation.getString("vCores");
assertEquals("8192", maxMemory);
assertEquals("4", maxVCores);
// 5.Verify that schedulerBusy is as expected
int schedulerBusy = json.getInt("schedulerBusy");
assertEquals(-1, schedulerBusy);
// 6.Verify that rmDispatcherEventQueueSize is as expected
int rmDispatcherEventQueueSize = json.getInt("rmDispatcherEventQueueSize");
assertEquals(0, rmDispatcherEventQueueSize);
// 7.Verify that schedulerDispatcherEventQueueSize is as expected
int schedulerDispatcherEventQueueSize = json.getInt("schedulerDispatcherEventQueueSize");
assertEquals(0, schedulerDispatcherEventQueueSize);
// 8.Verify that applicationPriority is as expected
int applicationPriority = json.getInt("applicationPriority");
assertEquals(0, applicationPriority);
}
} }

View File

@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.Guice; import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule; import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor; import com.sun.jersey.test.framework.WebAppDescriptor;
@ -413,4 +414,16 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
return new MockRM(conf); return new MockRM(conf);
} }
@Test
public void testClusterSchedulerOverviewCapacity() throws Exception {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
TestRMWebServices.verifyClusterSchedulerOverView(json, "Capacity Scheduler");
}
} }

View File

@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager
import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServices;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@ -157,4 +158,15 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
assertEquals("root", rootQueue.getString("queueName")); assertEquals("root", rootQueue.getString("queueName"));
} }
@Test
public void testClusterSchedulerOverviewFair() throws Exception {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
TestRMWebServices.verifyClusterSchedulerOverView(json, "Fair Scheduler");
}
} }

View File

@ -79,6 +79,7 @@ public class Router extends CompositeService {
private WebApp webApp; private WebApp webApp;
@VisibleForTesting @VisibleForTesting
protected String webAppAddress; protected String webAppAddress;
private static long clusterTimeStamp = System.currentTimeMillis();
/** /**
* Priority of the Router shutdown hook. * Priority of the Router shutdown hook.
@ -237,4 +238,8 @@ public class Router extends CompositeService {
} }
return name; return name;
} }
public static long getClusterTimeStamp() {
return clusterTimeStamp;
}
} }

View File

@ -18,15 +18,11 @@
package org.apache.hadoop.yarn.server.router.webapp; package org.apache.hadoop.yarn.server.router.webapp;
import com.sun.jersey.api.client.Client; import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
import org.apache.hadoop.yarn.server.router.Router; import org.apache.hadoop.yarn.server.router.Router;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.server.router.webapp.dao.RouterInfo;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject; import com.google.inject.Inject;
@ -34,62 +30,57 @@ import com.google.inject.Inject;
/** /**
* About block for the Router Web UI. * About block for the Router Web UI.
*/ */
public class AboutBlock extends HtmlBlock { public class AboutBlock extends RouterBlock {
private static final long BYTES_IN_MB = 1024 * 1024;
private final Router router; private final Router router;
@Inject @Inject
AboutBlock(Router router, ViewContext ctx) { AboutBlock(Router router, ViewContext ctx) {
super(ctx); super(router, ctx);
this.router = router; this.router = router;
} }
@Override @Override
protected void render(Block html) { protected void render(Block html) {
Configuration conf = this.router.getConfig();
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
Client client = RouterWebServiceUtil.createJerseyClient(conf);
ClusterMetricsInfo metrics = RouterWebServiceUtil boolean isEnabled = isYarnFederationEnabled();
.genericForward(webAppAddress, null, ClusterMetricsInfo.class,
HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
conf, client);
boolean isEnabled = conf.getBoolean(
YarnConfiguration.FEDERATION_ENABLED,
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
info("Cluster Status").
__("Federation Enabled", isEnabled).
__("Applications Submitted", metrics.getAppsSubmitted()).
__("Applications Pending", metrics.getAppsPending()).
__("Applications Running", metrics.getAppsRunning()).
__("Applications Failed", metrics.getAppsFailed()).
__("Applications Killed", metrics.getAppsKilled()).
__("Applications Completed", metrics.getAppsCompleted()).
__("Containers Allocated", metrics.getContainersAllocated()).
__("Containers Reserved", metrics.getReservedContainers()).
__("Containers Pending", metrics.getPendingContainers()).
__("Available Memory",
StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB)).
__("Allocated Memory",
StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB)).
__("Reserved Memory",
StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB)).
__("Total Memory",
StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB)).
__("Available VirtualCores", metrics.getAvailableVirtualCores()).
__("Allocated VirtualCores", metrics.getAllocatedVirtualCores()).
__("Reserved VirtualCores", metrics.getReservedVirtualCores()).
__("Total VirtualCores", metrics.getTotalVirtualCores()).
__("Active Nodes", metrics.getActiveNodes()).
__("Lost Nodes", metrics.getLostNodes()).
__("Available Nodes", metrics.getDecommissionedNodes()).
__("Unhealthy Nodes", metrics.getUnhealthyNodes()).
__("Rebooted Nodes", metrics.getRebootedNodes()).
__("Total Nodes", metrics.getTotalNodes());
// If Yarn Federation is not enabled, the user needs to be prompted.
initUserHelpInformationDiv(html, isEnabled);
// Metrics Overview Table
html.__(MetricsOverviewTable.class);
// Init Yarn Router Basic Information
initYarnRouterBasicInformation(isEnabled);
// InfoBlock
html.__(InfoBlock.class); html.__(InfoBlock.class);
} }
/**
* Init Yarn Router Basic Infomation.
* @param isEnabled true, federation is enabled; false, federation is not enabled.
*/
private void initYarnRouterBasicInformation(boolean isEnabled) {
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
RouterInfo routerInfo = new RouterInfo(router);
String lastStartTime =
DateFormatUtils.format(routerInfo.getStartedOn(), DATE_PATTERN);
try {
info("Yarn Router Overview").
__("Federation Enabled:", String.valueOf(isEnabled)).
__("Router ID:", routerInfo.getClusterId()).
__("Router state:", routerInfo.getState()).
__("Router SubCluster Count:", facade.getSubClusters(true).size()).
__("Router RMStateStore:", routerInfo.getRouterStateStore()).
__("Router started on:", lastStartTime).
__("Router version:", routerInfo.getRouterBuildVersion() +
" on " + routerInfo.getRouterVersionBuiltOn()).
__("Hadoop version:", routerInfo.getHadoopBuildVersion() +
" on " + routerInfo.getHadoopVersionBuiltOn());
} catch (YarnException e) {
LOG.error("initYarnRouterBasicInformation error.", e);
}
}
} }

View File

@ -20,50 +20,41 @@ package org.apache.hadoop.yarn.server.router.webapp;
import java.io.StringReader; import java.io.StringReader;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.HashMap; import java.util.HashMap;
import com.google.gson.Gson; import com.google.gson.Gson;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils; import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
import org.apache.hadoop.yarn.server.router.Router; import org.apache.hadoop.yarn.server.router.Router;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject; import com.google.inject.Inject;
import com.sun.jersey.api.json.JSONConfiguration; import com.sun.jersey.api.json.JSONConfiguration;
import com.sun.jersey.api.json.JSONJAXBContext; import com.sun.jersey.api.json.JSONJAXBContext;
import com.sun.jersey.api.json.JSONUnmarshaller; import com.sun.jersey.api.json.JSONUnmarshaller;
class FederationBlock extends HtmlBlock { class FederationBlock extends RouterBlock {
private final Router router; private final Router router;
@Inject @Inject
FederationBlock(ViewContext ctx, Router router) { FederationBlock(ViewContext ctx, Router router) {
super(ctx); super(router, ctx);
this.router = router; this.router = router;
} }
@Override @Override
public void render(Block html) { public void render(Block html) {
Configuration conf = this.router.getConfig(); boolean isEnabled = isYarnFederationEnabled();
boolean isEnabled = conf.getBoolean(
YarnConfiguration.FEDERATION_ENABLED,
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
// init Html Page Federation // init Html Page Federation
initHtmlPageFederation(html, isEnabled); initHtmlPageFederation(html, isEnabled);
@ -122,21 +113,7 @@ class FederationBlock extends HtmlBlock {
List<Map<String, String>> lists = new ArrayList<>(); List<Map<String, String>> lists = new ArrayList<>();
// If Yarn Federation is not enabled, the user needs to be prompted. // If Yarn Federation is not enabled, the user needs to be prompted.
if (!isEnabled) { initUserHelpInformationDiv(html, isEnabled);
html.style(".alert {padding: 15px; margin-bottom: 20px; " +
" border: 1px solid transparent; border-radius: 4px;}");
html.style(".alert-dismissable {padding-right: 35px;}");
html.style(".alert-info {color: #856404;background-color: #fff3cd;border-color: #ffeeba;}");
Hamlet.DIV<Hamlet> div = html.div("#div_id").$class("alert alert-dismissable alert-info");
div.p().$style("color:red").__("Federation is not Enabled.").__()
.p().__()
.p().__("We can refer to the following documents to configure Yarn Federation. ").__()
.p().__()
.a("https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/Federation.html",
"Hadoop: YARN Federation").
__();
}
// Table header // Table header
TBODY<TABLE<Hamlet>> tbody = TBODY<TABLE<Hamlet>> tbody =
@ -150,16 +127,9 @@ class FederationBlock extends HtmlBlock {
.__().__().tbody(); .__().__().tbody();
try { try {
// Binding to the FederationStateStore
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true);
// Sort the SubClusters // Sort the SubClusters
List<SubClusterInfo> subclusters = new ArrayList<>(); List<SubClusterInfo> subclusters = getSubClusterInfoList();
subclusters.addAll(subClustersInfo.values());
Comparator<? super SubClusterInfo> cmp = Comparator.comparing(o -> o.getSubClusterId());
Collections.sort(subclusters, cmp);
for (SubClusterInfo subcluster : subclusters) { for (SubClusterInfo subcluster : subclusters) {

View File

@ -0,0 +1,239 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.router.webapp;
import com.google.inject.Inject;
import com.sun.jersey.api.client.Client;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
import org.apache.hadoop.yarn.server.router.Router;
import org.apache.hadoop.yarn.server.router.webapp.dao.RouterClusterMetrics;
import org.apache.hadoop.yarn.server.router.webapp.dao.RouterSchedulerMetrics;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import java.io.IOException;
import java.util.List;
public class MetricsOverviewTable extends RouterBlock {
private final Router router;
@Inject
MetricsOverviewTable(Router router, ViewContext ctx) {
super(router, ctx);
this.router = router;
}
@Override
protected void render(Block html) {
// Initialize page styles
html.style(".metrics {margin-bottom:5px}");
// get routerClusterMetrics Info
ClusterMetricsInfo routerClusterMetricsInfo = getRouterClusterMetricsInfo();
RouterClusterMetrics routerClusterMetrics = new RouterClusterMetrics(routerClusterMetricsInfo);
// metrics div
Hamlet.DIV<Hamlet> div = html.div().$class("metrics");
try {
initFederationClusterAppsMetrics(div, routerClusterMetrics);
initFederationClusterNodesMetrics(div, routerClusterMetrics);
initFederationClusterSchedulersMetrics(div, routerClusterMetrics);
} catch (Exception e) {
LOG.error("MetricsOverviewTable init error.", e);
}
div.__();
}
/**
* Init Federation Cluster Apps Metrics.
* Contains App information, resource usage information.
*
* @param div data display div.
* @param metrics data metric information.
*/
private void initFederationClusterAppsMetrics(Hamlet.DIV<Hamlet> div,
RouterClusterMetrics metrics) {
div.h3("Federation Cluster Metrics").
table("#metricsoverview").
thead().$class("ui-widget-header").
// Initialize table header information
tr().
th().$class("ui-state-default").__("Apps Submitted").__().
th().$class("ui-state-default").__("Apps Pending").__().
th().$class("ui-state-default").__("Apps Running").__().
th().$class("ui-state-default").__("Apps Completed").__().
th().$class("ui-state-default").__("Containers Running").__().
th().$class("ui-state-default").__("Used Resources").__().
th().$class("ui-state-default").__("Total Resources").__().
th().$class("ui-state-default").__("Reserved Resources").__().
th().$class("ui-state-default").__("Physical Mem Used %").__().
th().$class("ui-state-default").__("Physical VCores Used %").__().
__().
__().
// Initialize table data information
tbody().$class("ui-widget-content").
tr().
td(metrics.getAppsSubmitted()).
td(metrics.getAppsPending()).
td(String.valueOf(metrics.getAppsRunning())).
td(metrics.getAppsCompleted()).
td(metrics.getAllocatedContainers()).
td(metrics.getUsedResources()).
td(metrics.getTotalResources()).
td(metrics.getReservedResources()).
td(metrics.getUtilizedMBPercent()).
td(metrics.getUtilizedVirtualCoresPercent()).
__().
__().__();
}
/**
* Init Federation Cluster Nodes Metrics.
*
* @param div data display div.
* @param metrics data metric information.
*/
private void initFederationClusterNodesMetrics(Hamlet.DIV<Hamlet> div,
RouterClusterMetrics metrics) {
div.h3("Federation Cluster Nodes Metrics").
table("#nodemetricsoverview").
thead().$class("ui-widget-header").
// Initialize table header information
tr().
th().$class("ui-state-default").__("Active Nodes").__().
th().$class("ui-state-default").__("Decommissioning Nodes").__().
th().$class("ui-state-default").__("Decommissioned Nodes").__().
th().$class("ui-state-default").__("Lost Nodes").__().
th().$class("ui-state-default").__("Unhealthy Nodes").__().
th().$class("ui-state-default").__("Rebooted Nodes").__().
th().$class("ui-state-default").__("Shutdown Nodes").__().
__().
__().
// Initialize table data information
tbody().$class("ui-widget-content").
tr().
td(String.valueOf(metrics.getActiveNodes())).
td(String.valueOf(metrics.getDecommissioningNodes())).
td(String.valueOf(metrics.getDecommissionedNodes())).
td(String.valueOf(metrics.getLostNodes())).
td(String.valueOf(metrics.getUnhealthyNodes())).
td(String.valueOf(metrics.getRebootedNodes())).
td(String.valueOf(metrics.getShutdownNodes())).
__().
__().__();
}
/**
* Init Federation Cluster SchedulersMetrics.
*
* @param div data display div.
* @param metrics data metric information.
* @throws YarnException yarn error.
* @throws IOException io error.
* @throws InterruptedException interrupt error.
*/
private void initFederationClusterSchedulersMetrics(Hamlet.DIV<Hamlet> div,
RouterClusterMetrics metrics) throws YarnException, IOException, InterruptedException {
// Sort the SubClusters.
List<SubClusterInfo> subclusters = getSubClusterInfoList();
Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr =
div.h3("Federation Scheduler Metrics").
table("#schedulermetricsoverview").
thead().$class("ui-widget-header").
tr().
th().$class("ui-state-default").__("SubCluster").__().
th().$class("ui-state-default").__("Scheduler Type").__().
th().$class("ui-state-default").__("Scheduling Resource Type").__().
th().$class("ui-state-default").__("Minimum Allocation").__().
th().$class("ui-state-default").__("Maximum Allocation").__().
th().$class("ui-state-default").__("Maximum Cluster Application Priority").__().
th().$class("ui-state-default").__("Scheduler Busy %").__().
th().$class("ui-state-default").__("RM Dispatcher EventQueue Size").__().
th().$class("ui-state-default")
.__("Scheduler Dispatcher EventQueue Size").__().
__().
__().
tbody().$class("ui-widget-content");
boolean isEnabled = isYarnFederationEnabled();
// If Federation mode is not enabled or there is currently no SubCluster available,
// each column in the list should be displayed as N/A
if (!isEnabled || subclusters == null || subclusters.isEmpty()) {
fsMetricsScheduleTr.tr().
td(UNAVAILABLE).
td(UNAVAILABLE).
td(UNAVAILABLE).
td(UNAVAILABLE).
td(UNAVAILABLE).
td(UNAVAILABLE).
td(UNAVAILABLE).
td(UNAVAILABLE).
td(UNAVAILABLE)
.__();
} else {
initSubClusterOverViewTable(metrics, fsMetricsScheduleTr, subclusters);
}
fsMetricsScheduleTr.__().__();
}
private void initSubClusterOverViewTable(RouterClusterMetrics metrics,
Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr,
List<SubClusterInfo> subclusters) {
// configuration
Configuration config = this.router.getConfig();
Client client = RouterWebServiceUtil.createJerseyClient(config);
// Traverse all SubClusters to get cluster information.
for (SubClusterInfo subcluster : subclusters) {
// Call the RM interface to obtain schedule information
String webAppAddress = WebAppUtils.getHttpSchemePrefix(config) +
subcluster.getRMWebServiceAddress();
SchedulerOverviewInfo typeInfo = RouterWebServiceUtil
.genericForward(webAppAddress, null, SchedulerOverviewInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_OVERVIEW, null, null,
config, client);
RouterSchedulerMetrics rsMetrics = new RouterSchedulerMetrics(subcluster, metrics, typeInfo);
// Basic information
fsMetricsScheduleTr.tr().
td(rsMetrics.getSubCluster()).
td(rsMetrics.getSchedulerType()).
td(rsMetrics.getSchedulingResourceType()).
td(rsMetrics.getMinimumAllocation()).
td(rsMetrics.getMaximumAllocation()).
td(rsMetrics.getApplicationPriority()).
td(rsMetrics.getSchedulerBusy()).
td(rsMetrics.getRmDispatcherEventQueueSize()).
td(rsMetrics.getSchedulerDispatcherEventQueueSize()).
__();
}
}
}

View File

@ -0,0 +1,99 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.router.webapp;
import com.sun.jersey.api.client.Client;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
import org.apache.hadoop.yarn.server.router.Router;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.Collections;
import java.util.Comparator;
public abstract class RouterBlock extends HtmlBlock {
private final Router router;
public RouterBlock(Router router, ViewContext ctx) {
super(ctx);
this.router = router;
}
/**
* Get RouterClusterMetrics Info.
*
* @return Router ClusterMetricsInfo.
*/
protected ClusterMetricsInfo getRouterClusterMetricsInfo() {
Configuration conf = this.router.getConfig();
boolean isEnabled = isYarnFederationEnabled();
if(isEnabled) {
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
Client client = RouterWebServiceUtil.createJerseyClient(conf);
ClusterMetricsInfo metrics = RouterWebServiceUtil
.genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
conf, client);
return metrics;
}
return null;
}
/**
* Get a list of subclusters.
*
* @return subcluster List.
* @throws YarnException if the call to the getSubClusters is unsuccessful.
*/
protected List<SubClusterInfo> getSubClusterInfoList() throws YarnException {
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true);
// Sort the SubClusters.
List<SubClusterInfo> subclusters = new ArrayList<>();
subclusters.addAll(subClustersInfo.values());
Comparator<? super SubClusterInfo> cmp = Comparator.comparing(o -> o.getSubClusterId());
Collections.sort(subclusters, cmp);
return subclusters;
}
/**
* Whether Yarn Federation is enabled.
*
* @return true, enable yarn federation; false, not enable yarn federation;
*/
protected boolean isYarnFederationEnabled() {
Configuration conf = this.router.getConfig();
boolean isEnabled = conf.getBoolean(
YarnConfiguration.FEDERATION_ENABLED,
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
return isEnabled;
}
}

View File

@ -0,0 +1,310 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.router.webapp.dao;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class RouterClusterMetrics {
protected static final long BYTES_IN_MB = 1024 * 1024;
private static final Logger LOG = LoggerFactory.getLogger(RouterClusterMetrics.class);
// Application Information.
private String appsSubmitted = "N/A";
private String appsCompleted = "N/A";
private String appsPending = "N/A";
private String appsRunning = "N/A";
private String appsFailed = "N/A";
private String appsKilled = "N/A";
// Memory Information.
private String totalMemory = "N/A";
private String reservedMemory = "N/A";
private String availableMemory = "N/A";
private String allocatedMemory = "N/A";
private String pendingMemory = "N/A";
// VirtualCores Information.
private String reservedVirtualCores = "N/A";
private String availableVirtualCores = "N/A";
private String allocatedVirtualCores = "N/A";
private String pendingVirtualCores = "N/A";
private String totalVirtualCores = "N/A";
// Resources Information.
private String usedResources = "N/A";
private String totalResources = "N/A";
private String reservedResources = "N/A";
private String allocatedContainers = "N/A";
// Resource Percent Information.
private String utilizedMBPercent = "N/A";
private String utilizedVirtualCoresPercent = "N/A";
// Node Information.
private String activeNodes = "N/A";
private String decommissioningNodes = "N/A";
private String decommissionedNodes = "N/A";
private String lostNodes = "N/A";
private String unhealthyNodes = "N/A";
private String rebootedNodes = "N/A";
private String shutdownNodes = "N/A";
public RouterClusterMetrics() {
}
public RouterClusterMetrics(ClusterMetricsInfo metrics) {
if (metrics != null) {
// Application Information Conversion.
conversionApplicationInformation(metrics);
// Memory Information Conversion.
conversionMemoryInformation(metrics);
// Resources Information Conversion.
conversionResourcesInformation(metrics);
// Percent Information Conversion.
conversionResourcesPercent(metrics);
// Node Information Conversion.
conversionNodeInformation(metrics);
}
}
// Get Key Metric Information
public String getAppsSubmitted() {
return appsSubmitted;
}
public String getAppsCompleted() {
return appsCompleted;
}
public String getAppsPending() {
return appsPending;
}
public String getAppsRunning() {
return appsRunning;
}
public String getAppsFailed() {
return appsFailed;
}
public String getAppsKilled() {
return appsKilled;
}
public String getTotalMemory() {
return totalMemory;
}
public String getReservedMemory() {
return reservedMemory;
}
public String getAvailableMemory() {
return availableMemory;
}
public String getAllocatedMemory() {
return allocatedMemory;
}
public String getPendingMemory() {
return pendingMemory;
}
public String getReservedVirtualCores() {
return reservedVirtualCores;
}
public String getAvailableVirtualCores() {
return availableVirtualCores;
}
public String getAllocatedVirtualCores() {
return allocatedVirtualCores;
}
public String getPendingVirtualCores() {
return pendingVirtualCores;
}
public String getTotalVirtualCores() {
return totalVirtualCores;
}
public String getUsedResources() {
return usedResources;
}
public String getTotalResources() {
return totalResources;
}
public String getReservedResources() {
return reservedResources;
}
public String getAllocatedContainers() {
return allocatedContainers;
}
public String getUtilizedMBPercent() {
return utilizedMBPercent;
}
public String getUtilizedVirtualCoresPercent() {
return utilizedVirtualCoresPercent;
}
public String getActiveNodes() {
return activeNodes;
}
public String getDecommissioningNodes() {
return decommissioningNodes;
}
public String getDecommissionedNodes() {
return decommissionedNodes;
}
public String getLostNodes() {
return lostNodes;
}
public String getUnhealthyNodes() {
return unhealthyNodes;
}
public String getRebootedNodes() {
return rebootedNodes;
}
public String getShutdownNodes() {
return shutdownNodes;
}
// Metric Information Conversion
public void conversionApplicationInformation(ClusterMetricsInfo metrics) {
try {
// Application Information.
this.appsSubmitted = String.valueOf(metrics.getAppsSubmitted());
this.appsCompleted = String.valueOf(metrics.getAppsCompleted() +
metrics.getAppsFailed() + metrics.getAppsKilled());
this.appsPending = String.valueOf(metrics.getAppsPending());
this.appsRunning = String.valueOf(metrics.getAppsRunning());
this.appsFailed = String.valueOf(metrics.getAppsFailed());
this.appsKilled = String.valueOf(metrics.getAppsKilled());
} catch (Exception e) {
LOG.error("conversionApplicationInformation error.", e);
}
}
// Metric Memory Information
public void conversionMemoryInformation(ClusterMetricsInfo metrics) {
try {
// Memory Information.
this.totalMemory = StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB);
this.reservedMemory = StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB);
this.availableMemory = StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB);
this.allocatedMemory = StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB);
this.pendingMemory = StringUtils.byteDesc(metrics.getPendingMB() * BYTES_IN_MB);
} catch (Exception e) {
LOG.error("conversionMemoryInformation error.", e);
}
}
// ResourcesInformation Conversion
public void conversionResourcesInformation(ClusterMetricsInfo metrics) {
try {
// Parse resource information from metrics.
Resource metricUsedResources;
Resource metricTotalResources;
Resource metricReservedResources;
int metricAllocatedContainers;
if (metrics.getCrossPartitionMetricsAvailable()) {
metricAllocatedContainers = metrics.getTotalAllocatedContainersAcrossPartition();
metricUsedResources = metrics.getTotalUsedResourcesAcrossPartition().getResource();
metricTotalResources = metrics.getTotalClusterResourcesAcrossPartition().getResource();
metricReservedResources = metrics.getTotalReservedResourcesAcrossPartition().getResource();
// getTotalUsedResourcesAcrossPartition includes reserved resources.
Resources.subtractFrom(metricUsedResources, metricReservedResources);
} else {
metricAllocatedContainers = metrics.getContainersAllocated();
metricUsedResources = Resource.newInstance(metrics.getAllocatedMB(),
(int) metrics.getAllocatedVirtualCores());
metricTotalResources = Resource.newInstance(metrics.getTotalMB(),
(int) metrics.getTotalVirtualCores());
metricReservedResources = Resource.newInstance(metrics.getReservedMB(),
(int) metrics.getReservedVirtualCores());
}
// Convert to standard format.
usedResources = metricUsedResources.getFormattedString();
totalResources = metricTotalResources.getFormattedString();
reservedResources = metricReservedResources.getFormattedString();
allocatedContainers = String.valueOf(metricAllocatedContainers);
} catch (Exception e) {
LOG.error("conversionResourcesInformation error.", e);
}
}
// ResourcesPercent Conversion
public void conversionResourcesPercent(ClusterMetricsInfo metrics) {
try {
this.utilizedMBPercent = String.valueOf(metrics.getUtilizedMBPercent());
this.utilizedVirtualCoresPercent = String.valueOf(metrics.getUtilizedVirtualCoresPercent());
} catch (Exception e) {
LOG.error("conversionResourcesPercent error.", e);
}
}
// NodeInformation Conversion
public void conversionNodeInformation(ClusterMetricsInfo metrics) {
try {
this.activeNodes = String.valueOf(metrics.getActiveNodes());
this.decommissioningNodes = String.valueOf(metrics.getDecommissioningNodes());
this.decommissionedNodes = String.valueOf(metrics.getDecommissionedNodes());
this.lostNodes = String.valueOf(metrics.getLostNodes());
this.unhealthyNodes = String.valueOf(metrics.getUnhealthyNodes());
this.rebootedNodes = String.valueOf(metrics.getRebootedNodes());
this.shutdownNodes = String.valueOf(metrics.getShutdownNodes());
} catch (Exception e) {
LOG.error("conversionNodeInformation error.", e);
}
}
}

View File

@ -0,0 +1,104 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.router.webapp.dao;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.router.Router;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class RouterInfo {
private long id;
private long startedOn;
private Service.STATE state;
private String routerStateStoreName;
private String routerVersion;
private String routerBuildVersion;
private String routerVersionBuiltOn;
private String hadoopVersion;
private String hadoopBuildVersion;
private String hadoopVersionBuiltOn;
public RouterInfo() {
} // JAXB needs this
public RouterInfo(Router router) {
long ts = Router.getClusterTimeStamp();
this.id = ts;
this.state = router.getServiceState();
Configuration configuration = router.getConfig();
this.routerStateStoreName = configuration.get(
YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS,
YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
this.routerVersion = YarnVersionInfo.getVersion();
this.routerBuildVersion = YarnVersionInfo.getBuildVersion();
this.routerVersionBuiltOn = YarnVersionInfo.getDate();
this.hadoopVersion = VersionInfo.getVersion();
this.hadoopBuildVersion = VersionInfo.getBuildVersion();
this.hadoopVersionBuiltOn = VersionInfo.getDate();
this.startedOn = ts;
}
public String getState() {
return this.state.toString();
}
public String getRouterStateStore() {
return this.routerStateStoreName;
}
public String getRouterVersion() {
return this.routerVersion;
}
public String getRouterBuildVersion() {
return this.routerBuildVersion;
}
public String getRouterVersionBuiltOn() {
return this.routerVersionBuiltOn;
}
public String getHadoopVersion() {
return this.hadoopVersion;
}
public String getHadoopBuildVersion() {
return this.hadoopBuildVersion;
}
public String getHadoopVersionBuiltOn() {
return this.hadoopVersionBuiltOn;
}
public long getClusterId() {
return this.id;
}
public long getStartedOn() {
return this.startedOn;
}
}

View File

@ -0,0 +1,109 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.router.webapp.dao;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class RouterSchedulerMetrics {
// Metrics Log.
private static final Logger LOG = LoggerFactory.getLogger(RouterSchedulerMetrics.class);
// Scheduler Information.
private String subCluster = "N/A";
private String schedulerType = "N/A";
private String schedulingResourceType = "N/A";
private String minimumAllocation = "N/A";
private String maximumAllocation = "N/A";
private String applicationPriority = "N/A";
private String schedulerBusy = "N/A";
private String rmDispatcherEventQueueSize = "N/A";
private String schedulerDispatcherEventQueueSize = "N/A";
public RouterSchedulerMetrics() {
}
public RouterSchedulerMetrics(SubClusterInfo subClusterInfo, RouterClusterMetrics metrics,
SchedulerOverviewInfo overview) {
try {
// Parse Scheduler Information.
this.subCluster = subClusterInfo.getSubClusterId().getId();
this.schedulerType = overview.getSchedulerType();
this.schedulingResourceType = overview.getSchedulingResourceType();
this.minimumAllocation = overview.getMinimumAllocation().toString();
this.maximumAllocation = overview.getMaximumAllocation().toString();
this.applicationPriority = String.valueOf(overview.getApplicationPriority());
if (overview.getSchedulerBusy() != -1) {
this.schedulerBusy = String.valueOf(overview.getSchedulerBusy());
}
this.rmDispatcherEventQueueSize =
String.valueOf(overview.getRmDispatcherEventQueueSize());
this.schedulerDispatcherEventQueueSize =
String.valueOf(overview.getSchedulerDispatcherEventQueueSize());
} catch (Exception ex) {
LOG.error("RouterSchedulerMetrics Error.", ex);
}
}
public String getSubCluster() {
return subCluster;
}
public String getSchedulerType() {
return schedulerType;
}
public String getSchedulingResourceType() {
return schedulingResourceType;
}
public String getMinimumAllocation() {
return minimumAllocation;
}
public String getMaximumAllocation() {
return maximumAllocation;
}
public String getApplicationPriority() {
return applicationPriority;
}
public String getRmDispatcherEventQueueSize() {
return rmDispatcherEventQueueSize;
}
public String getSchedulerDispatcherEventQueueSize() {
return schedulerDispatcherEventQueueSize;
}
public String getSchedulerBusy() {
return schedulerBusy;
}
}

View File

@ -0,0 +1,20 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/** Router Web Dao package. **/
package org.apache.hadoop.yarn.server.router.webapp.dao;

View File

@ -26,7 +26,7 @@ import org.junit.Test;
import java.io.IOException; import java.io.IOException;
public class TestFederationWebApp { public class TestFederationWebApp extends TestRouterWebServicesREST {
@Test @Test
public void testFederationWebViewNotEnable() public void testFederationWebViewNotEnable()
@ -45,4 +45,22 @@ public class TestFederationWebApp {
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
WebAppTests.testPage(FederationPage.class, Router.class, new MockRouter(config)); WebAppTests.testPage(FederationPage.class, Router.class, new MockRouter(config));
} }
@Test
public void testFederationAboutViewEnable()
throws InterruptedException, YarnException, IOException {
// Test Federation Enabled
Configuration config = new YarnConfiguration();
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config));
}
@Test
public void testFederationAboutViewNotEnable()
throws InterruptedException, YarnException, IOException {
// Test Federation Not Enabled
Configuration config = new YarnConfiguration();
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config));
}
} }