YARN-11317. [Federation] Refactoring Yarn Router's About Web Page. (#4946)
This commit is contained in:
slfan1989
GitHub
parent
82a88a8ae6
commit
9e16f1f883
|
|
@ -97,4 +97,28 @@ public abstract class HtmlBlock extends TextView implements SubView {
|
|||
return callerUGI;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize User Help Information Div.
|
||||
* When the user does not configure the Yarn Federation function, prompt the user.
|
||||
*
|
||||
* @param html HTML page.
|
||||
* @param isEnabled If federation is enabled.
|
||||
*/
|
||||
protected void initUserHelpInformationDiv(Block html, boolean isEnabled) {
|
||||
if (!isEnabled) {
|
||||
html.style(".alert {padding: 15px; margin-bottom: 20px; " +
|
||||
" border: 1px solid transparent; border-radius: 4px;}");
|
||||
html.style(".alert-dismissable {padding-right: 35px;}");
|
||||
html.style(".alert-info {color: #856404;background-color: #fff3cd;border-color: #ffeeba;}");
|
||||
|
||||
Hamlet.DIV<Hamlet> div = html.div("#div_id").$class("alert alert-dismissable alert-info");
|
||||
div.p().$style("color:red").__("Federation is not Enabled.").__()
|
||||
.p().__()
|
||||
.p().__("We can refer to the following documents to configure Yarn Federation. ").__()
|
||||
.p().__()
|
||||
.a("https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/Federation.html",
|
||||
"Hadoop: YARN Federation").
|
||||
__();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,6 +57,9 @@ public final class RMWSConsts {
|
|||
/** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */
|
||||
public static final String SCHEDULER_LOGS = "/scheduler/logs";
|
||||
|
||||
/** Path for {@code RMWebServiceProtocol#getSchedulerOverview}. */
|
||||
public static final String SCHEDULER_OVERVIEW = "/scheduler-overview";
|
||||
|
||||
/**
|
||||
* Path for {@code RMWebServiceProtocol#validateAndGetSchedulerConfiguration}.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -202,6 +202,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesIn
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
|
||||
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
|
||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||
import org.apache.hadoop.yarn.server.webapp.WebServices;
|
||||
|
|
@ -2942,4 +2943,14 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
|
|||
}
|
||||
return Response.status(Status.OK).build();
|
||||
}
|
||||
|
||||
@GET
|
||||
@Path(RMWSConsts.SCHEDULER_OVERVIEW)
|
||||
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
|
||||
public SchedulerOverviewInfo getSchedulerOverview() {
|
||||
initForReadableEndpoints();
|
||||
ResourceScheduler rs = rm.getResourceScheduler();
|
||||
return new SchedulerOverviewInfo(rs);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,118 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
import java.util.List;
|
||||
|
||||
@XmlRootElement(name = "scheduler")
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class SchedulerOverviewInfo {
|
||||
|
||||
private String schedulerType;
|
||||
private String schedulingResourceType;
|
||||
private ResourceInfo minimumAllocation;
|
||||
private ResourceInfo maximumAllocation;
|
||||
private int applicationPriority;
|
||||
private int schedulerBusy;
|
||||
private int rmDispatcherEventQueueSize;
|
||||
private int schedulerDispatcherEventQueueSize;
|
||||
|
||||
// JAXB needs this
|
||||
public SchedulerOverviewInfo() {
|
||||
|
||||
}
|
||||
|
||||
public SchedulerOverviewInfo(ResourceScheduler rs) {
|
||||
// Parse the schedule type
|
||||
this.schedulerType = getSchedulerName(rs);
|
||||
|
||||
// Parse and allocate resource information
|
||||
this.minimumAllocation = new ResourceInfo(rs.getMinimumResourceCapability());
|
||||
this.maximumAllocation = new ResourceInfo(rs.getMaximumResourceCapability());
|
||||
|
||||
// Parse App Priority
|
||||
this.applicationPriority = rs.getMaxClusterLevelAppPriority().getPriority();
|
||||
|
||||
// Resolving resource types
|
||||
List<ResourceTypeInfo> resourceTypeInfos = ResourceUtils.getResourcesTypeInfo();
|
||||
resourceTypeInfos.sort((o1, o2) -> o1.getName().compareToIgnoreCase(o2.getName()));
|
||||
this.schedulingResourceType = StringUtils.join(resourceTypeInfos, ",");
|
||||
|
||||
// clusterMetrics
|
||||
ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(rs);
|
||||
this.schedulerBusy = clusterMetrics.getRmSchedulerBusyPercent();
|
||||
this.rmDispatcherEventQueueSize = clusterMetrics.getRmEventQueueSize();
|
||||
this.schedulerDispatcherEventQueueSize = clusterMetrics.getSchedulerEventQueueSize();
|
||||
}
|
||||
|
||||
private static String getSchedulerName(ResourceScheduler rs) {
|
||||
if (rs instanceof CapacityScheduler) {
|
||||
return "Capacity Scheduler";
|
||||
}
|
||||
if (rs instanceof FairScheduler) {
|
||||
return "Fair Scheduler";
|
||||
}
|
||||
if (rs instanceof FifoScheduler) {
|
||||
return "Fifo Scheduler";
|
||||
}
|
||||
return rs.getClass().getSimpleName();
|
||||
}
|
||||
|
||||
public String getSchedulerType() {
|
||||
return schedulerType;
|
||||
}
|
||||
|
||||
public String getSchedulingResourceType() {
|
||||
return schedulingResourceType;
|
||||
}
|
||||
|
||||
public ResourceInfo getMinimumAllocation() {
|
||||
return minimumAllocation;
|
||||
}
|
||||
|
||||
public ResourceInfo getMaximumAllocation() {
|
||||
return maximumAllocation;
|
||||
}
|
||||
|
||||
public int getApplicationPriority() {
|
||||
return applicationPriority;
|
||||
}
|
||||
|
||||
public int getSchedulerBusy() {
|
||||
return schedulerBusy;
|
||||
}
|
||||
|
||||
public int getRmDispatcherEventQueueSize() {
|
||||
return rmDispatcherEventQueueSize;
|
||||
}
|
||||
|
||||
public int getSchedulerDispatcherEventQueueSize() {
|
||||
return schedulerDispatcherEventQueueSize;
|
||||
}
|
||||
}
|
||||
|
|
@ -1099,4 +1099,61 @@ public class TestRMWebServices extends JerseyTestBase {
|
|||
return webService;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClusterSchedulerOverviewFifo() throws JSONException, Exception {
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||
.path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
|
||||
.get(ClientResponse.class);
|
||||
|
||||
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
JSONObject json = response.getEntity(JSONObject.class);
|
||||
verifyClusterSchedulerOverView(json, "Fifo Scheduler");
|
||||
}
|
||||
|
||||
public static void verifyClusterSchedulerOverView(
|
||||
JSONObject json, String expectedSchedulerType) throws Exception {
|
||||
|
||||
// why json contains 8 elements because we defined 8 fields
|
||||
assertEquals("incorrect number of elements in: " + json, 8, json.length());
|
||||
|
||||
// 1.Verify that the schedulerType is as expected
|
||||
String schedulerType = json.getString("schedulerType");
|
||||
assertEquals(expectedSchedulerType, schedulerType);
|
||||
|
||||
// 2.Verify that schedulingResourceType is as expected
|
||||
String schedulingResourceType = json.getString("schedulingResourceType");
|
||||
assertEquals("memory-mb (unit=Mi),vcores", schedulingResourceType);
|
||||
|
||||
// 3.Verify that minimumAllocation is as expected
|
||||
JSONObject minimumAllocation = json.getJSONObject("minimumAllocation");
|
||||
String minMemory = minimumAllocation.getString("memory");
|
||||
String minVCores = minimumAllocation.getString("vCores");
|
||||
assertEquals("1024", minMemory);
|
||||
assertEquals("1", minVCores);
|
||||
|
||||
// 4.Verify that maximumAllocation is as expected
|
||||
JSONObject maximumAllocation = json.getJSONObject("maximumAllocation");
|
||||
String maxMemory = maximumAllocation.getString("memory");
|
||||
String maxVCores = maximumAllocation.getString("vCores");
|
||||
assertEquals("8192", maxMemory);
|
||||
assertEquals("4", maxVCores);
|
||||
|
||||
// 5.Verify that schedulerBusy is as expected
|
||||
int schedulerBusy = json.getInt("schedulerBusy");
|
||||
assertEquals(-1, schedulerBusy);
|
||||
|
||||
// 6.Verify that rmDispatcherEventQueueSize is as expected
|
||||
int rmDispatcherEventQueueSize = json.getInt("rmDispatcherEventQueueSize");
|
||||
assertEquals(0, rmDispatcherEventQueueSize);
|
||||
|
||||
// 7.Verify that schedulerDispatcherEventQueueSize is as expected
|
||||
int schedulerDispatcherEventQueueSize = json.getInt("schedulerDispatcherEventQueueSize");
|
||||
assertEquals(0, schedulerDispatcherEventQueueSize);
|
||||
|
||||
// 8.Verify that applicationPriority is as expected
|
||||
int applicationPriority = json.getInt("applicationPriority");
|
||||
assertEquals(0, applicationPriority);
|
||||
}
|
||||
}
|
||||
|
|
@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||
import com.google.inject.Guice;
|
||||
import com.google.inject.servlet.ServletModule;
|
||||
import com.sun.jersey.api.client.ClientResponse;
|
||||
import com.sun.jersey.api.client.WebResource;
|
||||
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
|
||||
import com.sun.jersey.test.framework.WebAppDescriptor;
|
||||
|
||||
|
|
@ -413,4 +414,16 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
|
|||
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
|
||||
return new MockRM(conf);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClusterSchedulerOverviewCapacity() throws Exception {
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||
.path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
|
||||
.get(ClientResponse.class);
|
||||
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
JSONObject json = response.getEntity(JSONObject.class);
|
||||
TestRMWebServices.verifyClusterSchedulerOverView(json, "Capacity Scheduler");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager
|
|||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServices;
|
||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
|
||||
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
|
||||
|
|
@ -157,4 +158,15 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
|
|||
assertEquals("root", rootQueue.getString("queueName"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClusterSchedulerOverviewFair() throws Exception {
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||
.path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
|
||||
.get(ClientResponse.class);
|
||||
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
JSONObject json = response.getEntity(JSONObject.class);
|
||||
TestRMWebServices.verifyClusterSchedulerOverView(json, "Fair Scheduler");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ public class Router extends CompositeService {
|
|||
private WebApp webApp;
|
||||
@VisibleForTesting
|
||||
protected String webAppAddress;
|
||||
private static long clusterTimeStamp = System.currentTimeMillis();
|
||||
|
||||
/**
|
||||
* Priority of the Router shutdown hook.
|
||||
|
|
@ -237,4 +238,8 @@ public class Router extends CompositeService {
|
|||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
public static long getClusterTimeStamp() {
|
||||
return clusterTimeStamp;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,15 +18,11 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import com.sun.jersey.api.client.Client;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||
import org.apache.commons.lang3.time.DateFormatUtils;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
|
||||
import org.apache.hadoop.yarn.server.router.Router;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
import org.apache.hadoop.yarn.server.router.webapp.dao.RouterInfo;
|
||||
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
|
|
@ -34,62 +30,57 @@ import com.google.inject.Inject;
|
|||
/**
|
||||
* About block for the Router Web UI.
|
||||
*/
|
||||
public class AboutBlock extends HtmlBlock {
|
||||
|
||||
private static final long BYTES_IN_MB = 1024 * 1024;
|
||||
public class AboutBlock extends RouterBlock {
|
||||
|
||||
private final Router router;
|
||||
|
||||
@Inject
|
||||
AboutBlock(Router router, ViewContext ctx) {
|
||||
super(ctx);
|
||||
super(router, ctx);
|
||||
this.router = router;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void render(Block html) {
|
||||
Configuration conf = this.router.getConfig();
|
||||
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
|
||||
Client client = RouterWebServiceUtil.createJerseyClient(conf);
|
||||
|
||||
ClusterMetricsInfo metrics = RouterWebServiceUtil
|
||||
.genericForward(webAppAddress, null, ClusterMetricsInfo.class,
|
||||
HTTPMethods.GET,
|
||||
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
|
||||
conf, client);
|
||||
boolean isEnabled = conf.getBoolean(
|
||||
YarnConfiguration.FEDERATION_ENABLED,
|
||||
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
|
||||
info("Cluster Status").
|
||||
__("Federation Enabled", isEnabled).
|
||||
__("Applications Submitted", metrics.getAppsSubmitted()).
|
||||
__("Applications Pending", metrics.getAppsPending()).
|
||||
__("Applications Running", metrics.getAppsRunning()).
|
||||
__("Applications Failed", metrics.getAppsFailed()).
|
||||
__("Applications Killed", metrics.getAppsKilled()).
|
||||
__("Applications Completed", metrics.getAppsCompleted()).
|
||||
__("Containers Allocated", metrics.getContainersAllocated()).
|
||||
__("Containers Reserved", metrics.getReservedContainers()).
|
||||
__("Containers Pending", metrics.getPendingContainers()).
|
||||
__("Available Memory",
|
||||
StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB)).
|
||||
__("Allocated Memory",
|
||||
StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB)).
|
||||
__("Reserved Memory",
|
||||
StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB)).
|
||||
__("Total Memory",
|
||||
StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB)).
|
||||
__("Available VirtualCores", metrics.getAvailableVirtualCores()).
|
||||
__("Allocated VirtualCores", metrics.getAllocatedVirtualCores()).
|
||||
__("Reserved VirtualCores", metrics.getReservedVirtualCores()).
|
||||
__("Total VirtualCores", metrics.getTotalVirtualCores()).
|
||||
__("Active Nodes", metrics.getActiveNodes()).
|
||||
__("Lost Nodes", metrics.getLostNodes()).
|
||||
__("Available Nodes", metrics.getDecommissionedNodes()).
|
||||
__("Unhealthy Nodes", metrics.getUnhealthyNodes()).
|
||||
__("Rebooted Nodes", metrics.getRebootedNodes()).
|
||||
__("Total Nodes", metrics.getTotalNodes());
|
||||
boolean isEnabled = isYarnFederationEnabled();
|
||||
|
||||
// If Yarn Federation is not enabled, the user needs to be prompted.
|
||||
initUserHelpInformationDiv(html, isEnabled);
|
||||
|
||||
// Metrics Overview Table
|
||||
html.__(MetricsOverviewTable.class);
|
||||
|
||||
// Init Yarn Router Basic Information
|
||||
initYarnRouterBasicInformation(isEnabled);
|
||||
|
||||
// InfoBlock
|
||||
html.__(InfoBlock.class);
|
||||
}
|
||||
|
||||
/**
|
||||
* Init Yarn Router Basic Infomation.
|
||||
* @param isEnabled true, federation is enabled; false, federation is not enabled.
|
||||
*/
|
||||
private void initYarnRouterBasicInformation(boolean isEnabled) {
|
||||
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
|
||||
RouterInfo routerInfo = new RouterInfo(router);
|
||||
String lastStartTime =
|
||||
DateFormatUtils.format(routerInfo.getStartedOn(), DATE_PATTERN);
|
||||
try {
|
||||
info("Yarn Router Overview").
|
||||
__("Federation Enabled:", String.valueOf(isEnabled)).
|
||||
__("Router ID:", routerInfo.getClusterId()).
|
||||
__("Router state:", routerInfo.getState()).
|
||||
__("Router SubCluster Count:", facade.getSubClusters(true).size()).
|
||||
__("Router RMStateStore:", routerInfo.getRouterStateStore()).
|
||||
__("Router started on:", lastStartTime).
|
||||
__("Router version:", routerInfo.getRouterBuildVersion() +
|
||||
" on " + routerInfo.getRouterVersionBuiltOn()).
|
||||
__("Hadoop version:", routerInfo.getHadoopBuildVersion() +
|
||||
" on " + routerInfo.getHadoopVersionBuiltOn());
|
||||
} catch (YarnException e) {
|
||||
LOG.error("initYarnRouterBasicInformation error.", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -20,50 +20,41 @@ package org.apache.hadoop.yarn.server.router.webapp;
|
|||
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.HashMap;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.commons.lang3.time.DateFormatUtils;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
|
||||
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
|
||||
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||
import org.apache.hadoop.yarn.server.router.Router;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
import com.sun.jersey.api.json.JSONConfiguration;
|
||||
import com.sun.jersey.api.json.JSONJAXBContext;
|
||||
import com.sun.jersey.api.json.JSONUnmarshaller;
|
||||
|
||||
class FederationBlock extends HtmlBlock {
|
||||
class FederationBlock extends RouterBlock {
|
||||
|
||||
private final Router router;
|
||||
|
||||
@Inject
|
||||
FederationBlock(ViewContext ctx, Router router) {
|
||||
super(ctx);
|
||||
super(router, ctx);
|
||||
this.router = router;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void render(Block html) {
|
||||
|
||||
Configuration conf = this.router.getConfig();
|
||||
boolean isEnabled = conf.getBoolean(
|
||||
YarnConfiguration.FEDERATION_ENABLED,
|
||||
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
|
||||
boolean isEnabled = isYarnFederationEnabled();
|
||||
|
||||
// init Html Page Federation
|
||||
initHtmlPageFederation(html, isEnabled);
|
||||
|
|
@ -122,21 +113,7 @@ class FederationBlock extends HtmlBlock {
|
|||
List<Map<String, String>> lists = new ArrayList<>();
|
||||
|
||||
// If Yarn Federation is not enabled, the user needs to be prompted.
|
||||
if (!isEnabled) {
|
||||
html.style(".alert {padding: 15px; margin-bottom: 20px; " +
|
||||
" border: 1px solid transparent; border-radius: 4px;}");
|
||||
html.style(".alert-dismissable {padding-right: 35px;}");
|
||||
html.style(".alert-info {color: #856404;background-color: #fff3cd;border-color: #ffeeba;}");
|
||||
|
||||
Hamlet.DIV<Hamlet> div = html.div("#div_id").$class("alert alert-dismissable alert-info");
|
||||
div.p().$style("color:red").__("Federation is not Enabled.").__()
|
||||
.p().__()
|
||||
.p().__("We can refer to the following documents to configure Yarn Federation. ").__()
|
||||
.p().__()
|
||||
.a("https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/Federation.html",
|
||||
"Hadoop: YARN Federation").
|
||||
__();
|
||||
}
|
||||
initUserHelpInformationDiv(html, isEnabled);
|
||||
|
||||
// Table header
|
||||
TBODY<TABLE<Hamlet>> tbody =
|
||||
|
|
@ -150,16 +127,9 @@ class FederationBlock extends HtmlBlock {
|
|||
.__().__().tbody();
|
||||
|
||||
try {
|
||||
// Binding to the FederationStateStore
|
||||
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
|
||||
|
||||
Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true);
|
||||
|
||||
// Sort the SubClusters
|
||||
List<SubClusterInfo> subclusters = new ArrayList<>();
|
||||
subclusters.addAll(subClustersInfo.values());
|
||||
Comparator<? super SubClusterInfo> cmp = Comparator.comparing(o -> o.getSubClusterId());
|
||||
Collections.sort(subclusters, cmp);
|
||||
List<SubClusterInfo> subclusters = getSubClusterInfoList();
|
||||
|
||||
for (SubClusterInfo subcluster : subclusters) {
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,239 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
import com.sun.jersey.api.client.Client;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
|
||||
import org.apache.hadoop.yarn.server.router.Router;
|
||||
import org.apache.hadoop.yarn.server.router.webapp.dao.RouterClusterMetrics;
|
||||
import org.apache.hadoop.yarn.server.router.webapp.dao.RouterSchedulerMetrics;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
public class MetricsOverviewTable extends RouterBlock {
|
||||
|
||||
private final Router router;
|
||||
|
||||
@Inject
|
||||
MetricsOverviewTable(Router router, ViewContext ctx) {
|
||||
super(router, ctx);
|
||||
this.router = router;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void render(Block html) {
|
||||
// Initialize page styles
|
||||
html.style(".metrics {margin-bottom:5px}");
|
||||
|
||||
// get routerClusterMetrics Info
|
||||
ClusterMetricsInfo routerClusterMetricsInfo = getRouterClusterMetricsInfo();
|
||||
RouterClusterMetrics routerClusterMetrics = new RouterClusterMetrics(routerClusterMetricsInfo);
|
||||
|
||||
// metrics div
|
||||
Hamlet.DIV<Hamlet> div = html.div().$class("metrics");
|
||||
try {
|
||||
initFederationClusterAppsMetrics(div, routerClusterMetrics);
|
||||
initFederationClusterNodesMetrics(div, routerClusterMetrics);
|
||||
initFederationClusterSchedulersMetrics(div, routerClusterMetrics);
|
||||
} catch (Exception e) {
|
||||
LOG.error("MetricsOverviewTable init error.", e);
|
||||
}
|
||||
div.__();
|
||||
}
|
||||
|
||||
/**
|
||||
* Init Federation Cluster Apps Metrics.
|
||||
* Contains App information, resource usage information.
|
||||
*
|
||||
* @param div data display div.
|
||||
* @param metrics data metric information.
|
||||
*/
|
||||
private void initFederationClusterAppsMetrics(Hamlet.DIV<Hamlet> div,
|
||||
RouterClusterMetrics metrics) {
|
||||
div.h3("Federation Cluster Metrics").
|
||||
table("#metricsoverview").
|
||||
thead().$class("ui-widget-header").
|
||||
// Initialize table header information
|
||||
tr().
|
||||
th().$class("ui-state-default").__("Apps Submitted").__().
|
||||
th().$class("ui-state-default").__("Apps Pending").__().
|
||||
th().$class("ui-state-default").__("Apps Running").__().
|
||||
th().$class("ui-state-default").__("Apps Completed").__().
|
||||
th().$class("ui-state-default").__("Containers Running").__().
|
||||
th().$class("ui-state-default").__("Used Resources").__().
|
||||
th().$class("ui-state-default").__("Total Resources").__().
|
||||
th().$class("ui-state-default").__("Reserved Resources").__().
|
||||
th().$class("ui-state-default").__("Physical Mem Used %").__().
|
||||
th().$class("ui-state-default").__("Physical VCores Used %").__().
|
||||
__().
|
||||
__().
|
||||
// Initialize table data information
|
||||
tbody().$class("ui-widget-content").
|
||||
tr().
|
||||
td(metrics.getAppsSubmitted()).
|
||||
td(metrics.getAppsPending()).
|
||||
td(String.valueOf(metrics.getAppsRunning())).
|
||||
td(metrics.getAppsCompleted()).
|
||||
td(metrics.getAllocatedContainers()).
|
||||
td(metrics.getUsedResources()).
|
||||
td(metrics.getTotalResources()).
|
||||
td(metrics.getReservedResources()).
|
||||
td(metrics.getUtilizedMBPercent()).
|
||||
td(metrics.getUtilizedVirtualCoresPercent()).
|
||||
__().
|
||||
__().__();
|
||||
}
|
||||
|
||||
/**
|
||||
* Init Federation Cluster Nodes Metrics.
|
||||
*
|
||||
* @param div data display div.
|
||||
* @param metrics data metric information.
|
||||
*/
|
||||
private void initFederationClusterNodesMetrics(Hamlet.DIV<Hamlet> div,
|
||||
RouterClusterMetrics metrics) {
|
||||
div.h3("Federation Cluster Nodes Metrics").
|
||||
table("#nodemetricsoverview").
|
||||
thead().$class("ui-widget-header").
|
||||
// Initialize table header information
|
||||
tr().
|
||||
th().$class("ui-state-default").__("Active Nodes").__().
|
||||
th().$class("ui-state-default").__("Decommissioning Nodes").__().
|
||||
th().$class("ui-state-default").__("Decommissioned Nodes").__().
|
||||
th().$class("ui-state-default").__("Lost Nodes").__().
|
||||
th().$class("ui-state-default").__("Unhealthy Nodes").__().
|
||||
th().$class("ui-state-default").__("Rebooted Nodes").__().
|
||||
th().$class("ui-state-default").__("Shutdown Nodes").__().
|
||||
__().
|
||||
__().
|
||||
// Initialize table data information
|
||||
tbody().$class("ui-widget-content").
|
||||
tr().
|
||||
td(String.valueOf(metrics.getActiveNodes())).
|
||||
td(String.valueOf(metrics.getDecommissioningNodes())).
|
||||
td(String.valueOf(metrics.getDecommissionedNodes())).
|
||||
td(String.valueOf(metrics.getLostNodes())).
|
||||
td(String.valueOf(metrics.getUnhealthyNodes())).
|
||||
td(String.valueOf(metrics.getRebootedNodes())).
|
||||
td(String.valueOf(metrics.getShutdownNodes())).
|
||||
__().
|
||||
__().__();
|
||||
}
|
||||
|
||||
/**
|
||||
* Init Federation Cluster SchedulersMetrics.
|
||||
*
|
||||
* @param div data display div.
|
||||
* @param metrics data metric information.
|
||||
* @throws YarnException yarn error.
|
||||
* @throws IOException io error.
|
||||
* @throws InterruptedException interrupt error.
|
||||
*/
|
||||
private void initFederationClusterSchedulersMetrics(Hamlet.DIV<Hamlet> div,
|
||||
RouterClusterMetrics metrics) throws YarnException, IOException, InterruptedException {
|
||||
// Sort the SubClusters.
|
||||
List<SubClusterInfo> subclusters = getSubClusterInfoList();
|
||||
|
||||
Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr =
|
||||
div.h3("Federation Scheduler Metrics").
|
||||
table("#schedulermetricsoverview").
|
||||
thead().$class("ui-widget-header").
|
||||
tr().
|
||||
th().$class("ui-state-default").__("SubCluster").__().
|
||||
th().$class("ui-state-default").__("Scheduler Type").__().
|
||||
th().$class("ui-state-default").__("Scheduling Resource Type").__().
|
||||
th().$class("ui-state-default").__("Minimum Allocation").__().
|
||||
th().$class("ui-state-default").__("Maximum Allocation").__().
|
||||
th().$class("ui-state-default").__("Maximum Cluster Application Priority").__().
|
||||
th().$class("ui-state-default").__("Scheduler Busy %").__().
|
||||
th().$class("ui-state-default").__("RM Dispatcher EventQueue Size").__().
|
||||
th().$class("ui-state-default")
|
||||
.__("Scheduler Dispatcher EventQueue Size").__().
|
||||
__().
|
||||
__().
|
||||
tbody().$class("ui-widget-content");
|
||||
|
||||
boolean isEnabled = isYarnFederationEnabled();
|
||||
|
||||
// If Federation mode is not enabled or there is currently no SubCluster available,
|
||||
// each column in the list should be displayed as N/A
|
||||
if (!isEnabled || subclusters == null || subclusters.isEmpty()) {
|
||||
fsMetricsScheduleTr.tr().
|
||||
td(UNAVAILABLE).
|
||||
td(UNAVAILABLE).
|
||||
td(UNAVAILABLE).
|
||||
td(UNAVAILABLE).
|
||||
td(UNAVAILABLE).
|
||||
td(UNAVAILABLE).
|
||||
td(UNAVAILABLE).
|
||||
td(UNAVAILABLE).
|
||||
td(UNAVAILABLE)
|
||||
.__();
|
||||
} else {
|
||||
initSubClusterOverViewTable(metrics, fsMetricsScheduleTr, subclusters);
|
||||
}
|
||||
|
||||
fsMetricsScheduleTr.__().__();
|
||||
}
|
||||
|
||||
private void initSubClusterOverViewTable(RouterClusterMetrics metrics,
|
||||
Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr,
|
||||
List<SubClusterInfo> subclusters) {
|
||||
|
||||
// configuration
|
||||
Configuration config = this.router.getConfig();
|
||||
|
||||
Client client = RouterWebServiceUtil.createJerseyClient(config);
|
||||
|
||||
// Traverse all SubClusters to get cluster information.
|
||||
for (SubClusterInfo subcluster : subclusters) {
|
||||
|
||||
// Call the RM interface to obtain schedule information
|
||||
String webAppAddress = WebAppUtils.getHttpSchemePrefix(config) +
|
||||
subcluster.getRMWebServiceAddress();
|
||||
|
||||
SchedulerOverviewInfo typeInfo = RouterWebServiceUtil
|
||||
.genericForward(webAppAddress, null, SchedulerOverviewInfo.class, HTTPMethods.GET,
|
||||
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_OVERVIEW, null, null,
|
||||
config, client);
|
||||
RouterSchedulerMetrics rsMetrics = new RouterSchedulerMetrics(subcluster, metrics, typeInfo);
|
||||
|
||||
// Basic information
|
||||
fsMetricsScheduleTr.tr().
|
||||
td(rsMetrics.getSubCluster()).
|
||||
td(rsMetrics.getSchedulerType()).
|
||||
td(rsMetrics.getSchedulingResourceType()).
|
||||
td(rsMetrics.getMinimumAllocation()).
|
||||
td(rsMetrics.getMaximumAllocation()).
|
||||
td(rsMetrics.getApplicationPriority()).
|
||||
td(rsMetrics.getSchedulerBusy()).
|
||||
td(rsMetrics.getRmDispatcherEventQueueSize()).
|
||||
td(rsMetrics.getSchedulerDispatcherEventQueueSize()).
|
||||
__();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import com.sun.jersey.api.client.Client;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
|
||||
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
|
||||
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||
import org.apache.hadoop.yarn.server.router.Router;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Map;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
|
||||
public abstract class RouterBlock extends HtmlBlock {
|
||||
|
||||
private final Router router;
|
||||
|
||||
public RouterBlock(Router router, ViewContext ctx) {
|
||||
super(ctx);
|
||||
this.router = router;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get RouterClusterMetrics Info.
|
||||
*
|
||||
* @return Router ClusterMetricsInfo.
|
||||
*/
|
||||
protected ClusterMetricsInfo getRouterClusterMetricsInfo() {
|
||||
Configuration conf = this.router.getConfig();
|
||||
boolean isEnabled = isYarnFederationEnabled();
|
||||
if(isEnabled) {
|
||||
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
|
||||
Client client = RouterWebServiceUtil.createJerseyClient(conf);
|
||||
ClusterMetricsInfo metrics = RouterWebServiceUtil
|
||||
.genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
|
||||
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
|
||||
conf, client);
|
||||
return metrics;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a list of subclusters.
|
||||
*
|
||||
* @return subcluster List.
|
||||
* @throws YarnException if the call to the getSubClusters is unsuccessful.
|
||||
*/
|
||||
protected List<SubClusterInfo> getSubClusterInfoList() throws YarnException {
|
||||
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
|
||||
Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true);
|
||||
|
||||
// Sort the SubClusters.
|
||||
List<SubClusterInfo> subclusters = new ArrayList<>();
|
||||
subclusters.addAll(subClustersInfo.values());
|
||||
Comparator<? super SubClusterInfo> cmp = Comparator.comparing(o -> o.getSubClusterId());
|
||||
Collections.sort(subclusters, cmp);
|
||||
|
||||
return subclusters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether Yarn Federation is enabled.
|
||||
*
|
||||
* @return true, enable yarn federation; false, not enable yarn federation;
|
||||
*/
|
||||
protected boolean isYarnFederationEnabled() {
|
||||
Configuration conf = this.router.getConfig();
|
||||
boolean isEnabled = conf.getBoolean(
|
||||
YarnConfiguration.FEDERATION_ENABLED,
|
||||
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
|
||||
return isEnabled;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,310 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.router.webapp.dao;
|
||||
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
|
||||
@XmlRootElement
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class RouterClusterMetrics {
|
||||
|
||||
protected static final long BYTES_IN_MB = 1024 * 1024;
|
||||
private static final Logger LOG = LoggerFactory.getLogger(RouterClusterMetrics.class);
|
||||
|
||||
// Application Information.
|
||||
private String appsSubmitted = "N/A";
|
||||
private String appsCompleted = "N/A";
|
||||
private String appsPending = "N/A";
|
||||
private String appsRunning = "N/A";
|
||||
private String appsFailed = "N/A";
|
||||
private String appsKilled = "N/A";
|
||||
|
||||
// Memory Information.
|
||||
private String totalMemory = "N/A";
|
||||
private String reservedMemory = "N/A";
|
||||
private String availableMemory = "N/A";
|
||||
private String allocatedMemory = "N/A";
|
||||
private String pendingMemory = "N/A";
|
||||
|
||||
// VirtualCores Information.
|
||||
private String reservedVirtualCores = "N/A";
|
||||
private String availableVirtualCores = "N/A";
|
||||
private String allocatedVirtualCores = "N/A";
|
||||
private String pendingVirtualCores = "N/A";
|
||||
private String totalVirtualCores = "N/A";
|
||||
|
||||
// Resources Information.
|
||||
private String usedResources = "N/A";
|
||||
private String totalResources = "N/A";
|
||||
private String reservedResources = "N/A";
|
||||
private String allocatedContainers = "N/A";
|
||||
|
||||
// Resource Percent Information.
|
||||
private String utilizedMBPercent = "N/A";
|
||||
private String utilizedVirtualCoresPercent = "N/A";
|
||||
|
||||
// Node Information.
|
||||
private String activeNodes = "N/A";
|
||||
private String decommissioningNodes = "N/A";
|
||||
private String decommissionedNodes = "N/A";
|
||||
private String lostNodes = "N/A";
|
||||
private String unhealthyNodes = "N/A";
|
||||
private String rebootedNodes = "N/A";
|
||||
private String shutdownNodes = "N/A";
|
||||
|
||||
public RouterClusterMetrics() {
|
||||
|
||||
}
|
||||
|
||||
public RouterClusterMetrics(ClusterMetricsInfo metrics) {
|
||||
if (metrics != null) {
|
||||
// Application Information Conversion.
|
||||
conversionApplicationInformation(metrics);
|
||||
|
||||
// Memory Information Conversion.
|
||||
conversionMemoryInformation(metrics);
|
||||
|
||||
// Resources Information Conversion.
|
||||
conversionResourcesInformation(metrics);
|
||||
|
||||
// Percent Information Conversion.
|
||||
conversionResourcesPercent(metrics);
|
||||
|
||||
// Node Information Conversion.
|
||||
conversionNodeInformation(metrics);
|
||||
}
|
||||
}
|
||||
|
||||
// Get Key Metric Information
|
||||
public String getAppsSubmitted() {
|
||||
return appsSubmitted;
|
||||
}
|
||||
|
||||
public String getAppsCompleted() {
|
||||
return appsCompleted;
|
||||
}
|
||||
|
||||
public String getAppsPending() {
|
||||
return appsPending;
|
||||
}
|
||||
|
||||
public String getAppsRunning() {
|
||||
return appsRunning;
|
||||
}
|
||||
|
||||
public String getAppsFailed() {
|
||||
return appsFailed;
|
||||
}
|
||||
|
||||
public String getAppsKilled() {
|
||||
return appsKilled;
|
||||
}
|
||||
|
||||
public String getTotalMemory() {
|
||||
return totalMemory;
|
||||
}
|
||||
|
||||
public String getReservedMemory() {
|
||||
return reservedMemory;
|
||||
}
|
||||
|
||||
public String getAvailableMemory() {
|
||||
return availableMemory;
|
||||
}
|
||||
|
||||
public String getAllocatedMemory() {
|
||||
return allocatedMemory;
|
||||
}
|
||||
|
||||
public String getPendingMemory() {
|
||||
return pendingMemory;
|
||||
}
|
||||
|
||||
public String getReservedVirtualCores() {
|
||||
return reservedVirtualCores;
|
||||
}
|
||||
|
||||
public String getAvailableVirtualCores() {
|
||||
return availableVirtualCores;
|
||||
}
|
||||
|
||||
public String getAllocatedVirtualCores() {
|
||||
return allocatedVirtualCores;
|
||||
}
|
||||
|
||||
public String getPendingVirtualCores() {
|
||||
return pendingVirtualCores;
|
||||
}
|
||||
|
||||
public String getTotalVirtualCores() {
|
||||
return totalVirtualCores;
|
||||
}
|
||||
|
||||
public String getUsedResources() {
|
||||
return usedResources;
|
||||
}
|
||||
|
||||
public String getTotalResources() {
|
||||
return totalResources;
|
||||
}
|
||||
|
||||
public String getReservedResources() {
|
||||
return reservedResources;
|
||||
}
|
||||
|
||||
public String getAllocatedContainers() {
|
||||
return allocatedContainers;
|
||||
}
|
||||
|
||||
public String getUtilizedMBPercent() {
|
||||
return utilizedMBPercent;
|
||||
}
|
||||
|
||||
public String getUtilizedVirtualCoresPercent() {
|
||||
return utilizedVirtualCoresPercent;
|
||||
}
|
||||
|
||||
public String getActiveNodes() {
|
||||
return activeNodes;
|
||||
}
|
||||
|
||||
public String getDecommissioningNodes() {
|
||||
return decommissioningNodes;
|
||||
}
|
||||
|
||||
public String getDecommissionedNodes() {
|
||||
return decommissionedNodes;
|
||||
}
|
||||
|
||||
public String getLostNodes() {
|
||||
return lostNodes;
|
||||
}
|
||||
|
||||
public String getUnhealthyNodes() {
|
||||
return unhealthyNodes;
|
||||
}
|
||||
|
||||
public String getRebootedNodes() {
|
||||
return rebootedNodes;
|
||||
}
|
||||
|
||||
public String getShutdownNodes() {
|
||||
return shutdownNodes;
|
||||
}
|
||||
|
||||
// Metric Information Conversion
|
||||
public void conversionApplicationInformation(ClusterMetricsInfo metrics) {
|
||||
try {
|
||||
// Application Information.
|
||||
this.appsSubmitted = String.valueOf(metrics.getAppsSubmitted());
|
||||
this.appsCompleted = String.valueOf(metrics.getAppsCompleted() +
|
||||
metrics.getAppsFailed() + metrics.getAppsKilled());
|
||||
this.appsPending = String.valueOf(metrics.getAppsPending());
|
||||
this.appsRunning = String.valueOf(metrics.getAppsRunning());
|
||||
this.appsFailed = String.valueOf(metrics.getAppsFailed());
|
||||
this.appsKilled = String.valueOf(metrics.getAppsKilled());
|
||||
} catch (Exception e) {
|
||||
LOG.error("conversionApplicationInformation error.", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Metric Memory Information
|
||||
public void conversionMemoryInformation(ClusterMetricsInfo metrics) {
|
||||
try {
|
||||
// Memory Information.
|
||||
this.totalMemory = StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB);
|
||||
this.reservedMemory = StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB);
|
||||
this.availableMemory = StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB);
|
||||
this.allocatedMemory = StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB);
|
||||
this.pendingMemory = StringUtils.byteDesc(metrics.getPendingMB() * BYTES_IN_MB);
|
||||
} catch (Exception e) {
|
||||
LOG.error("conversionMemoryInformation error.", e);
|
||||
}
|
||||
}
|
||||
|
||||
// ResourcesInformation Conversion
|
||||
public void conversionResourcesInformation(ClusterMetricsInfo metrics) {
|
||||
try {
|
||||
// Parse resource information from metrics.
|
||||
Resource metricUsedResources;
|
||||
Resource metricTotalResources;
|
||||
Resource metricReservedResources;
|
||||
|
||||
int metricAllocatedContainers;
|
||||
if (metrics.getCrossPartitionMetricsAvailable()) {
|
||||
metricAllocatedContainers = metrics.getTotalAllocatedContainersAcrossPartition();
|
||||
metricUsedResources = metrics.getTotalUsedResourcesAcrossPartition().getResource();
|
||||
metricTotalResources = metrics.getTotalClusterResourcesAcrossPartition().getResource();
|
||||
metricReservedResources = metrics.getTotalReservedResourcesAcrossPartition().getResource();
|
||||
// getTotalUsedResourcesAcrossPartition includes reserved resources.
|
||||
Resources.subtractFrom(metricUsedResources, metricReservedResources);
|
||||
} else {
|
||||
metricAllocatedContainers = metrics.getContainersAllocated();
|
||||
metricUsedResources = Resource.newInstance(metrics.getAllocatedMB(),
|
||||
(int) metrics.getAllocatedVirtualCores());
|
||||
metricTotalResources = Resource.newInstance(metrics.getTotalMB(),
|
||||
(int) metrics.getTotalVirtualCores());
|
||||
metricReservedResources = Resource.newInstance(metrics.getReservedMB(),
|
||||
(int) metrics.getReservedVirtualCores());
|
||||
}
|
||||
|
||||
// Convert to standard format.
|
||||
usedResources = metricUsedResources.getFormattedString();
|
||||
totalResources = metricTotalResources.getFormattedString();
|
||||
reservedResources = metricReservedResources.getFormattedString();
|
||||
allocatedContainers = String.valueOf(metricAllocatedContainers);
|
||||
|
||||
} catch (Exception e) {
|
||||
LOG.error("conversionResourcesInformation error.", e);
|
||||
}
|
||||
}
|
||||
|
||||
// ResourcesPercent Conversion
|
||||
public void conversionResourcesPercent(ClusterMetricsInfo metrics) {
|
||||
try {
|
||||
this.utilizedMBPercent = String.valueOf(metrics.getUtilizedMBPercent());
|
||||
this.utilizedVirtualCoresPercent = String.valueOf(metrics.getUtilizedVirtualCoresPercent());
|
||||
} catch (Exception e) {
|
||||
LOG.error("conversionResourcesPercent error.", e);
|
||||
}
|
||||
}
|
||||
|
||||
// NodeInformation Conversion
|
||||
public void conversionNodeInformation(ClusterMetricsInfo metrics) {
|
||||
try {
|
||||
this.activeNodes = String.valueOf(metrics.getActiveNodes());
|
||||
this.decommissioningNodes = String.valueOf(metrics.getDecommissioningNodes());
|
||||
this.decommissionedNodes = String.valueOf(metrics.getDecommissionedNodes());
|
||||
this.lostNodes = String.valueOf(metrics.getLostNodes());
|
||||
this.unhealthyNodes = String.valueOf(metrics.getUnhealthyNodes());
|
||||
this.rebootedNodes = String.valueOf(metrics.getRebootedNodes());
|
||||
this.shutdownNodes = String.valueOf(metrics.getShutdownNodes());
|
||||
} catch (Exception e) {
|
||||
LOG.error("conversionNodeInformation error.", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.router.webapp.dao;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.service.Service;
|
||||
import org.apache.hadoop.util.VersionInfo;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.router.Router;
|
||||
import org.apache.hadoop.yarn.util.YarnVersionInfo;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
|
||||
@XmlRootElement
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class RouterInfo {
|
||||
private long id;
|
||||
private long startedOn;
|
||||
private Service.STATE state;
|
||||
private String routerStateStoreName;
|
||||
private String routerVersion;
|
||||
private String routerBuildVersion;
|
||||
private String routerVersionBuiltOn;
|
||||
private String hadoopVersion;
|
||||
private String hadoopBuildVersion;
|
||||
private String hadoopVersionBuiltOn;
|
||||
|
||||
public RouterInfo() {
|
||||
} // JAXB needs this
|
||||
|
||||
public RouterInfo(Router router) {
|
||||
long ts = Router.getClusterTimeStamp();
|
||||
this.id = ts;
|
||||
this.state = router.getServiceState();
|
||||
Configuration configuration = router.getConfig();
|
||||
this.routerStateStoreName = configuration.get(
|
||||
YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS,
|
||||
YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
|
||||
this.routerVersion = YarnVersionInfo.getVersion();
|
||||
this.routerBuildVersion = YarnVersionInfo.getBuildVersion();
|
||||
this.routerVersionBuiltOn = YarnVersionInfo.getDate();
|
||||
this.hadoopVersion = VersionInfo.getVersion();
|
||||
this.hadoopBuildVersion = VersionInfo.getBuildVersion();
|
||||
this.hadoopVersionBuiltOn = VersionInfo.getDate();
|
||||
this.startedOn = ts;
|
||||
}
|
||||
|
||||
public String getState() {
|
||||
return this.state.toString();
|
||||
}
|
||||
|
||||
public String getRouterStateStore() {
|
||||
return this.routerStateStoreName;
|
||||
}
|
||||
|
||||
public String getRouterVersion() {
|
||||
return this.routerVersion;
|
||||
}
|
||||
|
||||
public String getRouterBuildVersion() {
|
||||
return this.routerBuildVersion;
|
||||
}
|
||||
|
||||
public String getRouterVersionBuiltOn() {
|
||||
return this.routerVersionBuiltOn;
|
||||
}
|
||||
|
||||
public String getHadoopVersion() {
|
||||
return this.hadoopVersion;
|
||||
}
|
||||
|
||||
public String getHadoopBuildVersion() {
|
||||
return this.hadoopBuildVersion;
|
||||
}
|
||||
|
||||
public String getHadoopVersionBuiltOn() {
|
||||
return this.hadoopVersionBuiltOn;
|
||||
}
|
||||
|
||||
public long getClusterId() {
|
||||
return this.id;
|
||||
}
|
||||
|
||||
public long getStartedOn() {
|
||||
return this.startedOn;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.router.webapp.dao;
|
||||
|
||||
|
||||
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
|
||||
@XmlRootElement
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class RouterSchedulerMetrics {
|
||||
|
||||
// Metrics Log.
|
||||
private static final Logger LOG = LoggerFactory.getLogger(RouterSchedulerMetrics.class);
|
||||
|
||||
// Scheduler Information.
|
||||
private String subCluster = "N/A";
|
||||
private String schedulerType = "N/A";
|
||||
private String schedulingResourceType = "N/A";
|
||||
private String minimumAllocation = "N/A";
|
||||
private String maximumAllocation = "N/A";
|
||||
private String applicationPriority = "N/A";
|
||||
private String schedulerBusy = "N/A";
|
||||
private String rmDispatcherEventQueueSize = "N/A";
|
||||
private String schedulerDispatcherEventQueueSize = "N/A";
|
||||
|
||||
public RouterSchedulerMetrics() {
|
||||
|
||||
}
|
||||
|
||||
public RouterSchedulerMetrics(SubClusterInfo subClusterInfo, RouterClusterMetrics metrics,
|
||||
SchedulerOverviewInfo overview) {
|
||||
try {
|
||||
// Parse Scheduler Information.
|
||||
this.subCluster = subClusterInfo.getSubClusterId().getId();
|
||||
this.schedulerType = overview.getSchedulerType();
|
||||
this.schedulingResourceType = overview.getSchedulingResourceType();
|
||||
this.minimumAllocation = overview.getMinimumAllocation().toString();
|
||||
this.maximumAllocation = overview.getMaximumAllocation().toString();
|
||||
this.applicationPriority = String.valueOf(overview.getApplicationPriority());
|
||||
if (overview.getSchedulerBusy() != -1) {
|
||||
this.schedulerBusy = String.valueOf(overview.getSchedulerBusy());
|
||||
}
|
||||
this.rmDispatcherEventQueueSize =
|
||||
String.valueOf(overview.getRmDispatcherEventQueueSize());
|
||||
this.schedulerDispatcherEventQueueSize =
|
||||
String.valueOf(overview.getSchedulerDispatcherEventQueueSize());
|
||||
} catch (Exception ex) {
|
||||
LOG.error("RouterSchedulerMetrics Error.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
public String getSubCluster() {
|
||||
return subCluster;
|
||||
}
|
||||
|
||||
public String getSchedulerType() {
|
||||
return schedulerType;
|
||||
}
|
||||
|
||||
public String getSchedulingResourceType() {
|
||||
return schedulingResourceType;
|
||||
}
|
||||
|
||||
public String getMinimumAllocation() {
|
||||
return minimumAllocation;
|
||||
}
|
||||
|
||||
public String getMaximumAllocation() {
|
||||
return maximumAllocation;
|
||||
}
|
||||
|
||||
public String getApplicationPriority() {
|
||||
return applicationPriority;
|
||||
}
|
||||
|
||||
public String getRmDispatcherEventQueueSize() {
|
||||
return rmDispatcherEventQueueSize;
|
||||
}
|
||||
|
||||
public String getSchedulerDispatcherEventQueueSize() {
|
||||
return schedulerDispatcherEventQueueSize;
|
||||
}
|
||||
|
||||
public String getSchedulerBusy() {
|
||||
return schedulerBusy;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/** Router Web Dao package. **/
|
||||
package org.apache.hadoop.yarn.server.router.webapp.dao;
|
||||
|
|
@ -26,7 +26,7 @@ import org.junit.Test;
|
|||
|
||||
import java.io.IOException;
|
||||
|
||||
public class TestFederationWebApp {
|
||||
public class TestFederationWebApp extends TestRouterWebServicesREST {
|
||||
|
||||
@Test
|
||||
public void testFederationWebViewNotEnable()
|
||||
|
|
@ -45,4 +45,22 @@ public class TestFederationWebApp {
|
|||
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
|
||||
WebAppTests.testPage(FederationPage.class, Router.class, new MockRouter(config));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFederationAboutViewEnable()
|
||||
throws InterruptedException, YarnException, IOException {
|
||||
// Test Federation Enabled
|
||||
Configuration config = new YarnConfiguration();
|
||||
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
|
||||
WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFederationAboutViewNotEnable()
|
||||
throws InterruptedException, YarnException, IOException {
|
||||
// Test Federation Not Enabled
|
||||
Configuration config = new YarnConfiguration();
|
||||
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
|
||||
WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue