YARN-11317. [Federation] Refactoring Yarn Router's About Web Page. (#4946)
This commit is contained in:
parent
82a88a8ae6
commit
9e16f1f883
|
@ -97,4 +97,28 @@ public abstract class HtmlBlock extends TextView implements SubView {
|
||||||
return callerUGI;
|
return callerUGI;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize User Help Information Div.
|
||||||
|
* When the user does not configure the Yarn Federation function, prompt the user.
|
||||||
|
*
|
||||||
|
* @param html HTML page.
|
||||||
|
* @param isEnabled If federation is enabled.
|
||||||
|
*/
|
||||||
|
protected void initUserHelpInformationDiv(Block html, boolean isEnabled) {
|
||||||
|
if (!isEnabled) {
|
||||||
|
html.style(".alert {padding: 15px; margin-bottom: 20px; " +
|
||||||
|
" border: 1px solid transparent; border-radius: 4px;}");
|
||||||
|
html.style(".alert-dismissable {padding-right: 35px;}");
|
||||||
|
html.style(".alert-info {color: #856404;background-color: #fff3cd;border-color: #ffeeba;}");
|
||||||
|
|
||||||
|
Hamlet.DIV<Hamlet> div = html.div("#div_id").$class("alert alert-dismissable alert-info");
|
||||||
|
div.p().$style("color:red").__("Federation is not Enabled.").__()
|
||||||
|
.p().__()
|
||||||
|
.p().__("We can refer to the following documents to configure Yarn Federation. ").__()
|
||||||
|
.p().__()
|
||||||
|
.a("https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/Federation.html",
|
||||||
|
"Hadoop: YARN Federation").
|
||||||
|
__();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -57,6 +57,9 @@ public final class RMWSConsts {
|
||||||
/** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */
|
/** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */
|
||||||
public static final String SCHEDULER_LOGS = "/scheduler/logs";
|
public static final String SCHEDULER_LOGS = "/scheduler/logs";
|
||||||
|
|
||||||
|
/** Path for {@code RMWebServiceProtocol#getSchedulerOverview}. */
|
||||||
|
public static final String SCHEDULER_OVERVIEW = "/scheduler-overview";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Path for {@code RMWebServiceProtocol#validateAndGetSchedulerConfiguration}.
|
* Path for {@code RMWebServiceProtocol#validateAndGetSchedulerConfiguration}.
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -202,6 +202,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesIn
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
|
||||||
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
|
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
|
||||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||||
import org.apache.hadoop.yarn.server.webapp.WebServices;
|
import org.apache.hadoop.yarn.server.webapp.WebServices;
|
||||||
|
@ -2942,4 +2943,14 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
|
||||||
}
|
}
|
||||||
return Response.status(Status.OK).build();
|
return Response.status(Status.OK).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@GET
|
||||||
|
@Path(RMWSConsts.SCHEDULER_OVERVIEW)
|
||||||
|
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||||
|
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
|
||||||
|
public SchedulerOverviewInfo getSchedulerOverview() {
|
||||||
|
initForReadableEndpoints();
|
||||||
|
ResourceScheduler rs = rm.getResourceScheduler();
|
||||||
|
return new SchedulerOverviewInfo(rs);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,118 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
|
||||||
|
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||||
|
|
||||||
|
import javax.xml.bind.annotation.XmlAccessType;
|
||||||
|
import javax.xml.bind.annotation.XmlAccessorType;
|
||||||
|
import javax.xml.bind.annotation.XmlRootElement;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@XmlRootElement(name = "scheduler")
|
||||||
|
@XmlAccessorType(XmlAccessType.FIELD)
|
||||||
|
public class SchedulerOverviewInfo {
|
||||||
|
|
||||||
|
private String schedulerType;
|
||||||
|
private String schedulingResourceType;
|
||||||
|
private ResourceInfo minimumAllocation;
|
||||||
|
private ResourceInfo maximumAllocation;
|
||||||
|
private int applicationPriority;
|
||||||
|
private int schedulerBusy;
|
||||||
|
private int rmDispatcherEventQueueSize;
|
||||||
|
private int schedulerDispatcherEventQueueSize;
|
||||||
|
|
||||||
|
// JAXB needs this
|
||||||
|
public SchedulerOverviewInfo() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public SchedulerOverviewInfo(ResourceScheduler rs) {
|
||||||
|
// Parse the schedule type
|
||||||
|
this.schedulerType = getSchedulerName(rs);
|
||||||
|
|
||||||
|
// Parse and allocate resource information
|
||||||
|
this.minimumAllocation = new ResourceInfo(rs.getMinimumResourceCapability());
|
||||||
|
this.maximumAllocation = new ResourceInfo(rs.getMaximumResourceCapability());
|
||||||
|
|
||||||
|
// Parse App Priority
|
||||||
|
this.applicationPriority = rs.getMaxClusterLevelAppPriority().getPriority();
|
||||||
|
|
||||||
|
// Resolving resource types
|
||||||
|
List<ResourceTypeInfo> resourceTypeInfos = ResourceUtils.getResourcesTypeInfo();
|
||||||
|
resourceTypeInfos.sort((o1, o2) -> o1.getName().compareToIgnoreCase(o2.getName()));
|
||||||
|
this.schedulingResourceType = StringUtils.join(resourceTypeInfos, ",");
|
||||||
|
|
||||||
|
// clusterMetrics
|
||||||
|
ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(rs);
|
||||||
|
this.schedulerBusy = clusterMetrics.getRmSchedulerBusyPercent();
|
||||||
|
this.rmDispatcherEventQueueSize = clusterMetrics.getRmEventQueueSize();
|
||||||
|
this.schedulerDispatcherEventQueueSize = clusterMetrics.getSchedulerEventQueueSize();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String getSchedulerName(ResourceScheduler rs) {
|
||||||
|
if (rs instanceof CapacityScheduler) {
|
||||||
|
return "Capacity Scheduler";
|
||||||
|
}
|
||||||
|
if (rs instanceof FairScheduler) {
|
||||||
|
return "Fair Scheduler";
|
||||||
|
}
|
||||||
|
if (rs instanceof FifoScheduler) {
|
||||||
|
return "Fifo Scheduler";
|
||||||
|
}
|
||||||
|
return rs.getClass().getSimpleName();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSchedulerType() {
|
||||||
|
return schedulerType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSchedulingResourceType() {
|
||||||
|
return schedulingResourceType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ResourceInfo getMinimumAllocation() {
|
||||||
|
return minimumAllocation;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ResourceInfo getMaximumAllocation() {
|
||||||
|
return maximumAllocation;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getApplicationPriority() {
|
||||||
|
return applicationPriority;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getSchedulerBusy() {
|
||||||
|
return schedulerBusy;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getRmDispatcherEventQueueSize() {
|
||||||
|
return rmDispatcherEventQueueSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getSchedulerDispatcherEventQueueSize() {
|
||||||
|
return schedulerDispatcherEventQueueSize;
|
||||||
|
}
|
||||||
|
}
|
|
@ -1099,4 +1099,61 @@ public class TestRMWebServices extends JerseyTestBase {
|
||||||
return webService;
|
return webService;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testClusterSchedulerOverviewFifo() throws JSONException, Exception {
|
||||||
|
WebResource r = resource();
|
||||||
|
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||||
|
.path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
|
||||||
|
.get(ClientResponse.class);
|
||||||
|
|
||||||
|
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||||
|
response.getType().toString());
|
||||||
|
JSONObject json = response.getEntity(JSONObject.class);
|
||||||
|
verifyClusterSchedulerOverView(json, "Fifo Scheduler");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void verifyClusterSchedulerOverView(
|
||||||
|
JSONObject json, String expectedSchedulerType) throws Exception {
|
||||||
|
|
||||||
|
// why json contains 8 elements because we defined 8 fields
|
||||||
|
assertEquals("incorrect number of elements in: " + json, 8, json.length());
|
||||||
|
|
||||||
|
// 1.Verify that the schedulerType is as expected
|
||||||
|
String schedulerType = json.getString("schedulerType");
|
||||||
|
assertEquals(expectedSchedulerType, schedulerType);
|
||||||
|
|
||||||
|
// 2.Verify that schedulingResourceType is as expected
|
||||||
|
String schedulingResourceType = json.getString("schedulingResourceType");
|
||||||
|
assertEquals("memory-mb (unit=Mi),vcores", schedulingResourceType);
|
||||||
|
|
||||||
|
// 3.Verify that minimumAllocation is as expected
|
||||||
|
JSONObject minimumAllocation = json.getJSONObject("minimumAllocation");
|
||||||
|
String minMemory = minimumAllocation.getString("memory");
|
||||||
|
String minVCores = minimumAllocation.getString("vCores");
|
||||||
|
assertEquals("1024", minMemory);
|
||||||
|
assertEquals("1", minVCores);
|
||||||
|
|
||||||
|
// 4.Verify that maximumAllocation is as expected
|
||||||
|
JSONObject maximumAllocation = json.getJSONObject("maximumAllocation");
|
||||||
|
String maxMemory = maximumAllocation.getString("memory");
|
||||||
|
String maxVCores = maximumAllocation.getString("vCores");
|
||||||
|
assertEquals("8192", maxMemory);
|
||||||
|
assertEquals("4", maxVCores);
|
||||||
|
|
||||||
|
// 5.Verify that schedulerBusy is as expected
|
||||||
|
int schedulerBusy = json.getInt("schedulerBusy");
|
||||||
|
assertEquals(-1, schedulerBusy);
|
||||||
|
|
||||||
|
// 6.Verify that rmDispatcherEventQueueSize is as expected
|
||||||
|
int rmDispatcherEventQueueSize = json.getInt("rmDispatcherEventQueueSize");
|
||||||
|
assertEquals(0, rmDispatcherEventQueueSize);
|
||||||
|
|
||||||
|
// 7.Verify that schedulerDispatcherEventQueueSize is as expected
|
||||||
|
int schedulerDispatcherEventQueueSize = json.getInt("schedulerDispatcherEventQueueSize");
|
||||||
|
assertEquals(0, schedulerDispatcherEventQueueSize);
|
||||||
|
|
||||||
|
// 8.Verify that applicationPriority is as expected
|
||||||
|
int applicationPriority = json.getInt("applicationPriority");
|
||||||
|
assertEquals(0, applicationPriority);
|
||||||
|
}
|
||||||
}
|
}
|
|
@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.google.inject.Guice;
|
import com.google.inject.Guice;
|
||||||
import com.google.inject.servlet.ServletModule;
|
import com.google.inject.servlet.ServletModule;
|
||||||
import com.sun.jersey.api.client.ClientResponse;
|
import com.sun.jersey.api.client.ClientResponse;
|
||||||
|
import com.sun.jersey.api.client.WebResource;
|
||||||
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
|
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
|
||||||
import com.sun.jersey.test.framework.WebAppDescriptor;
|
import com.sun.jersey.test.framework.WebAppDescriptor;
|
||||||
|
|
||||||
|
@ -413,4 +414,16 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
|
||||||
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
|
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
|
||||||
return new MockRM(conf);
|
return new MockRM(conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testClusterSchedulerOverviewCapacity() throws Exception {
|
||||||
|
WebResource r = resource();
|
||||||
|
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||||
|
.path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
|
||||||
|
.get(ClientResponse.class);
|
||||||
|
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||||
|
response.getType().toString());
|
||||||
|
JSONObject json = response.getEntity(JSONObject.class);
|
||||||
|
TestRMWebServices.verifyClusterSchedulerOverView(json, "Capacity Scheduler");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager
|
||||||
|
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServices;
|
||||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||||
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
|
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
|
||||||
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
|
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
|
||||||
|
@ -157,4 +158,15 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
|
||||||
assertEquals("root", rootQueue.getString("queueName"));
|
assertEquals("root", rootQueue.getString("queueName"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testClusterSchedulerOverviewFair() throws Exception {
|
||||||
|
WebResource r = resource();
|
||||||
|
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||||
|
.path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
|
||||||
|
.get(ClientResponse.class);
|
||||||
|
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||||
|
response.getType().toString());
|
||||||
|
JSONObject json = response.getEntity(JSONObject.class);
|
||||||
|
TestRMWebServices.verifyClusterSchedulerOverView(json, "Fair Scheduler");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -79,6 +79,7 @@ public class Router extends CompositeService {
|
||||||
private WebApp webApp;
|
private WebApp webApp;
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
protected String webAppAddress;
|
protected String webAppAddress;
|
||||||
|
private static long clusterTimeStamp = System.currentTimeMillis();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Priority of the Router shutdown hook.
|
* Priority of the Router shutdown hook.
|
||||||
|
@ -237,4 +238,8 @@ public class Router extends CompositeService {
|
||||||
}
|
}
|
||||||
return name;
|
return name;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static long getClusterTimeStamp() {
|
||||||
|
return clusterTimeStamp;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,15 +18,11 @@
|
||||||
|
|
||||||
package org.apache.hadoop.yarn.server.router.webapp;
|
package org.apache.hadoop.yarn.server.router.webapp;
|
||||||
|
|
||||||
import com.sun.jersey.api.client.Client;
|
import org.apache.commons.lang3.time.DateFormatUtils;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
|
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
|
||||||
import org.apache.hadoop.yarn.server.router.Router;
|
import org.apache.hadoop.yarn.server.router.Router;
|
||||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
import org.apache.hadoop.yarn.server.router.webapp.dao.RouterInfo;
|
||||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
|
||||||
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
|
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
|
||||||
|
|
||||||
import com.google.inject.Inject;
|
import com.google.inject.Inject;
|
||||||
|
@ -34,62 +30,57 @@ import com.google.inject.Inject;
|
||||||
/**
|
/**
|
||||||
* About block for the Router Web UI.
|
* About block for the Router Web UI.
|
||||||
*/
|
*/
|
||||||
public class AboutBlock extends HtmlBlock {
|
public class AboutBlock extends RouterBlock {
|
||||||
|
|
||||||
private static final long BYTES_IN_MB = 1024 * 1024;
|
|
||||||
|
|
||||||
private final Router router;
|
private final Router router;
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
AboutBlock(Router router, ViewContext ctx) {
|
AboutBlock(Router router, ViewContext ctx) {
|
||||||
super(ctx);
|
super(router, ctx);
|
||||||
this.router = router;
|
this.router = router;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void render(Block html) {
|
protected void render(Block html) {
|
||||||
Configuration conf = this.router.getConfig();
|
|
||||||
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
|
|
||||||
Client client = RouterWebServiceUtil.createJerseyClient(conf);
|
|
||||||
|
|
||||||
ClusterMetricsInfo metrics = RouterWebServiceUtil
|
boolean isEnabled = isYarnFederationEnabled();
|
||||||
.genericForward(webAppAddress, null, ClusterMetricsInfo.class,
|
|
||||||
HTTPMethods.GET,
|
|
||||||
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
|
|
||||||
conf, client);
|
|
||||||
boolean isEnabled = conf.getBoolean(
|
|
||||||
YarnConfiguration.FEDERATION_ENABLED,
|
|
||||||
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
|
|
||||||
info("Cluster Status").
|
|
||||||
__("Federation Enabled", isEnabled).
|
|
||||||
__("Applications Submitted", metrics.getAppsSubmitted()).
|
|
||||||
__("Applications Pending", metrics.getAppsPending()).
|
|
||||||
__("Applications Running", metrics.getAppsRunning()).
|
|
||||||
__("Applications Failed", metrics.getAppsFailed()).
|
|
||||||
__("Applications Killed", metrics.getAppsKilled()).
|
|
||||||
__("Applications Completed", metrics.getAppsCompleted()).
|
|
||||||
__("Containers Allocated", metrics.getContainersAllocated()).
|
|
||||||
__("Containers Reserved", metrics.getReservedContainers()).
|
|
||||||
__("Containers Pending", metrics.getPendingContainers()).
|
|
||||||
__("Available Memory",
|
|
||||||
StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB)).
|
|
||||||
__("Allocated Memory",
|
|
||||||
StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB)).
|
|
||||||
__("Reserved Memory",
|
|
||||||
StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB)).
|
|
||||||
__("Total Memory",
|
|
||||||
StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB)).
|
|
||||||
__("Available VirtualCores", metrics.getAvailableVirtualCores()).
|
|
||||||
__("Allocated VirtualCores", metrics.getAllocatedVirtualCores()).
|
|
||||||
__("Reserved VirtualCores", metrics.getReservedVirtualCores()).
|
|
||||||
__("Total VirtualCores", metrics.getTotalVirtualCores()).
|
|
||||||
__("Active Nodes", metrics.getActiveNodes()).
|
|
||||||
__("Lost Nodes", metrics.getLostNodes()).
|
|
||||||
__("Available Nodes", metrics.getDecommissionedNodes()).
|
|
||||||
__("Unhealthy Nodes", metrics.getUnhealthyNodes()).
|
|
||||||
__("Rebooted Nodes", metrics.getRebootedNodes()).
|
|
||||||
__("Total Nodes", metrics.getTotalNodes());
|
|
||||||
|
|
||||||
|
// If Yarn Federation is not enabled, the user needs to be prompted.
|
||||||
|
initUserHelpInformationDiv(html, isEnabled);
|
||||||
|
|
||||||
|
// Metrics Overview Table
|
||||||
|
html.__(MetricsOverviewTable.class);
|
||||||
|
|
||||||
|
// Init Yarn Router Basic Information
|
||||||
|
initYarnRouterBasicInformation(isEnabled);
|
||||||
|
|
||||||
|
// InfoBlock
|
||||||
html.__(InfoBlock.class);
|
html.__(InfoBlock.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Init Yarn Router Basic Infomation.
|
||||||
|
* @param isEnabled true, federation is enabled; false, federation is not enabled.
|
||||||
|
*/
|
||||||
|
private void initYarnRouterBasicInformation(boolean isEnabled) {
|
||||||
|
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
|
||||||
|
RouterInfo routerInfo = new RouterInfo(router);
|
||||||
|
String lastStartTime =
|
||||||
|
DateFormatUtils.format(routerInfo.getStartedOn(), DATE_PATTERN);
|
||||||
|
try {
|
||||||
|
info("Yarn Router Overview").
|
||||||
|
__("Federation Enabled:", String.valueOf(isEnabled)).
|
||||||
|
__("Router ID:", routerInfo.getClusterId()).
|
||||||
|
__("Router state:", routerInfo.getState()).
|
||||||
|
__("Router SubCluster Count:", facade.getSubClusters(true).size()).
|
||||||
|
__("Router RMStateStore:", routerInfo.getRouterStateStore()).
|
||||||
|
__("Router started on:", lastStartTime).
|
||||||
|
__("Router version:", routerInfo.getRouterBuildVersion() +
|
||||||
|
" on " + routerInfo.getRouterVersionBuiltOn()).
|
||||||
|
__("Hadoop version:", routerInfo.getHadoopBuildVersion() +
|
||||||
|
" on " + routerInfo.getHadoopVersionBuiltOn());
|
||||||
|
} catch (YarnException e) {
|
||||||
|
LOG.error("initYarnRouterBasicInformation error.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
|
@ -20,50 +20,41 @@ package org.apache.hadoop.yarn.server.router.webapp;
|
||||||
|
|
||||||
import java.io.StringReader;
|
import java.io.StringReader;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.Comparator;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
|
||||||
import com.google.gson.Gson;
|
import com.google.gson.Gson;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
|
||||||
import org.apache.hadoop.util.StringUtils;
|
import org.apache.hadoop.util.StringUtils;
|
||||||
import org.apache.commons.lang3.time.DateFormatUtils;
|
import org.apache.commons.lang3.time.DateFormatUtils;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
|
||||||
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
|
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
|
||||||
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
|
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
|
||||||
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
|
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||||
import org.apache.hadoop.yarn.server.router.Router;
|
import org.apache.hadoop.yarn.server.router.Router;
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
|
||||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
|
||||||
|
|
||||||
import com.google.inject.Inject;
|
import com.google.inject.Inject;
|
||||||
import com.sun.jersey.api.json.JSONConfiguration;
|
import com.sun.jersey.api.json.JSONConfiguration;
|
||||||
import com.sun.jersey.api.json.JSONJAXBContext;
|
import com.sun.jersey.api.json.JSONJAXBContext;
|
||||||
import com.sun.jersey.api.json.JSONUnmarshaller;
|
import com.sun.jersey.api.json.JSONUnmarshaller;
|
||||||
|
|
||||||
class FederationBlock extends HtmlBlock {
|
class FederationBlock extends RouterBlock {
|
||||||
|
|
||||||
private final Router router;
|
private final Router router;
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
FederationBlock(ViewContext ctx, Router router) {
|
FederationBlock(ViewContext ctx, Router router) {
|
||||||
super(ctx);
|
super(router, ctx);
|
||||||
this.router = router;
|
this.router = router;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void render(Block html) {
|
public void render(Block html) {
|
||||||
|
|
||||||
Configuration conf = this.router.getConfig();
|
boolean isEnabled = isYarnFederationEnabled();
|
||||||
boolean isEnabled = conf.getBoolean(
|
|
||||||
YarnConfiguration.FEDERATION_ENABLED,
|
|
||||||
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
|
|
||||||
|
|
||||||
// init Html Page Federation
|
// init Html Page Federation
|
||||||
initHtmlPageFederation(html, isEnabled);
|
initHtmlPageFederation(html, isEnabled);
|
||||||
|
@ -122,21 +113,7 @@ class FederationBlock extends HtmlBlock {
|
||||||
List<Map<String, String>> lists = new ArrayList<>();
|
List<Map<String, String>> lists = new ArrayList<>();
|
||||||
|
|
||||||
// If Yarn Federation is not enabled, the user needs to be prompted.
|
// If Yarn Federation is not enabled, the user needs to be prompted.
|
||||||
if (!isEnabled) {
|
initUserHelpInformationDiv(html, isEnabled);
|
||||||
html.style(".alert {padding: 15px; margin-bottom: 20px; " +
|
|
||||||
" border: 1px solid transparent; border-radius: 4px;}");
|
|
||||||
html.style(".alert-dismissable {padding-right: 35px;}");
|
|
||||||
html.style(".alert-info {color: #856404;background-color: #fff3cd;border-color: #ffeeba;}");
|
|
||||||
|
|
||||||
Hamlet.DIV<Hamlet> div = html.div("#div_id").$class("alert alert-dismissable alert-info");
|
|
||||||
div.p().$style("color:red").__("Federation is not Enabled.").__()
|
|
||||||
.p().__()
|
|
||||||
.p().__("We can refer to the following documents to configure Yarn Federation. ").__()
|
|
||||||
.p().__()
|
|
||||||
.a("https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/Federation.html",
|
|
||||||
"Hadoop: YARN Federation").
|
|
||||||
__();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Table header
|
// Table header
|
||||||
TBODY<TABLE<Hamlet>> tbody =
|
TBODY<TABLE<Hamlet>> tbody =
|
||||||
|
@ -150,16 +127,9 @@ class FederationBlock extends HtmlBlock {
|
||||||
.__().__().tbody();
|
.__().__().tbody();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Binding to the FederationStateStore
|
|
||||||
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
|
|
||||||
|
|
||||||
Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true);
|
|
||||||
|
|
||||||
// Sort the SubClusters
|
// Sort the SubClusters
|
||||||
List<SubClusterInfo> subclusters = new ArrayList<>();
|
List<SubClusterInfo> subclusters = getSubClusterInfoList();
|
||||||
subclusters.addAll(subClustersInfo.values());
|
|
||||||
Comparator<? super SubClusterInfo> cmp = Comparator.comparing(o -> o.getSubClusterId());
|
|
||||||
Collections.sort(subclusters, cmp);
|
|
||||||
|
|
||||||
for (SubClusterInfo subcluster : subclusters) {
|
for (SubClusterInfo subcluster : subclusters) {
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,239 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* <p>
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* <p>
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.server.router.webapp;
|
||||||
|
|
||||||
|
import com.google.inject.Inject;
|
||||||
|
import com.sun.jersey.api.client.Client;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
|
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
|
||||||
|
import org.apache.hadoop.yarn.server.router.Router;
|
||||||
|
import org.apache.hadoop.yarn.server.router.webapp.dao.RouterClusterMetrics;
|
||||||
|
import org.apache.hadoop.yarn.server.router.webapp.dao.RouterSchedulerMetrics;
|
||||||
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
||||||
|
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class MetricsOverviewTable extends RouterBlock {
|
||||||
|
|
||||||
|
private final Router router;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
MetricsOverviewTable(Router router, ViewContext ctx) {
|
||||||
|
super(router, ctx);
|
||||||
|
this.router = router;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void render(Block html) {
|
||||||
|
// Initialize page styles
|
||||||
|
html.style(".metrics {margin-bottom:5px}");
|
||||||
|
|
||||||
|
// get routerClusterMetrics Info
|
||||||
|
ClusterMetricsInfo routerClusterMetricsInfo = getRouterClusterMetricsInfo();
|
||||||
|
RouterClusterMetrics routerClusterMetrics = new RouterClusterMetrics(routerClusterMetricsInfo);
|
||||||
|
|
||||||
|
// metrics div
|
||||||
|
Hamlet.DIV<Hamlet> div = html.div().$class("metrics");
|
||||||
|
try {
|
||||||
|
initFederationClusterAppsMetrics(div, routerClusterMetrics);
|
||||||
|
initFederationClusterNodesMetrics(div, routerClusterMetrics);
|
||||||
|
initFederationClusterSchedulersMetrics(div, routerClusterMetrics);
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("MetricsOverviewTable init error.", e);
|
||||||
|
}
|
||||||
|
div.__();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Init Federation Cluster Apps Metrics.
|
||||||
|
* Contains App information, resource usage information.
|
||||||
|
*
|
||||||
|
* @param div data display div.
|
||||||
|
* @param metrics data metric information.
|
||||||
|
*/
|
||||||
|
private void initFederationClusterAppsMetrics(Hamlet.DIV<Hamlet> div,
|
||||||
|
RouterClusterMetrics metrics) {
|
||||||
|
div.h3("Federation Cluster Metrics").
|
||||||
|
table("#metricsoverview").
|
||||||
|
thead().$class("ui-widget-header").
|
||||||
|
// Initialize table header information
|
||||||
|
tr().
|
||||||
|
th().$class("ui-state-default").__("Apps Submitted").__().
|
||||||
|
th().$class("ui-state-default").__("Apps Pending").__().
|
||||||
|
th().$class("ui-state-default").__("Apps Running").__().
|
||||||
|
th().$class("ui-state-default").__("Apps Completed").__().
|
||||||
|
th().$class("ui-state-default").__("Containers Running").__().
|
||||||
|
th().$class("ui-state-default").__("Used Resources").__().
|
||||||
|
th().$class("ui-state-default").__("Total Resources").__().
|
||||||
|
th().$class("ui-state-default").__("Reserved Resources").__().
|
||||||
|
th().$class("ui-state-default").__("Physical Mem Used %").__().
|
||||||
|
th().$class("ui-state-default").__("Physical VCores Used %").__().
|
||||||
|
__().
|
||||||
|
__().
|
||||||
|
// Initialize table data information
|
||||||
|
tbody().$class("ui-widget-content").
|
||||||
|
tr().
|
||||||
|
td(metrics.getAppsSubmitted()).
|
||||||
|
td(metrics.getAppsPending()).
|
||||||
|
td(String.valueOf(metrics.getAppsRunning())).
|
||||||
|
td(metrics.getAppsCompleted()).
|
||||||
|
td(metrics.getAllocatedContainers()).
|
||||||
|
td(metrics.getUsedResources()).
|
||||||
|
td(metrics.getTotalResources()).
|
||||||
|
td(metrics.getReservedResources()).
|
||||||
|
td(metrics.getUtilizedMBPercent()).
|
||||||
|
td(metrics.getUtilizedVirtualCoresPercent()).
|
||||||
|
__().
|
||||||
|
__().__();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Init Federation Cluster Nodes Metrics.
|
||||||
|
*
|
||||||
|
* @param div data display div.
|
||||||
|
* @param metrics data metric information.
|
||||||
|
*/
|
||||||
|
private void initFederationClusterNodesMetrics(Hamlet.DIV<Hamlet> div,
|
||||||
|
RouterClusterMetrics metrics) {
|
||||||
|
div.h3("Federation Cluster Nodes Metrics").
|
||||||
|
table("#nodemetricsoverview").
|
||||||
|
thead().$class("ui-widget-header").
|
||||||
|
// Initialize table header information
|
||||||
|
tr().
|
||||||
|
th().$class("ui-state-default").__("Active Nodes").__().
|
||||||
|
th().$class("ui-state-default").__("Decommissioning Nodes").__().
|
||||||
|
th().$class("ui-state-default").__("Decommissioned Nodes").__().
|
||||||
|
th().$class("ui-state-default").__("Lost Nodes").__().
|
||||||
|
th().$class("ui-state-default").__("Unhealthy Nodes").__().
|
||||||
|
th().$class("ui-state-default").__("Rebooted Nodes").__().
|
||||||
|
th().$class("ui-state-default").__("Shutdown Nodes").__().
|
||||||
|
__().
|
||||||
|
__().
|
||||||
|
// Initialize table data information
|
||||||
|
tbody().$class("ui-widget-content").
|
||||||
|
tr().
|
||||||
|
td(String.valueOf(metrics.getActiveNodes())).
|
||||||
|
td(String.valueOf(metrics.getDecommissioningNodes())).
|
||||||
|
td(String.valueOf(metrics.getDecommissionedNodes())).
|
||||||
|
td(String.valueOf(metrics.getLostNodes())).
|
||||||
|
td(String.valueOf(metrics.getUnhealthyNodes())).
|
||||||
|
td(String.valueOf(metrics.getRebootedNodes())).
|
||||||
|
td(String.valueOf(metrics.getShutdownNodes())).
|
||||||
|
__().
|
||||||
|
__().__();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Init Federation Cluster SchedulersMetrics.
|
||||||
|
*
|
||||||
|
* @param div data display div.
|
||||||
|
* @param metrics data metric information.
|
||||||
|
* @throws YarnException yarn error.
|
||||||
|
* @throws IOException io error.
|
||||||
|
* @throws InterruptedException interrupt error.
|
||||||
|
*/
|
||||||
|
private void initFederationClusterSchedulersMetrics(Hamlet.DIV<Hamlet> div,
|
||||||
|
RouterClusterMetrics metrics) throws YarnException, IOException, InterruptedException {
|
||||||
|
// Sort the SubClusters.
|
||||||
|
List<SubClusterInfo> subclusters = getSubClusterInfoList();
|
||||||
|
|
||||||
|
Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr =
|
||||||
|
div.h3("Federation Scheduler Metrics").
|
||||||
|
table("#schedulermetricsoverview").
|
||||||
|
thead().$class("ui-widget-header").
|
||||||
|
tr().
|
||||||
|
th().$class("ui-state-default").__("SubCluster").__().
|
||||||
|
th().$class("ui-state-default").__("Scheduler Type").__().
|
||||||
|
th().$class("ui-state-default").__("Scheduling Resource Type").__().
|
||||||
|
th().$class("ui-state-default").__("Minimum Allocation").__().
|
||||||
|
th().$class("ui-state-default").__("Maximum Allocation").__().
|
||||||
|
th().$class("ui-state-default").__("Maximum Cluster Application Priority").__().
|
||||||
|
th().$class("ui-state-default").__("Scheduler Busy %").__().
|
||||||
|
th().$class("ui-state-default").__("RM Dispatcher EventQueue Size").__().
|
||||||
|
th().$class("ui-state-default")
|
||||||
|
.__("Scheduler Dispatcher EventQueue Size").__().
|
||||||
|
__().
|
||||||
|
__().
|
||||||
|
tbody().$class("ui-widget-content");
|
||||||
|
|
||||||
|
boolean isEnabled = isYarnFederationEnabled();
|
||||||
|
|
||||||
|
// If Federation mode is not enabled or there is currently no SubCluster available,
|
||||||
|
// each column in the list should be displayed as N/A
|
||||||
|
if (!isEnabled || subclusters == null || subclusters.isEmpty()) {
|
||||||
|
fsMetricsScheduleTr.tr().
|
||||||
|
td(UNAVAILABLE).
|
||||||
|
td(UNAVAILABLE).
|
||||||
|
td(UNAVAILABLE).
|
||||||
|
td(UNAVAILABLE).
|
||||||
|
td(UNAVAILABLE).
|
||||||
|
td(UNAVAILABLE).
|
||||||
|
td(UNAVAILABLE).
|
||||||
|
td(UNAVAILABLE).
|
||||||
|
td(UNAVAILABLE)
|
||||||
|
.__();
|
||||||
|
} else {
|
||||||
|
initSubClusterOverViewTable(metrics, fsMetricsScheduleTr, subclusters);
|
||||||
|
}
|
||||||
|
|
||||||
|
fsMetricsScheduleTr.__().__();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initSubClusterOverViewTable(RouterClusterMetrics metrics,
|
||||||
|
Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr,
|
||||||
|
List<SubClusterInfo> subclusters) {
|
||||||
|
|
||||||
|
// configuration
|
||||||
|
Configuration config = this.router.getConfig();
|
||||||
|
|
||||||
|
Client client = RouterWebServiceUtil.createJerseyClient(config);
|
||||||
|
|
||||||
|
// Traverse all SubClusters to get cluster information.
|
||||||
|
for (SubClusterInfo subcluster : subclusters) {
|
||||||
|
|
||||||
|
// Call the RM interface to obtain schedule information
|
||||||
|
String webAppAddress = WebAppUtils.getHttpSchemePrefix(config) +
|
||||||
|
subcluster.getRMWebServiceAddress();
|
||||||
|
|
||||||
|
SchedulerOverviewInfo typeInfo = RouterWebServiceUtil
|
||||||
|
.genericForward(webAppAddress, null, SchedulerOverviewInfo.class, HTTPMethods.GET,
|
||||||
|
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_OVERVIEW, null, null,
|
||||||
|
config, client);
|
||||||
|
RouterSchedulerMetrics rsMetrics = new RouterSchedulerMetrics(subcluster, metrics, typeInfo);
|
||||||
|
|
||||||
|
// Basic information
|
||||||
|
fsMetricsScheduleTr.tr().
|
||||||
|
td(rsMetrics.getSubCluster()).
|
||||||
|
td(rsMetrics.getSchedulerType()).
|
||||||
|
td(rsMetrics.getSchedulingResourceType()).
|
||||||
|
td(rsMetrics.getMinimumAllocation()).
|
||||||
|
td(rsMetrics.getMaximumAllocation()).
|
||||||
|
td(rsMetrics.getApplicationPriority()).
|
||||||
|
td(rsMetrics.getSchedulerBusy()).
|
||||||
|
td(rsMetrics.getRmDispatcherEventQueueSize()).
|
||||||
|
td(rsMetrics.getSchedulerDispatcherEventQueueSize()).
|
||||||
|
__();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,99 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* <p>
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* <p>
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.server.router.webapp;
|
||||||
|
|
||||||
|
import com.sun.jersey.api.client.Client;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
|
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
|
||||||
|
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
|
||||||
|
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||||
|
import org.apache.hadoop.yarn.server.router.Router;
|
||||||
|
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
|
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Comparator;
|
||||||
|
|
||||||
|
public abstract class RouterBlock extends HtmlBlock {
|
||||||
|
|
||||||
|
private final Router router;
|
||||||
|
|
||||||
|
public RouterBlock(Router router, ViewContext ctx) {
|
||||||
|
super(ctx);
|
||||||
|
this.router = router;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get RouterClusterMetrics Info.
|
||||||
|
*
|
||||||
|
* @return Router ClusterMetricsInfo.
|
||||||
|
*/
|
||||||
|
protected ClusterMetricsInfo getRouterClusterMetricsInfo() {
|
||||||
|
Configuration conf = this.router.getConfig();
|
||||||
|
boolean isEnabled = isYarnFederationEnabled();
|
||||||
|
if(isEnabled) {
|
||||||
|
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
|
||||||
|
Client client = RouterWebServiceUtil.createJerseyClient(conf);
|
||||||
|
ClusterMetricsInfo metrics = RouterWebServiceUtil
|
||||||
|
.genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
|
||||||
|
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
|
||||||
|
conf, client);
|
||||||
|
return metrics;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get a list of subclusters.
|
||||||
|
*
|
||||||
|
* @return subcluster List.
|
||||||
|
* @throws YarnException if the call to the getSubClusters is unsuccessful.
|
||||||
|
*/
|
||||||
|
protected List<SubClusterInfo> getSubClusterInfoList() throws YarnException {
|
||||||
|
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
|
||||||
|
Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true);
|
||||||
|
|
||||||
|
// Sort the SubClusters.
|
||||||
|
List<SubClusterInfo> subclusters = new ArrayList<>();
|
||||||
|
subclusters.addAll(subClustersInfo.values());
|
||||||
|
Comparator<? super SubClusterInfo> cmp = Comparator.comparing(o -> o.getSubClusterId());
|
||||||
|
Collections.sort(subclusters, cmp);
|
||||||
|
|
||||||
|
return subclusters;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether Yarn Federation is enabled.
|
||||||
|
*
|
||||||
|
* @return true, enable yarn federation; false, not enable yarn federation;
|
||||||
|
*/
|
||||||
|
protected boolean isYarnFederationEnabled() {
|
||||||
|
Configuration conf = this.router.getConfig();
|
||||||
|
boolean isEnabled = conf.getBoolean(
|
||||||
|
YarnConfiguration.FEDERATION_ENABLED,
|
||||||
|
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
|
||||||
|
return isEnabled;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,310 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* <p>
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* <p>
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.server.router.webapp.dao;
|
||||||
|
|
||||||
|
import org.apache.hadoop.util.StringUtils;
|
||||||
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||||
|
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import javax.xml.bind.annotation.XmlAccessType;
|
||||||
|
import javax.xml.bind.annotation.XmlAccessorType;
|
||||||
|
import javax.xml.bind.annotation.XmlRootElement;
|
||||||
|
|
||||||
|
@XmlRootElement
|
||||||
|
@XmlAccessorType(XmlAccessType.FIELD)
|
||||||
|
public class RouterClusterMetrics {
|
||||||
|
|
||||||
|
protected static final long BYTES_IN_MB = 1024 * 1024;
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(RouterClusterMetrics.class);
|
||||||
|
|
||||||
|
// Application Information.
|
||||||
|
private String appsSubmitted = "N/A";
|
||||||
|
private String appsCompleted = "N/A";
|
||||||
|
private String appsPending = "N/A";
|
||||||
|
private String appsRunning = "N/A";
|
||||||
|
private String appsFailed = "N/A";
|
||||||
|
private String appsKilled = "N/A";
|
||||||
|
|
||||||
|
// Memory Information.
|
||||||
|
private String totalMemory = "N/A";
|
||||||
|
private String reservedMemory = "N/A";
|
||||||
|
private String availableMemory = "N/A";
|
||||||
|
private String allocatedMemory = "N/A";
|
||||||
|
private String pendingMemory = "N/A";
|
||||||
|
|
||||||
|
// VirtualCores Information.
|
||||||
|
private String reservedVirtualCores = "N/A";
|
||||||
|
private String availableVirtualCores = "N/A";
|
||||||
|
private String allocatedVirtualCores = "N/A";
|
||||||
|
private String pendingVirtualCores = "N/A";
|
||||||
|
private String totalVirtualCores = "N/A";
|
||||||
|
|
||||||
|
// Resources Information.
|
||||||
|
private String usedResources = "N/A";
|
||||||
|
private String totalResources = "N/A";
|
||||||
|
private String reservedResources = "N/A";
|
||||||
|
private String allocatedContainers = "N/A";
|
||||||
|
|
||||||
|
// Resource Percent Information.
|
||||||
|
private String utilizedMBPercent = "N/A";
|
||||||
|
private String utilizedVirtualCoresPercent = "N/A";
|
||||||
|
|
||||||
|
// Node Information.
|
||||||
|
private String activeNodes = "N/A";
|
||||||
|
private String decommissioningNodes = "N/A";
|
||||||
|
private String decommissionedNodes = "N/A";
|
||||||
|
private String lostNodes = "N/A";
|
||||||
|
private String unhealthyNodes = "N/A";
|
||||||
|
private String rebootedNodes = "N/A";
|
||||||
|
private String shutdownNodes = "N/A";
|
||||||
|
|
||||||
|
public RouterClusterMetrics() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public RouterClusterMetrics(ClusterMetricsInfo metrics) {
|
||||||
|
if (metrics != null) {
|
||||||
|
// Application Information Conversion.
|
||||||
|
conversionApplicationInformation(metrics);
|
||||||
|
|
||||||
|
// Memory Information Conversion.
|
||||||
|
conversionMemoryInformation(metrics);
|
||||||
|
|
||||||
|
// Resources Information Conversion.
|
||||||
|
conversionResourcesInformation(metrics);
|
||||||
|
|
||||||
|
// Percent Information Conversion.
|
||||||
|
conversionResourcesPercent(metrics);
|
||||||
|
|
||||||
|
// Node Information Conversion.
|
||||||
|
conversionNodeInformation(metrics);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get Key Metric Information
|
||||||
|
public String getAppsSubmitted() {
|
||||||
|
return appsSubmitted;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAppsCompleted() {
|
||||||
|
return appsCompleted;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAppsPending() {
|
||||||
|
return appsPending;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAppsRunning() {
|
||||||
|
return appsRunning;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAppsFailed() {
|
||||||
|
return appsFailed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAppsKilled() {
|
||||||
|
return appsKilled;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTotalMemory() {
|
||||||
|
return totalMemory;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getReservedMemory() {
|
||||||
|
return reservedMemory;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAvailableMemory() {
|
||||||
|
return availableMemory;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAllocatedMemory() {
|
||||||
|
return allocatedMemory;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPendingMemory() {
|
||||||
|
return pendingMemory;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getReservedVirtualCores() {
|
||||||
|
return reservedVirtualCores;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAvailableVirtualCores() {
|
||||||
|
return availableVirtualCores;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAllocatedVirtualCores() {
|
||||||
|
return allocatedVirtualCores;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPendingVirtualCores() {
|
||||||
|
return pendingVirtualCores;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTotalVirtualCores() {
|
||||||
|
return totalVirtualCores;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUsedResources() {
|
||||||
|
return usedResources;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTotalResources() {
|
||||||
|
return totalResources;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getReservedResources() {
|
||||||
|
return reservedResources;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAllocatedContainers() {
|
||||||
|
return allocatedContainers;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUtilizedMBPercent() {
|
||||||
|
return utilizedMBPercent;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUtilizedVirtualCoresPercent() {
|
||||||
|
return utilizedVirtualCoresPercent;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getActiveNodes() {
|
||||||
|
return activeNodes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDecommissioningNodes() {
|
||||||
|
return decommissioningNodes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDecommissionedNodes() {
|
||||||
|
return decommissionedNodes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getLostNodes() {
|
||||||
|
return lostNodes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUnhealthyNodes() {
|
||||||
|
return unhealthyNodes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRebootedNodes() {
|
||||||
|
return rebootedNodes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getShutdownNodes() {
|
||||||
|
return shutdownNodes;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Metric Information Conversion
|
||||||
|
public void conversionApplicationInformation(ClusterMetricsInfo metrics) {
|
||||||
|
try {
|
||||||
|
// Application Information.
|
||||||
|
this.appsSubmitted = String.valueOf(metrics.getAppsSubmitted());
|
||||||
|
this.appsCompleted = String.valueOf(metrics.getAppsCompleted() +
|
||||||
|
metrics.getAppsFailed() + metrics.getAppsKilled());
|
||||||
|
this.appsPending = String.valueOf(metrics.getAppsPending());
|
||||||
|
this.appsRunning = String.valueOf(metrics.getAppsRunning());
|
||||||
|
this.appsFailed = String.valueOf(metrics.getAppsFailed());
|
||||||
|
this.appsKilled = String.valueOf(metrics.getAppsKilled());
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("conversionApplicationInformation error.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Metric Memory Information
|
||||||
|
public void conversionMemoryInformation(ClusterMetricsInfo metrics) {
|
||||||
|
try {
|
||||||
|
// Memory Information.
|
||||||
|
this.totalMemory = StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB);
|
||||||
|
this.reservedMemory = StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB);
|
||||||
|
this.availableMemory = StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB);
|
||||||
|
this.allocatedMemory = StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB);
|
||||||
|
this.pendingMemory = StringUtils.byteDesc(metrics.getPendingMB() * BYTES_IN_MB);
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("conversionMemoryInformation error.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResourcesInformation Conversion
|
||||||
|
public void conversionResourcesInformation(ClusterMetricsInfo metrics) {
|
||||||
|
try {
|
||||||
|
// Parse resource information from metrics.
|
||||||
|
Resource metricUsedResources;
|
||||||
|
Resource metricTotalResources;
|
||||||
|
Resource metricReservedResources;
|
||||||
|
|
||||||
|
int metricAllocatedContainers;
|
||||||
|
if (metrics.getCrossPartitionMetricsAvailable()) {
|
||||||
|
metricAllocatedContainers = metrics.getTotalAllocatedContainersAcrossPartition();
|
||||||
|
metricUsedResources = metrics.getTotalUsedResourcesAcrossPartition().getResource();
|
||||||
|
metricTotalResources = metrics.getTotalClusterResourcesAcrossPartition().getResource();
|
||||||
|
metricReservedResources = metrics.getTotalReservedResourcesAcrossPartition().getResource();
|
||||||
|
// getTotalUsedResourcesAcrossPartition includes reserved resources.
|
||||||
|
Resources.subtractFrom(metricUsedResources, metricReservedResources);
|
||||||
|
} else {
|
||||||
|
metricAllocatedContainers = metrics.getContainersAllocated();
|
||||||
|
metricUsedResources = Resource.newInstance(metrics.getAllocatedMB(),
|
||||||
|
(int) metrics.getAllocatedVirtualCores());
|
||||||
|
metricTotalResources = Resource.newInstance(metrics.getTotalMB(),
|
||||||
|
(int) metrics.getTotalVirtualCores());
|
||||||
|
metricReservedResources = Resource.newInstance(metrics.getReservedMB(),
|
||||||
|
(int) metrics.getReservedVirtualCores());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert to standard format.
|
||||||
|
usedResources = metricUsedResources.getFormattedString();
|
||||||
|
totalResources = metricTotalResources.getFormattedString();
|
||||||
|
reservedResources = metricReservedResources.getFormattedString();
|
||||||
|
allocatedContainers = String.valueOf(metricAllocatedContainers);
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("conversionResourcesInformation error.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResourcesPercent Conversion
|
||||||
|
public void conversionResourcesPercent(ClusterMetricsInfo metrics) {
|
||||||
|
try {
|
||||||
|
this.utilizedMBPercent = String.valueOf(metrics.getUtilizedMBPercent());
|
||||||
|
this.utilizedVirtualCoresPercent = String.valueOf(metrics.getUtilizedVirtualCoresPercent());
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("conversionResourcesPercent error.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NodeInformation Conversion
|
||||||
|
public void conversionNodeInformation(ClusterMetricsInfo metrics) {
|
||||||
|
try {
|
||||||
|
this.activeNodes = String.valueOf(metrics.getActiveNodes());
|
||||||
|
this.decommissioningNodes = String.valueOf(metrics.getDecommissioningNodes());
|
||||||
|
this.decommissionedNodes = String.valueOf(metrics.getDecommissionedNodes());
|
||||||
|
this.lostNodes = String.valueOf(metrics.getLostNodes());
|
||||||
|
this.unhealthyNodes = String.valueOf(metrics.getUnhealthyNodes());
|
||||||
|
this.rebootedNodes = String.valueOf(metrics.getRebootedNodes());
|
||||||
|
this.shutdownNodes = String.valueOf(metrics.getShutdownNodes());
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("conversionNodeInformation error.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,104 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* <p>
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* <p>
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.server.router.webapp.dao;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.service.Service;
|
||||||
|
import org.apache.hadoop.util.VersionInfo;
|
||||||
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
|
import org.apache.hadoop.yarn.server.router.Router;
|
||||||
|
import org.apache.hadoop.yarn.util.YarnVersionInfo;
|
||||||
|
|
||||||
|
import javax.xml.bind.annotation.XmlAccessType;
|
||||||
|
import javax.xml.bind.annotation.XmlAccessorType;
|
||||||
|
import javax.xml.bind.annotation.XmlRootElement;
|
||||||
|
|
||||||
|
@XmlRootElement
|
||||||
|
@XmlAccessorType(XmlAccessType.FIELD)
|
||||||
|
public class RouterInfo {
|
||||||
|
private long id;
|
||||||
|
private long startedOn;
|
||||||
|
private Service.STATE state;
|
||||||
|
private String routerStateStoreName;
|
||||||
|
private String routerVersion;
|
||||||
|
private String routerBuildVersion;
|
||||||
|
private String routerVersionBuiltOn;
|
||||||
|
private String hadoopVersion;
|
||||||
|
private String hadoopBuildVersion;
|
||||||
|
private String hadoopVersionBuiltOn;
|
||||||
|
|
||||||
|
public RouterInfo() {
|
||||||
|
} // JAXB needs this
|
||||||
|
|
||||||
|
public RouterInfo(Router router) {
|
||||||
|
long ts = Router.getClusterTimeStamp();
|
||||||
|
this.id = ts;
|
||||||
|
this.state = router.getServiceState();
|
||||||
|
Configuration configuration = router.getConfig();
|
||||||
|
this.routerStateStoreName = configuration.get(
|
||||||
|
YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS,
|
||||||
|
YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
|
||||||
|
this.routerVersion = YarnVersionInfo.getVersion();
|
||||||
|
this.routerBuildVersion = YarnVersionInfo.getBuildVersion();
|
||||||
|
this.routerVersionBuiltOn = YarnVersionInfo.getDate();
|
||||||
|
this.hadoopVersion = VersionInfo.getVersion();
|
||||||
|
this.hadoopBuildVersion = VersionInfo.getBuildVersion();
|
||||||
|
this.hadoopVersionBuiltOn = VersionInfo.getDate();
|
||||||
|
this.startedOn = ts;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getState() {
|
||||||
|
return this.state.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRouterStateStore() {
|
||||||
|
return this.routerStateStoreName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRouterVersion() {
|
||||||
|
return this.routerVersion;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRouterBuildVersion() {
|
||||||
|
return this.routerBuildVersion;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRouterVersionBuiltOn() {
|
||||||
|
return this.routerVersionBuiltOn;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getHadoopVersion() {
|
||||||
|
return this.hadoopVersion;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getHadoopBuildVersion() {
|
||||||
|
return this.hadoopBuildVersion;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getHadoopVersionBuiltOn() {
|
||||||
|
return this.hadoopVersionBuiltOn;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getClusterId() {
|
||||||
|
return this.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getStartedOn() {
|
||||||
|
return this.startedOn;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,109 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* <p>
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* <p>
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.server.router.webapp.dao;
|
||||||
|
|
||||||
|
|
||||||
|
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import javax.xml.bind.annotation.XmlAccessType;
|
||||||
|
import javax.xml.bind.annotation.XmlAccessorType;
|
||||||
|
import javax.xml.bind.annotation.XmlRootElement;
|
||||||
|
|
||||||
|
@XmlRootElement
|
||||||
|
@XmlAccessorType(XmlAccessType.FIELD)
|
||||||
|
public class RouterSchedulerMetrics {
|
||||||
|
|
||||||
|
// Metrics Log.
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(RouterSchedulerMetrics.class);
|
||||||
|
|
||||||
|
// Scheduler Information.
|
||||||
|
private String subCluster = "N/A";
|
||||||
|
private String schedulerType = "N/A";
|
||||||
|
private String schedulingResourceType = "N/A";
|
||||||
|
private String minimumAllocation = "N/A";
|
||||||
|
private String maximumAllocation = "N/A";
|
||||||
|
private String applicationPriority = "N/A";
|
||||||
|
private String schedulerBusy = "N/A";
|
||||||
|
private String rmDispatcherEventQueueSize = "N/A";
|
||||||
|
private String schedulerDispatcherEventQueueSize = "N/A";
|
||||||
|
|
||||||
|
public RouterSchedulerMetrics() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public RouterSchedulerMetrics(SubClusterInfo subClusterInfo, RouterClusterMetrics metrics,
|
||||||
|
SchedulerOverviewInfo overview) {
|
||||||
|
try {
|
||||||
|
// Parse Scheduler Information.
|
||||||
|
this.subCluster = subClusterInfo.getSubClusterId().getId();
|
||||||
|
this.schedulerType = overview.getSchedulerType();
|
||||||
|
this.schedulingResourceType = overview.getSchedulingResourceType();
|
||||||
|
this.minimumAllocation = overview.getMinimumAllocation().toString();
|
||||||
|
this.maximumAllocation = overview.getMaximumAllocation().toString();
|
||||||
|
this.applicationPriority = String.valueOf(overview.getApplicationPriority());
|
||||||
|
if (overview.getSchedulerBusy() != -1) {
|
||||||
|
this.schedulerBusy = String.valueOf(overview.getSchedulerBusy());
|
||||||
|
}
|
||||||
|
this.rmDispatcherEventQueueSize =
|
||||||
|
String.valueOf(overview.getRmDispatcherEventQueueSize());
|
||||||
|
this.schedulerDispatcherEventQueueSize =
|
||||||
|
String.valueOf(overview.getSchedulerDispatcherEventQueueSize());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
LOG.error("RouterSchedulerMetrics Error.", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSubCluster() {
|
||||||
|
return subCluster;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSchedulerType() {
|
||||||
|
return schedulerType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSchedulingResourceType() {
|
||||||
|
return schedulingResourceType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getMinimumAllocation() {
|
||||||
|
return minimumAllocation;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getMaximumAllocation() {
|
||||||
|
return maximumAllocation;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getApplicationPriority() {
|
||||||
|
return applicationPriority;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRmDispatcherEventQueueSize() {
|
||||||
|
return rmDispatcherEventQueueSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSchedulerDispatcherEventQueueSize() {
|
||||||
|
return schedulerDispatcherEventQueueSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSchedulerBusy() {
|
||||||
|
return schedulerBusy;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,20 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** Router Web Dao package. **/
|
||||||
|
package org.apache.hadoop.yarn.server.router.webapp.dao;
|
|
@ -26,7 +26,7 @@ import org.junit.Test;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
public class TestFederationWebApp {
|
public class TestFederationWebApp extends TestRouterWebServicesREST {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testFederationWebViewNotEnable()
|
public void testFederationWebViewNotEnable()
|
||||||
|
@ -45,4 +45,22 @@ public class TestFederationWebApp {
|
||||||
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
|
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
|
||||||
WebAppTests.testPage(FederationPage.class, Router.class, new MockRouter(config));
|
WebAppTests.testPage(FederationPage.class, Router.class, new MockRouter(config));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFederationAboutViewEnable()
|
||||||
|
throws InterruptedException, YarnException, IOException {
|
||||||
|
// Test Federation Enabled
|
||||||
|
Configuration config = new YarnConfiguration();
|
||||||
|
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
|
||||||
|
WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFederationAboutViewNotEnable()
|
||||||
|
throws InterruptedException, YarnException, IOException {
|
||||||
|
// Test Federation Not Enabled
|
||||||
|
Configuration config = new YarnConfiguration();
|
||||||
|
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
|
||||||
|
WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue