YARN-3661. Basic Federation UI. (Contributed by Inigo Goiri via curino)
(cherry picked from commit ceca9694f9)
This commit is contained in:
Carlo Curino
Andrew Wang
parent
e785e83270
commit
1fbc72704e
|
|
@ -0,0 +1,91 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||
import org.apache.hadoop.yarn.server.router.Router;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
|
||||
/**
|
||||
* About block for the Router Web UI.
|
||||
*/
|
||||
public class AboutBlock extends HtmlBlock {
|
||||
|
||||
private static final long BYTES_IN_MB = 1024 * 1024;
|
||||
|
||||
private final Router router;
|
||||
|
||||
@Inject
|
||||
AboutBlock(Router router, ViewContext ctx) {
|
||||
super(ctx);
|
||||
this.router = router;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void render(Block html) {
|
||||
Configuration conf = this.router.getConfig();
|
||||
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
|
||||
|
||||
ClusterMetricsInfo metrics = RouterWebServiceUtil.genericForward(
|
||||
webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
|
||||
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null);
|
||||
boolean isEnabled = conf.getBoolean(
|
||||
YarnConfiguration.FEDERATION_ENABLED,
|
||||
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
|
||||
info("Cluster Status").
|
||||
__("Federation Enabled", isEnabled).
|
||||
__("Applications Submitted", "N/A").
|
||||
__("Applications Pending", "N/A").
|
||||
__("Applications Running", "N/A").
|
||||
__("Applications Failed", "N/A").
|
||||
__("Applications Killed", "N/A").
|
||||
__("Applications Completed", "N/A").
|
||||
__("Containers Allocated", metrics.getContainersAllocated()).
|
||||
__("Containers Reserved", metrics.getReservedContainers()).
|
||||
__("Containers Pending", metrics.getPendingContainers()).
|
||||
__("Available Memory",
|
||||
StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB)).
|
||||
__("Allocated Memory",
|
||||
StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB)).
|
||||
__("Reserved Memory",
|
||||
StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB)).
|
||||
__("Total Memory",
|
||||
StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB)).
|
||||
__("Available VirtualCores", metrics.getAvailableVirtualCores()).
|
||||
__("Allocated VirtualCores", metrics.getAllocatedVirtualCores()).
|
||||
__("Reserved VirtualCores", metrics.getReservedVirtualCores()).
|
||||
__("Total VirtualCores", metrics.getTotalVirtualCores()).
|
||||
__("Active Nodes", metrics.getActiveNodes()).
|
||||
__("Lost Nodes", metrics.getLostNodes()).
|
||||
__("Available Nodes", metrics.getDecommissionedNodes()).
|
||||
__("Unhealthy Nodes", metrics.getUnhealthyNodes()).
|
||||
__("Rebooted Nodes", metrics.getRebootedNodes()).
|
||||
__("Total Nodes", metrics.getTotalNodes());
|
||||
|
||||
html.__(InfoBlock.class);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import org.apache.hadoop.yarn.webapp.SubView;
|
||||
|
||||
/**
|
||||
* About page for the Router Web UI.
|
||||
*/
|
||||
public class AboutPage extends RouterView {
|
||||
|
||||
@Override
|
||||
protected void preHead(Page.HTML<__> html) {
|
||||
commonPreHead(html);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Class<? extends SubView> content() {
|
||||
return AboutBlock.class;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
|
||||
import static org.apache.commons.lang.StringEscapeUtils.escapeJavaScript;
|
||||
import static org.apache.hadoop.yarn.util.StringHelper.join;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
|
||||
import org.apache.hadoop.yarn.server.router.Router;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
|
||||
/**
|
||||
* Applications block for the Router Web UI.
|
||||
*/
|
||||
public class AppsBlock extends HtmlBlock {
|
||||
private final Router router;
|
||||
|
||||
@Inject
|
||||
AppsBlock(Router router, ViewContext ctx) {
|
||||
super(ctx);
|
||||
this.router = router;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void render(Block html) {
|
||||
// Get the applications from the Resource Managers
|
||||
Configuration conf = this.router.getConfig();
|
||||
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
|
||||
AppsInfo apps = RouterWebServiceUtil.genericForward(webAppAddress, null,
|
||||
AppsInfo.class, HTTPMethods.GET,
|
||||
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null);
|
||||
|
||||
setTitle("Applications");
|
||||
|
||||
TBODY<TABLE<Hamlet>> tbody = html.table("#apps").thead()
|
||||
.tr()
|
||||
.th(".id", "ID")
|
||||
.th(".user", "User")
|
||||
.th(".name", "Name")
|
||||
.th(".type", "Application Type")
|
||||
.th(".queue", "Queue")
|
||||
.th(".priority", "Application Priority")
|
||||
.th(".starttime", "StartTime")
|
||||
.th(".finishtime", "FinishTime")
|
||||
.th(".state", "State")
|
||||
.th(".finalstatus", "FinalStatus")
|
||||
.th(".progress", "Progress")
|
||||
.th(".ui", "Tracking UI")
|
||||
.__().__().tbody();
|
||||
|
||||
// Render the applications
|
||||
StringBuilder appsTableData = new StringBuilder("[\n");
|
||||
for (AppInfo app : apps.getApps()) {
|
||||
try {
|
||||
|
||||
String percent = String.format("%.1f", app.getProgress() * 100.0F);
|
||||
String trackingURL =
|
||||
app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
|
||||
// AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
|
||||
appsTableData.append("[\"")
|
||||
.append("<a href='").append(trackingURL).append("'>")
|
||||
.append(app.getAppId()).append("</a>\",\"")
|
||||
.append(escape(app.getUser())).append("\",\"")
|
||||
.append(escape(app.getName())).append("\",\"")
|
||||
.append(escape(app.getApplicationType())).append("\",\"")
|
||||
.append(escape(app.getQueue())).append("\",\"")
|
||||
.append(String.valueOf(app.getPriority())).append("\",\"")
|
||||
.append(app.getStartTime()).append("\",\"")
|
||||
.append(app.getFinishTime()).append("\",\"")
|
||||
.append(app.getState()).append("\",\"")
|
||||
.append(app.getFinalStatus()).append("\",\"")
|
||||
// Progress bar
|
||||
.append("<br title='").append(percent).append("'> <div class='")
|
||||
.append(C_PROGRESSBAR).append("' title='")
|
||||
.append(join(percent, '%')).append("'> ").append("<div class='")
|
||||
.append(C_PROGRESSBAR_VALUE).append("' style='")
|
||||
.append(join("width:", percent, '%')).append("'> </div> </div>")
|
||||
// History link
|
||||
.append("\",\"<a href='").append(trackingURL).append("'>")
|
||||
.append("History").append("</a>");
|
||||
appsTableData.append("\"],\n");
|
||||
|
||||
} catch (Exception e) {
|
||||
LOG.info(
|
||||
"Cannot add application {}: {}", app.getAppId(), e.getMessage());
|
||||
}
|
||||
}
|
||||
if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
|
||||
appsTableData.delete(appsTableData.length() - 2,
|
||||
appsTableData.length() - 1);
|
||||
}
|
||||
appsTableData.append("]");
|
||||
html.script().$type("text/javascript")
|
||||
.__("var appsTableData=" + appsTableData).__();
|
||||
|
||||
tbody.__().__();
|
||||
}
|
||||
|
||||
private static String escape(String str) {
|
||||
return escapeJavaScript(escapeHtml(str));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
|
||||
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
|
||||
|
||||
import org.apache.hadoop.yarn.webapp.SubView;
|
||||
|
||||
class AppsPage extends RouterView {
|
||||
|
||||
@Override
|
||||
protected void preHead(Page.HTML<__> html) {
|
||||
commonPreHead(html);
|
||||
set(DATATABLES_ID, "apps");
|
||||
set(initID(DATATABLES, "apps"), appsTableInit());
|
||||
setTableStyles(html, "apps", ".queue {width:6em}", ".ui {width:8em}");
|
||||
|
||||
// Set the correct title.
|
||||
String reqState = $(APP_STATE);
|
||||
reqState = (reqState == null || reqState.isEmpty() ? "All" : reqState);
|
||||
setTitle(sjoin(reqState, "Applications"));
|
||||
}
|
||||
|
||||
private String appsTableInit() {
|
||||
// id, user, name, queue, starttime, finishtime, state, status, progress, ui
|
||||
return tableInit()
|
||||
.append(", 'aaData': appsTableData")
|
||||
.append(", bDeferRender: true")
|
||||
.append(", bProcessing: true")
|
||||
|
||||
.append("\n, aoColumnDefs: ")
|
||||
.append(getAppsTableColumnDefs())
|
||||
|
||||
// Sort by id upon page load
|
||||
.append(", aaSorting: [[0, 'desc']]}").toString();
|
||||
}
|
||||
|
||||
protected String getAppsTableColumnDefs() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
return sb
|
||||
.append("[\n")
|
||||
.append("{'sType':'string', 'aTargets': [0]")
|
||||
.append(", 'mRender': parseHadoopID }")
|
||||
|
||||
.append("\n, {'sType':'numeric', 'aTargets': [6, 7]")
|
||||
.append(", 'mRender': renderHadoopDate }")
|
||||
|
||||
.append("\n, {'sType':'numeric', bSearchable:false, 'aTargets': [10]")
|
||||
.append(", 'mRender': parseHadoopProgress }]").toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Class<? extends SubView> content() {
|
||||
return AppsBlock.class;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,176 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
|
||||
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
|
||||
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||
import org.apache.hadoop.yarn.server.router.Router;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
import com.sun.jersey.api.json.JSONConfiguration;
|
||||
import com.sun.jersey.api.json.JSONJAXBContext;
|
||||
import com.sun.jersey.api.json.JSONUnmarshaller;
|
||||
|
||||
class FederationBlock extends HtmlBlock {
|
||||
|
||||
private static final long BYTES_IN_MB = 1024 * 1024;
|
||||
|
||||
private final Router router;
|
||||
|
||||
@Inject
|
||||
FederationBlock(ViewContext ctx, Router router) {
|
||||
super(ctx);
|
||||
this.router = router;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void render(Block html) {
|
||||
Configuration conf = this.router.getConfig();
|
||||
boolean isEnabled = conf.getBoolean(
|
||||
YarnConfiguration.FEDERATION_ENABLED,
|
||||
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
|
||||
if (isEnabled) {
|
||||
setTitle("Federation");
|
||||
|
||||
// Table header
|
||||
TBODY<TABLE<Hamlet>> tbody = html.table("#rms").thead().tr()
|
||||
.th(".id", "SubCluster")
|
||||
.th(".submittedA", "Applications Submitted*")
|
||||
.th(".pendingA", "Applications Pending*")
|
||||
.th(".runningA", "Applications Running*")
|
||||
.th(".failedA", "Applications Failed*")
|
||||
.th(".killedA", "Applications Killed*")
|
||||
.th(".completedA", "Applications Completed*")
|
||||
.th(".contAllocated", "Containers Allocated")
|
||||
.th(".contReserved", "Containers Reserved")
|
||||
.th(".contPending", "Containers Pending")
|
||||
.th(".availableM", "Available Memory")
|
||||
.th(".allocatedM", "Allocated Memory")
|
||||
.th(".reservedM", "Reserved Memory")
|
||||
.th(".totalM", "Total Memory")
|
||||
.th(".availableVC", "Available VirtualCores")
|
||||
.th(".allocatedVC", "Allocated VirtualCores")
|
||||
.th(".reservedVC", "Reserved VirtualCores")
|
||||
.th(".totalVC", "Total VirtualCores")
|
||||
.th(".activeN", "Active Nodes")
|
||||
.th(".lostN", "Lost Nodes")
|
||||
.th(".availableN", "Available Nodes")
|
||||
.th(".unhealtyN", "Unhealthy Nodes")
|
||||
.th(".rebootedN", "Rebooted Nodes")
|
||||
.th(".totalN", "Total Nodes")
|
||||
.__().__().tbody();
|
||||
|
||||
try {
|
||||
// Binding to the FederationStateStore
|
||||
FederationStateStoreFacade facade =
|
||||
FederationStateStoreFacade.getInstance();
|
||||
Map<SubClusterId, SubClusterInfo> subClustersInfo =
|
||||
facade.getSubClusters(true);
|
||||
|
||||
// Sort the SubClusters
|
||||
List<SubClusterInfo> subclusters = new ArrayList<>();
|
||||
subclusters.addAll(subClustersInfo.values());
|
||||
Comparator<? super SubClusterInfo> cmp =
|
||||
new Comparator<SubClusterInfo>() {
|
||||
@Override
|
||||
public int compare(SubClusterInfo o1, SubClusterInfo o2) {
|
||||
return o1.getSubClusterId().compareTo(o2.getSubClusterId());
|
||||
}
|
||||
};
|
||||
Collections.sort(subclusters, cmp);
|
||||
|
||||
for (SubClusterInfo subcluster : subclusters) {
|
||||
SubClusterId subClusterId = subcluster.getSubClusterId();
|
||||
String webAppAddress = subcluster.getRMWebServiceAddress();
|
||||
String capability = subcluster.getCapability();
|
||||
ClusterMetricsInfo subClusterInfo = getClusterMetricsInfo(capability);
|
||||
|
||||
// Building row per SubCluster
|
||||
tbody.tr().td().a("//" + webAppAddress, subClusterId.toString()).__()
|
||||
.td(Integer.toString(subClusterInfo.getAppsSubmitted()))
|
||||
.td(Integer.toString(subClusterInfo.getAppsPending()))
|
||||
.td(Integer.toString(subClusterInfo.getAppsRunning()))
|
||||
.td(Integer.toString(subClusterInfo.getAppsFailed()))
|
||||
.td(Integer.toString(subClusterInfo.getAppsKilled()))
|
||||
.td(Integer.toString(subClusterInfo.getAppsCompleted()))
|
||||
.td(Integer.toString(subClusterInfo.getContainersAllocated()))
|
||||
.td(Integer.toString(subClusterInfo.getReservedContainers()))
|
||||
.td(Integer.toString(subClusterInfo.getPendingContainers()))
|
||||
.td(StringUtils.byteDesc(
|
||||
subClusterInfo.getAvailableMB() * BYTES_IN_MB))
|
||||
.td(StringUtils.byteDesc(
|
||||
subClusterInfo.getAllocatedMB() * BYTES_IN_MB))
|
||||
.td(StringUtils.byteDesc(
|
||||
subClusterInfo.getReservedMB() * BYTES_IN_MB))
|
||||
.td(StringUtils.byteDesc(
|
||||
subClusterInfo.getTotalMB() * BYTES_IN_MB))
|
||||
.td(Long.toString(subClusterInfo.getAvailableVirtualCores()))
|
||||
.td(Long.toString(subClusterInfo.getAllocatedVirtualCores()))
|
||||
.td(Long.toString(subClusterInfo.getReservedVirtualCores()))
|
||||
.td(Long.toString(subClusterInfo.getTotalVirtualCores()))
|
||||
.td(Integer.toString(subClusterInfo.getActiveNodes()))
|
||||
.td(Integer.toString(subClusterInfo.getLostNodes()))
|
||||
.td(Integer.toString(subClusterInfo.getDecommissionedNodes()))
|
||||
.td(Integer.toString(subClusterInfo.getUnhealthyNodes()))
|
||||
.td(Integer.toString(subClusterInfo.getRebootedNodes()))
|
||||
.td(Integer.toString(subClusterInfo.getTotalNodes())).__();
|
||||
}
|
||||
} catch (YarnException e) {
|
||||
LOG.error("Cannot render ResourceManager", e);
|
||||
}
|
||||
|
||||
tbody.__().__().div()
|
||||
.p().__("*The application counts are local per subcluster").__().__();
|
||||
} else {
|
||||
setTitle("Federation is not Enabled!");
|
||||
}
|
||||
}
|
||||
|
||||
private static ClusterMetricsInfo getClusterMetricsInfo(String capability) {
|
||||
ClusterMetricsInfo clusterMetrics = null;
|
||||
try {
|
||||
JSONJAXBContext jc = new JSONJAXBContext(
|
||||
JSONConfiguration.mapped().rootUnwrapping(false).build(),
|
||||
ClusterMetricsInfo.class);
|
||||
JSONUnmarshaller unmarshaller = jc.createJSONUnmarshaller();
|
||||
clusterMetrics = unmarshaller.unmarshalFromJSON(
|
||||
new StringReader(capability), ClusterMetricsInfo.class);
|
||||
} catch (Exception e) {
|
||||
LOG.error("Cannot parse SubCluster info", e);
|
||||
}
|
||||
return clusterMetrics;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
|
||||
|
||||
import org.apache.hadoop.yarn.webapp.SubView;
|
||||
|
||||
/**
|
||||
* Renders a block for the applications with metrics information.
|
||||
*/
|
||||
class FederationPage extends RouterView {
|
||||
|
||||
@Override
|
||||
protected void preHead(Page.HTML<__> html) {
|
||||
commonPreHead(html);
|
||||
setTitle("Federation");
|
||||
set(DATATABLES_ID, "rms");
|
||||
set(initID(DATATABLES, "rms"), rmsTableInit());
|
||||
setTableStyles(html, "rms", ".healthStatus {width:10em}",
|
||||
".healthReport {width:10em}");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Class<? extends SubView> content() {
|
||||
return FederationBlock.class;
|
||||
}
|
||||
|
||||
private String rmsTableInit() {
|
||||
StringBuilder b = tableInit().append(", aoColumnDefs: [");
|
||||
b.append("{'bSearchable': false, 'aTargets': [ 7 ]}");
|
||||
b.append(", {'sType': 'title-numeric', 'bSearchable': false, "
|
||||
+ "'aTargets': [ 8, 9 ] }");
|
||||
b.append(", {'sType': 'title-numeric', 'aTargets': [ 5 ]}");
|
||||
b.append("]}");
|
||||
return b.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
|
||||
/**
|
||||
* Navigation block for the Router Web UI.
|
||||
*/
|
||||
public class NavBlock extends HtmlBlock {
|
||||
|
||||
@Override
|
||||
public void render(Block html) {
|
||||
html.
|
||||
div("#nav").
|
||||
h3("Cluster").
|
||||
ul().
|
||||
li().a(url(""), "About").__().
|
||||
li().a(url("federation"), "Federation").__().
|
||||
li().a(url("nodes"), "Nodes").__().
|
||||
li().a(url("apps"), "Applications").__().
|
||||
__().
|
||||
h3("Tools").
|
||||
ul().
|
||||
li().a("/conf", "Configuration").__().
|
||||
li().a("/logs", "Local logs").__().
|
||||
li().a("/stacks", "Server stacks").__().
|
||||
li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
|
||||
import org.apache.hadoop.yarn.server.router.Router;
|
||||
import org.apache.hadoop.yarn.util.Times;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
|
||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TR;
|
||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
|
||||
/**
|
||||
* Nodes block for the Router Web UI.
|
||||
*/
|
||||
public class NodesBlock extends HtmlBlock {
|
||||
|
||||
private static final long BYTES_IN_MB = 1024 * 1024;
|
||||
|
||||
private final Router router;
|
||||
|
||||
@Inject
|
||||
NodesBlock(Router router, ViewContext ctx) {
|
||||
super(ctx);
|
||||
this.router = router;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void render(Block html) {
|
||||
// Get the node info from the federation
|
||||
Configuration conf = this.router.getConfig();
|
||||
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
|
||||
NodesInfo nodes = RouterWebServiceUtil.genericForward(webAppAddress, null,
|
||||
NodesInfo.class, HTTPMethods.GET,
|
||||
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null);
|
||||
|
||||
setTitle("Nodes");
|
||||
|
||||
TBODY<TABLE<Hamlet>> tbody = html.table("#nodes").thead().tr()
|
||||
.th(".nodelabels", "Node Labels")
|
||||
.th(".rack", "Rack")
|
||||
.th(".state", "Node State")
|
||||
.th(".nodeaddress", "Node Address")
|
||||
.th(".nodehttpaddress", "Node HTTP Address")
|
||||
.th(".lastHealthUpdate", "Last health-update")
|
||||
.th(".healthReport", "Health-report")
|
||||
.th(".containers", "Containers")
|
||||
.th(".mem", "Mem Used")
|
||||
.th(".mem", "Mem Avail")
|
||||
.th(".vcores", "VCores Used")
|
||||
.th(".vcores", "VCores Avail")
|
||||
.th(".nodeManagerVersion", "Version")
|
||||
.__().__().tbody();
|
||||
|
||||
// Add nodes to the web UI
|
||||
for (NodeInfo info : nodes.getNodes()) {
|
||||
int usedMemory = (int) info.getUsedMemory();
|
||||
int availableMemory = (int) info.getAvailableMemory();
|
||||
TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
|
||||
row.td().__(StringUtils.join(",", info.getNodeLabels())).__();
|
||||
row.td().__(info.getRack()).__();
|
||||
row.td().__(info.getState()).__();
|
||||
row.td().__(info.getNodeId()).__();
|
||||
boolean isInactive = false;
|
||||
if (isInactive) {
|
||||
row.td().__("N/A").__();
|
||||
} else {
|
||||
String httpAddress = info.getNodeHTTPAddress();
|
||||
row.td().a("//" + httpAddress, httpAddress).__();
|
||||
}
|
||||
row.td().br().$title(String.valueOf(info.getLastHealthUpdate())).__()
|
||||
.__(Times.format(info.getLastHealthUpdate())).__()
|
||||
.td(info.getHealthReport())
|
||||
.td(String.valueOf(info.getNumContainers())).td().br()
|
||||
.$title(String.valueOf(usedMemory)).__()
|
||||
.__(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)).__().td().br()
|
||||
.$title(String.valueOf(availableMemory)).__()
|
||||
.__(StringUtils.byteDesc(availableMemory * BYTES_IN_MB)).__()
|
||||
.td(String.valueOf(info.getUsedVirtualCores()))
|
||||
.td(String.valueOf(info.getAvailableVirtualCores()))
|
||||
.td(info.getVersion()).__();
|
||||
}
|
||||
tbody.__().__();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
|
||||
|
||||
import org.apache.hadoop.yarn.webapp.SubView;
|
||||
|
||||
class NodesPage extends RouterView {
|
||||
|
||||
@Override
|
||||
protected void preHead(Page.HTML<__> html) {
|
||||
commonPreHead(html);
|
||||
String type = $(NODE_STATE);
|
||||
String title = "Nodes of the cluster";
|
||||
if (type != null && !type.isEmpty()) {
|
||||
title = title + " (" + type + ")";
|
||||
}
|
||||
setTitle(title);
|
||||
set(DATATABLES_ID, "nodes");
|
||||
set(initID(DATATABLES, "nodes"), nodesTableInit());
|
||||
setTableStyles(html, "nodes", ".healthStatus {width:10em}",
|
||||
".healthReport {width:10em}");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Class<? extends SubView> content() {
|
||||
return NodesBlock.class;
|
||||
}
|
||||
|
||||
private String nodesTableInit() {
|
||||
StringBuilder b = tableInit().append(", aoColumnDefs: [");
|
||||
b.append("{'bSearchable': false, 'aTargets': [ 7 ]}");
|
||||
b.append(", {'sType': 'title-numeric', 'bSearchable': false, "
|
||||
+ "'aTargets': [ 2, 3, 4, 5, 6 ] }");
|
||||
b.append(", {'sType': 'title-numeric', 'aTargets': [ 5 ]}");
|
||||
b.append("]}");
|
||||
return b.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import org.apache.hadoop.yarn.webapp.Controller;
|
||||
|
||||
import com.google.inject.Inject;
|
||||
|
||||
/**
|
||||
* Controller for the Router Web UI.
|
||||
*/
|
||||
public class RouterController extends Controller {
|
||||
|
||||
@Inject
|
||||
RouterController(RequestContext ctx) {
|
||||
super(ctx);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void index() {
|
||||
setTitle("Router");
|
||||
render(AboutPage.class);
|
||||
}
|
||||
|
||||
public void about() {
|
||||
setTitle("About the Cluster");
|
||||
render(AboutPage.class);
|
||||
}
|
||||
|
||||
public void federation() {
|
||||
setTitle("Federation");
|
||||
render(FederationPage.class);
|
||||
}
|
||||
|
||||
public void apps() {
|
||||
setTitle("Applications");
|
||||
render(AppsPage.class);
|
||||
}
|
||||
|
||||
public void nodes() {
|
||||
setTitle("Nodes");
|
||||
render(NodesPage.class);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.router.webapp;
|
||||
|
||||
import org.apache.hadoop.yarn.webapp.SubView;
|
||||
import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
|
||||
|
||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
|
||||
|
||||
/**
|
||||
* View for the Router Web UI.
|
||||
*/
|
||||
public class RouterView extends TwoColumnLayout {
|
||||
|
||||
@Override
|
||||
protected void preHead(Page.HTML<__> html) {
|
||||
commonPreHead(html);
|
||||
|
||||
setTitle("Router");
|
||||
}
|
||||
|
||||
protected void commonPreHead(Page.HTML<__> html) {
|
||||
set(ACCORDION_ID, "nav");
|
||||
set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Class<? extends SubView> nav() {
|
||||
return NavBlock.class;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Class<? extends SubView> content() {
|
||||
return AboutBlock.class;
|
||||
}
|
||||
}
|
||||
|
|
@ -44,5 +44,11 @@ public class RouterWebApp extends WebApp implements YarnWebParams {
|
|||
if (router != null) {
|
||||
bind(Router.class).toInstance(router);
|
||||
}
|
||||
route("/", RouterController.class);
|
||||
route("/cluster", RouterController.class, "about");
|
||||
route("/about", RouterController.class, "about");
|
||||
route("/apps", RouterController.class, "apps");
|
||||
route("/nodes", RouterController.class, "nodes");
|
||||
route("/federation", RouterController.class, "federation");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ URL and redirects the application submission request to the appropriate sub-clus
|
|||
The Router exposes the ApplicationClientProtocol to the outside world, transparently hiding the presence of multiple RMs. To achieve this the Router also persists the mapping
|
||||
between the application and its home sub-cluster into the State Store. This allows Routers to be soft-state while supporting user requests cheaply, as any Router can recover
|
||||
this application to home sub-cluster mapping and direct requests to the right RM without broadcasting them. For performance caching and session stickiness might be advisable.
|
||||
The state of the federation (including applications and nodes) is exposed through the Web UI.
|
||||
|
||||
###AMRMProxy
|
||||
The AMRMProxy is a key component to allow the application to scale and run across sub-clusters. The AMRMProxy runs on all the NM machines and acts as a proxy to the
|
||||
|
|
@ -246,9 +247,9 @@ Optional:
|
|||
|:---- |:---- |
|
||||
|`yarn.router.hostname` | `0.0.0.0` | Router host name.
|
||||
|`yarn.router.clientrm.address` | `0.0.0.0:8050` | Router client address. |
|
||||
|`yarn.router.webapp.address` | `0.0.0.0:80` | Webapp address at the router. |
|
||||
|`yarn.router.webapp.address` | `0.0.0.0:8089` | Webapp address at the router. |
|
||||
|`yarn.router.admin.address` | `0.0.0.0:8052` | Admin address at the router. |
|
||||
|`yarn.router.webapp.https.address` | `0.0.0.0:443` | Secure webapp address at the router. |
|
||||
|`yarn.router.webapp.https.address` | `0.0.0.0:8091` | Secure webapp address at the router. |
|
||||
|`yarn.router.submit.retry` | `3` | The number of retries in the router before we give up. |
|
||||
|`yarn.federation.statestore.max-connections` | `10` | This is the maximum number of parallel connections each Router makes to the state-store. |
|
||||
|`yarn.federation.cache-ttl.secs` | `60` | The Router caches informations, and this is the time to leave before the cache is invalidated. |
|
||||
|
|
@ -306,4 +307,5 @@ The output from this particular example job should be something like:
|
|||
Job Finished in 30.586 seconds
|
||||
Estimated value of Pi is 3.14250000......
|
||||
|
||||
The state of the job can also be tracked on the Router Web UI at `routerhost:8089`.
|
||||
Note that no change in the code or recompilation of the input jar was required to use federation. Also, the output of this job is the exact same as it would be when run without federation. Also, in order to get the full benefit of federation, use a large enough number of mappers such that more than one cluster is required. That number happens to be 16 in the case of the above example.
|
||||
|
|
|
|||
Loading…
Reference in New Issue