Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-3747. Initialize queue metrics upfront and added start/finish time to RM Web-UI. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1240886 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2012-02-06 03:38:44 +00:00
parent f88574acde
commit c7007a8a35
13 changed files with 260 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -705,6 +705,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3765. FifoScheduler does not respect yarn.scheduler.fifo.minimum-
allocation-mb setting (Hitesh Shah via mahadev)
MAPREDUCE-3747. Initialize queue metrics upfront and added start/finish
time to RM Web-UI. (acmurthy)
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES

12
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
View File

@ -98,6 +98,12 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
*/
public float getUsedCapacity();
/**
* Set used capacity of the queue.
* @param usedCapacity used capacity of the queue
*/
public void setUsedCapacity(float usedCapacity);
/**
* Get the currently utilized resources in the cluster
* by the queue and children (if any).
@ -114,6 +120,12 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
*/
public float getUtilization();
/**
* Get the current <em>utilization</em> of the queue.
* @param utilization queue utilization
*/
public void setUtilization(float utilization);
/**
* Get the current run-state of the queue
* @return current run-state

38
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
View File

@ -17,7 +17,9 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.yarn.Lock;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
class CSQueueUtils {
@ -65,4 +67,40 @@ class CSQueueUtils {
1);
}
@Lock(CSQueue.class)
public static void updateQueueStatistics(
final CSQueue childQueue, final CSQueue parentQueue,
final Resource clusterResource, final Resource minimumAllocation) {
final int clusterMemory = clusterResource.getMemory();
final int usedMemory = childQueue.getUsedResources().getMemory();
float queueLimit = 0.0f;
float utilization = 0.0f;
float usedCapacity = 0.0f;
if (clusterMemory > 0) {
queueLimit = clusterMemory * childQueue.getAbsoluteCapacity();
final float parentAbsoluteCapacity =
(parentQueue == null) ? 1.0f : parentQueue.getAbsoluteCapacity();
utilization = (usedMemory / queueLimit);
usedCapacity = (usedMemory / (clusterMemory * parentAbsoluteCapacity));
}
childQueue.setUtilization(utilization);
childQueue.setUsedCapacity(usedCapacity);
int available =
Math.max((roundUp(minimumAllocation, (int)queueLimit) - usedMemory), 0);
childQueue.getMetrics().setAvailableResourcesToQueue(
Resources.createResource(available));
}
public static int roundUp(Resource minimumAllocation, int memory) {
int minMemory = minimumAllocation.getMemory();
return LeafQueue.divideAndCeil(memory, minMemory) * minMemory;
}
public static int roundDown(Resource minimumAllocation, int memory) {
int minMemory = minimumAllocation.getMemory();
return (memory / minMemory) * minMemory;
}
}

66
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -180,7 +180,9 @@ public class LeafQueue implements CSQueue {
Map<QueueACL, AccessControlList> acls =
cs.getConfiguration().getAcls(getQueuePath());
setupQueueConfigs(capacity, absoluteCapacity,
setupQueueConfigs(
cs.getClusterResources(),
capacity, absoluteCapacity,
maximumCapacity, absoluteMaxCapacity,
userLimit, userLimitFactor,
maxApplications, maxApplicationsPerUser,
@ -198,6 +200,7 @@ public class LeafQueue implements CSQueue {
}
private synchronized void setupQueueConfigs(
Resource clusterResource,
float capacity, float absoluteCapacity,
float maximumCapacity, float absoluteMaxCapacity,
int userLimit, float userLimitFactor,
@ -235,6 +238,10 @@ public class LeafQueue implements CSQueue {
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
}
// Update metrics
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
LOG.info("Initializing " + queueName + "\n" +
"capacity = " + capacity +
@ -386,11 +393,11 @@ public class LeafQueue implements CSQueue {
return null;
}
synchronized void setUtilization(float utilization) {
public synchronized void setUtilization(float utilization) {
this.utilization = utilization;
}
synchronized void setUsedCapacity(float usedCapacity) {
public synchronized void setUsedCapacity(float usedCapacity) {
this.usedCapacity = usedCapacity;
}
@ -534,7 +541,9 @@ public class LeafQueue implements CSQueue {
}
LeafQueue leafQueue = (LeafQueue)queue;
setupQueueConfigs(leafQueue.capacity, leafQueue.absoluteCapacity,
setupQueueConfigs(
clusterResource,
leafQueue.capacity, leafQueue.absoluteCapacity,
leafQueue.maximumCapacity, leafQueue.absoluteMaxCapacity,
leafQueue.userLimit, leafQueue.userLimitFactor,
leafQueue.maxApplications,
@ -542,8 +551,6 @@ public class LeafQueue implements CSQueue {
leafQueue.getMaximumActiveApplications(),
leafQueue.getMaximumActiveApplicationsPerUser(),
leafQueue.state, leafQueue.acls);
updateResource(clusterResource);
}
@Override
@ -883,7 +890,8 @@ public class LeafQueue implements CSQueue {
Resource queueMaxCap = // Queue Max-Capacity
Resources.createResource(
roundDown((int)(absoluteMaxCapacity * clusterResource.getMemory()))
CSQueueUtils.roundDown(minimumAllocation,
(int)(absoluteMaxCapacity * clusterResource.getMemory()))
);
Resource userConsumed = getUser(user).getConsumedResources();
@ -904,16 +912,6 @@ public class LeafQueue implements CSQueue {
return userLimit;
}
private int roundUp(int memory) {
int minMemory = minimumAllocation.getMemory();
return divideAndCeil(memory, minMemory) * minMemory;
}
private int roundDown(int memory) {
int minMemory = minimumAllocation.getMemory();
return (memory / minMemory) * minMemory;
}
@Lock(NoLock.class)
private Resource computeUserLimit(SchedulerApp application,
Resource clusterResource, Resource required) {
@ -927,8 +925,11 @@ public class LeafQueue implements CSQueue {
// Allow progress for queues with miniscule capacity
final int queueCapacity =
Math.max(
roundUp((int)(absoluteCapacity * clusterResource.getMemory())),
required.getMemory());
CSQueueUtils.roundUp(
minimumAllocation,
(int)(absoluteCapacity * clusterResource.getMemory())),
required.getMemory()
);
final int consumed = usedResources.getMemory();
final int currentCapacity =
@ -943,7 +944,8 @@ public class LeafQueue implements CSQueue {
final int activeUsers = activeUsersManager.getNumActiveUsers();
int limit =
roundUp(
CSQueueUtils.roundUp(
minimumAllocation,
Math.min(
Math.max(divideAndCeil(currentCapacity, activeUsers),
divideAndCeil((int)userLimit*currentCapacity, 100)),
@ -991,7 +993,7 @@ public class LeafQueue implements CSQueue {
return true;
}
private static int divideAndCeil(int a, int b) {
static int divideAndCeil(int a, int b) {
if (b == 0) {
LOG.info("divideAndCeil called with a=" + a + " b=" + b);
return 0;
@ -1325,7 +1327,8 @@ public class LeafQueue implements CSQueue {
SchedulerApp application, Resource resource) {
// Update queue metrics
Resources.addTo(usedResources, resource);
updateResource(clusterResource);
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
++numContainers;
// Update user metrics
@ -1349,7 +1352,8 @@ public class LeafQueue implements CSQueue {
SchedulerApp application, Resource resource) {
// Update queue metrics
Resources.subtractFrom(usedResources, resource);
updateResource(clusterResource);
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
--numContainers;
// Update user metrics
@ -1374,6 +1378,10 @@ public class LeafQueue implements CSQueue {
CSQueueUtils.computeMaxActiveApplicationsPerUser(
maxActiveApplications, userLimit, userLimitFactor);
// Update metrics
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
// Update application properties
for (SchedulerApp application : activeApplications) {
synchronized (application) {
@ -1383,18 +1391,6 @@ public class LeafQueue implements CSQueue {
}
}
private synchronized void updateResource(Resource clusterResource) {
float queueLimit = clusterResource.getMemory() * absoluteCapacity;
setUtilization(usedResources.getMemory() / queueLimit);
setUsedCapacity(usedResources.getMemory()
/ (clusterResource.getMemory() * parent.getAbsoluteCapacity()));
Resource resourceLimit =
Resources.createResource(roundUp((int)queueLimit));
metrics.setAvailableResourcesToQueue(
Resources.subtractFrom(resourceLimit, usedResources));
}
@Override
public QueueMetrics getMetrics() {
return metrics;

51
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
View File

@ -97,7 +97,8 @@ public class ParentQueue implements CSQueue {
RecordFactoryProvider.getRecordFactory(null);
public ParentQueue(CapacitySchedulerContext cs,
String queueName, Comparator<CSQueue> comparator, CSQueue parent, CSQueue old) {
String queueName, Comparator<CSQueue> comparator,
CSQueue parent, CSQueue old) {
minimumAllocation = cs.getMinimumResourceCapability();
this.parent = parent;
@ -137,7 +138,8 @@ public class ParentQueue implements CSQueue {
this.queueInfo.setQueueName(queueName);
this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
setupQueueConfigs(capacity, absoluteCapacity,
setupQueueConfigs(cs.getClusterResources(),
capacity, absoluteCapacity,
maximumCapacity, absoluteMaxCapacity, state, acls);
this.queueComparator = comparator;
@ -149,9 +151,10 @@ public class ParentQueue implements CSQueue {
}
private synchronized void setupQueueConfigs(
float capacity, float absoluteCapacity,
float maximumCapacity, float absoluteMaxCapacity,
QueueState state, Map<QueueACL, AccessControlList> acls
Resource clusterResource,
float capacity, float absoluteCapacity,
float maximumCapacity, float absoluteMaxCapacity,
QueueState state, Map<QueueACL, AccessControlList> acls
) {
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
@ -174,6 +177,10 @@ public class ParentQueue implements CSQueue {
aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
}
// Update metrics
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
LOG.info(queueName +
", capacity=" + capacity +
", asboluteCapacity=" + absoluteCapacity +
@ -384,12 +391,10 @@ public class ParentQueue implements CSQueue {
childQueues.addAll(currentChildQueues.values());
// Set new configs
setupQueueConfigs(parentQueue.capacity, parentQueue.absoluteCapacity,
setupQueueConfigs(clusterResource,
parentQueue.capacity, parentQueue.absoluteCapacity,
parentQueue.maximumCapacity, parentQueue.absoluteMaxCapacity,
parentQueue.state, parentQueue.acls);
// Update
updateResource(clusterResource);
}
Map<String, CSQueue> getQueues(Set<CSQueue> queues) {
@ -485,11 +490,11 @@ public class ParentQueue implements CSQueue {
" #applications: " + getNumApplications());
}
synchronized void setUsedCapacity(float usedCapacity) {
public synchronized void setUsedCapacity(float usedCapacity) {
this.usedCapacity = usedCapacity;
}
synchronized void setUtilization(float utilization) {
public synchronized void setUtilization(float utilization) {
this.utilization = utilization;
}
@ -674,14 +679,16 @@ public class ParentQueue implements CSQueue {
synchronized void allocateResource(Resource clusterResource,
Resource resource) {
Resources.addTo(usedResources, resource);
updateResource(clusterResource);
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
++numContainers;
}
synchronized void releaseResource(Resource clusterResource,
Resource resource) {
Resources.subtractFrom(usedResources, resource);
updateResource(clusterResource);
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
--numContainers;
}
@ -691,22 +698,12 @@ public class ParentQueue implements CSQueue {
for (CSQueue childQueue : childQueues) {
childQueue.updateClusterResource(clusterResource);
}
// Update metrics
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
}
private synchronized void updateResource(Resource clusterResource) {
float queueLimit = clusterResource.getMemory() * absoluteCapacity;
float parentAbsoluteCapacity =
(rootQueue) ? 1.0f : parent.getAbsoluteCapacity();
setUtilization(usedResources.getMemory() / queueLimit);
setUsedCapacity(usedResources.getMemory()
/ (clusterResource.getMemory() * parentAbsoluteCapacity));
Resource resourceLimit =
Resources.createResource((int)queueLimit);
metrics.setAvailableResourcesToQueue(
Resources.subtractFrom(resourceLimit, usedResources));
}
@Override
public QueueMetrics getMetrics() {
return metrics;

16
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java
View File

@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE;
@ -27,6 +26,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
@ -52,11 +52,12 @@ class AppsBlock extends HtmlBlock {
th(".user", "User").
th(".name", "Name").
th(".queue", "Queue").
th(".starttime", "StartTime").
th(".finishtime", "FinishTime").
th(".state", "State").
th(".finalstatus", "FinalStatus").
th(".progress", "Progress").
th(".ui", "Tracking UI").
th(".note", "Note")._()._().
th(".ui", "Tracking UI")._()._().
tbody();
int i = 0;
String reqState = $(APP_STATE);
@ -67,6 +68,8 @@ class AppsBlock extends HtmlBlock {
}
AppInfo appInfo = new AppInfo(app, true);
String percent = String.format("%.1f", appInfo.getProgress());
String startTime = Times.format(appInfo.getStartTime());
String finishTime = Times.format(appInfo.getFinishTime());
tbody.
tr().
td().
@ -75,6 +78,10 @@ class AppsBlock extends HtmlBlock {
td(appInfo.getUser()).
td(appInfo.getName()).
td(appInfo.getQueue()).
td().
br().$title(startTime)._()._(startTime)._().
td().
br().$title(startTime)._()._(finishTime)._().
td(appInfo.getState()).
td(appInfo.getFinalStatus()).
td().
@ -85,8 +92,7 @@ class AppsBlock extends HtmlBlock {
$style(join("width:", percent, '%'))._()._()._().
td().
a(!appInfo.isTrackingUrlReady()?
"#" : appInfo.getTrackingUrlPretty(), appInfo.getTrackingUI())._().
td(appInfo.getNote())._();
"#" : appInfo.getTrackingUrlPretty(), appInfo.getTrackingUI())._()._();
if (list.rendering != Render.HTML && ++i >= 20) break;
}
tbody._()._();

49
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
View File

@ -55,15 +55,19 @@ public class MetricsOverviewTable extends HtmlBlock {
//CSS in the correct spot
html.style(".metrics {margin-bottom:5px}");
ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(this.rm, this.rmContext);
ClusterMetricsInfo clusterMetrics =
new ClusterMetricsInfo(this.rm, this.rmContext);
DIV<Hamlet> div = html.div().$class("metrics");
div.table("#metricsoverview").
div.h3("Cluster Metrics").
table("#metricsoverview").
thead().$class("ui-widget-header").
tr().
th().$class("ui-state-default")._("Apps Submitted")._().
th().$class("ui-state-default")._("Apps Pending")._().
th().$class("ui-state-default")._("Apps Running")._().
th().$class("ui-state-default")._("Apps Completed")._().
th().$class("ui-state-default")._("Containers Running")._().
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Total")._().
@ -78,6 +82,14 @@ public class MetricsOverviewTable extends HtmlBlock {
tbody().$class("ui-widget-content").
tr().
td(String.valueOf(clusterMetrics.getAppsSubmitted())).
td(String.valueOf(clusterMetrics.getAppsPending())).
td(String.valueOf(clusterMetrics.getAppsRunning())).
td(
String.valueOf(
clusterMetrics.getAppsCompleted() +
clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled()
)
).
td(String.valueOf(clusterMetrics.getContainersAllocated())).
td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)).
@ -89,26 +101,38 @@ public class MetricsOverviewTable extends HtmlBlock {
td().a(url("nodes/rebooted"),String.valueOf(clusterMetrics.getRebootedNodes()))._().
_().
_()._();
String user = request().getRemoteUser();
if (user != null) {
UserMetricsInfo userMetrics = new UserMetricsInfo(this.rm, this.rmContext, user);
if (userMetrics.metricsAvailable()) {
div.table("#usermetricsoverview").
div.h3("User Metrics for " + user).
table("#usermetricsoverview").
thead().$class("ui-widget-header").
tr().
th().$class("ui-state-default")._("Apps Submitted ("+user+")")._().
th().$class("ui-state-default")._("Containers Running ("+user+")")._().
th().$class("ui-state-default")._("Containers Pending ("+user+")")._().
th().$class("ui-state-default")._("Containers Reserved ("+user+")")._().
th().$class("ui-state-default")._("Memory Used ("+user+")")._().
th().$class("ui-state-default")._("Memory Pending ("+user+")")._().
th().$class("ui-state-default")._("Memory Reserved ("+user+")")._().
th().$class("ui-state-default")._("Apps Submitted")._().
th().$class("ui-state-default")._("Apps Pending")._().
th().$class("ui-state-default")._("Apps Running")._().
th().$class("ui-state-default")._("Apps Completed")._().
th().$class("ui-state-default")._("Containers Running")._().
th().$class("ui-state-default")._("Containers Pending")._().
th().$class("ui-state-default")._("Containers Reserved")._().
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Pending")._().
th().$class("ui-state-default")._("Memory Reserved")._().
_().
_().
tbody().$class("ui-widget-content").
tr().
td(String.valueOf(userMetrics.getAppsSubmitted())).
td(String.valueOf(userMetrics.getAppsPending())).
td(String.valueOf(userMetrics.getAppsRunning())).
td(
String.valueOf(
(userMetrics.getAppsCompleted() +
userMetrics.getAppsFailed() + userMetrics.getAppsKilled())
)
).
td(String.valueOf(userMetrics.getRunningContainers())).
td(String.valueOf(userMetrics.getPendingContainers())).
td(String.valueOf(userMetrics.getReservedContainers())).
@ -117,6 +141,7 @@ public class MetricsOverviewTable extends HtmlBlock {
td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)).
_().
_()._();
}
}

7
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java
View File

@ -63,10 +63,11 @@ public class RmView extends TwoColumnLayout {
private String appsTableInit() {
AppsList list = getInstance(AppsList.class);
// id, user, name, queue, state, progress, ui, note
// id, user, name, queue, starttime, finishtime, state, progress, ui
StringBuilder init = tableInit().
append(", aoColumns:[{sType:'title-numeric'}, null, null, null, null,").
append("null,{sType:'title-numeric', bSearchable:false}, null, null]");
append(", aoColumns:[{sType:'title-numeric'}, null, null, null, ").
append("null, null , null, ").
append("null,{sType:'title-numeric', bSearchable:false}, null]");
// Sort by id upon page load
init.append(", aaSorting: [[0, 'asc']]");

48
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
View File

@ -32,10 +32,20 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
public class ClusterMetricsInfo {
protected int appsSubmitted;
protected int appsCompleted;
protected int appsPending;
protected int appsRunning;
protected int appsFailed;
protected int appsKilled;
protected long reservedMB;
protected long availableMB;
protected long allocatedMB;
protected int containersAllocated;
protected int containersReserved;
protected int containersPending;
protected long totalMB;
protected int totalNodes;
protected int lostNodes;
@ -53,10 +63,20 @@ public class ClusterMetricsInfo {
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
this.appsSubmitted = metrics.getAppsSubmitted();
this.appsCompleted = metrics.getAppsCompleted();
this.appsPending = metrics.getAppsPending();
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
this.reservedMB = metrics.getReservedMB();
this.availableMB = metrics.getAvailableMB();
this.allocatedMB = metrics.getAllocatedMB();
this.containersAllocated = metrics.getAllocatedContainers();
this.containersPending = metrics.getPendingContainers();
this.containersReserved = metrics.getReservedContainers();
this.totalMB = availableMB + reservedMB + allocatedMB;
this.activeNodes = clusterMetrics.getNumActiveNMs();
this.lostNodes = clusterMetrics.getNumLostNMs();
@ -71,6 +91,26 @@ public class ClusterMetricsInfo {
return this.appsSubmitted;
}
public int getAppsCompleted() {
return appsCompleted;
}
public int getAppsPending() {
return appsPending;
}
public int getAppsRunning() {
return appsRunning;
}
public int getAppsFailed() {
return appsFailed;
}
public int getAppsKilled() {
return appsKilled;
}
public long getReservedMB() {
return this.reservedMB;
}
@ -87,6 +127,14 @@ public class ClusterMetricsInfo {
return this.containersAllocated;
}
public int getReservedContainers() {
return this.containersReserved;
}
public int getPendingContainers() {
return this.containersPending;
}
public long getTotalMB() {
return this.totalMB;
}

33
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java
View File

@ -32,6 +32,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
public class UserMetricsInfo {
protected int appsSubmitted;
protected int appsCompleted;
protected int appsPending;
protected int appsRunning;
protected int appsFailed;
protected int appsKilled;
protected int runningContainers;
protected int pendingContainers;
protected int reservedContainers;
@ -54,10 +59,18 @@ public class UserMetricsInfo {
if (userMetrics != null) {
this.userMetricsAvailable = true;
this.appsSubmitted = userMetrics.getAppsSubmitted();
this.appsCompleted = metrics.getAppsCompleted();
this.appsPending = metrics.getAppsPending();
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
this.runningContainers = userMetrics.getAllocatedContainers();
this.pendingContainers = userMetrics.getPendingContainers();
this.reservedContainers = userMetrics.getReservedContainers();
this.reservedMB = userMetrics.getReservedMB();
this.pendingMB = userMetrics.getPendingMB();
this.allocatedMB = userMetrics.getAllocatedMB();
@ -72,6 +85,26 @@ public class UserMetricsInfo {
return this.appsSubmitted;
}
public int getAppsCompleted() {
return appsCompleted;
}
public int getAppsPending() {
return appsPending;
}
public int getAppsRunning() {
return appsRunning;
}
public int getAppsFailed() {
return appsFailed;
}
public int getAppsKilled() {
return appsKilled;
}
public long getReservedMB() {
return this.reservedMB;
}

10
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
View File

@ -142,7 +142,7 @@ public class TestApplicationLimits {
CapacityScheduler.parseQueue(csContext, csConf, null, "root",
queues, queues,
CapacityScheduler.queueComparator,
CapacityScheduler.applicationComparator,
CapacityScheduler.applicationComparator,
TestUtils.spyHook);
LeafQueue queue = (LeafQueue)queues.get(A);
@ -163,6 +163,10 @@ public class TestApplicationLimits {
expectedMaxActiveApps * (queue.getUserLimit() / 100.0f) *
queue.getUserLimitFactor()),
queue.getMaximumActiveApplicationsPerUser());
assertEquals(
(int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB()
);
// Add some nodes to the cluster & test new limits
clusterResource = Resources.createResource(120 * 16 * GB);
@ -178,6 +182,10 @@ public class TestApplicationLimits {
(int)Math.ceil(expectedMaxActiveApps *
(queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()),
queue.getMaximumActiveApplicationsPerUser());
assertEquals(
(int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB()
);
}
@Test

2
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
View File

@ -48,7 +48,7 @@ public class TestNodesPage {
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
// future. In that case this value should be adjusted to the new value.
final int numberOfThInMetricsTable = 10;
final int numberOfThInMetricsTable = 13;
final int numberOfActualTableHeaders = 10;
private Injector injector;

13
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
View File

@ -361,6 +361,7 @@ public class TestRMWebServices extends JerseyTest {
verifyClusterMetrics(
WebServicesTestUtils.getXmlInt(element, "appsSubmitted"),
WebServicesTestUtils.getXmlInt(element, "appsCompleted"),
WebServicesTestUtils.getXmlInt(element, "reservedMB"),
WebServicesTestUtils.getXmlInt(element, "availableMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
@ -379,8 +380,9 @@ public class TestRMWebServices extends JerseyTest {
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
assertEquals("incorrect number of elements", 12, clusterinfo.length());
verifyClusterMetrics(clusterinfo.getInt("appsSubmitted"),
assertEquals("incorrect number of elements", 19, clusterinfo.length());
verifyClusterMetrics(
clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
clusterinfo.getInt("allocatedMB"),
clusterinfo.getInt("containersAllocated"),
@ -390,7 +392,8 @@ public class TestRMWebServices extends JerseyTest {
clusterinfo.getInt("rebootedNodes"),clusterinfo.getInt("activeNodes"));
}
public void verifyClusterMetrics(int sub, int reservedMB, int availableMB,
public void verifyClusterMetrics(int submittedApps, int completedApps,
int reservedMB, int availableMB,
int allocMB, int containersAlloc, int totalMB, int totalNodes,
int lostNodes, int unhealthyNodes, int decommissionedNodes,
int rebootedNodes, int activeNodes) throws JSONException, Exception {
@ -404,7 +407,9 @@ public class TestRMWebServices extends JerseyTest {
+ metrics.getAllocatedMB();
assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), sub);
metrics.getAppsSubmitted(), submittedApps);
assertEquals("appsCompleted doesn't match",
metrics.getAppsCompleted(), completedApps);
assertEquals("reservedMB doesn't match",
metrics.getReservedMB(), reservedMB);
assertEquals("availableMB doesn't match",