MAPREDUCE-3764. Fixed resource usage metrics for queues and users.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1238255 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2012-01-31 08:30:31 +00:00
parent 7ed0f74997
commit 7f23d72352
7 changed files with 97 additions and 98 deletions

View File

@ -621,6 +621,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3748. Changed a log in CapacityScheduler.nodeUpdate to debug. MAPREDUCE-3748. Changed a log in CapacityScheduler.nodeUpdate to debug.
(ramya via acmurthy) (ramya via acmurthy)
MAPREDUCE-3764. Fixed resource usage metrics for queues and users.
(acmurthy)
Release 0.23.0 - 2011-11-01 Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

View File

@ -51,20 +51,19 @@ public class QueueMetrics {
@Metric("# of apps killed") MutableCounterInt appsKilled; @Metric("# of apps killed") MutableCounterInt appsKilled;
@Metric("# of apps failed") MutableCounterInt appsFailed; @Metric("# of apps failed") MutableCounterInt appsFailed;
@Metric("Allocated memory in GiB") MutableGaugeInt allocatedGB; @Metric("Allocated memory in MB") MutableGaugeInt allocatedMB;
@Metric("# of allocated containers") MutableGaugeInt allocatedContainers; @Metric("# of allocated containers") MutableGaugeInt allocatedContainers;
@Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated; @Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated;
@Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased; @Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased;
@Metric("Available memory in GiB") MutableGaugeInt availableGB; @Metric("Available memory in MB") MutableGaugeInt availableMB;
@Metric("Pending memory allocation in GiB") MutableGaugeInt pendingGB; @Metric("Pending memory allocation in MB") MutableGaugeInt pendingMB;
@Metric("# of pending containers") MutableGaugeInt pendingContainers; @Metric("# of pending containers") MutableGaugeInt pendingContainers;
@Metric("# of reserved memory in GiB") MutableGaugeInt reservedGB; @Metric("# of reserved memory in MB") MutableGaugeInt reservedMB;
@Metric("# of reserved containers") MutableGaugeInt reservedContainers; @Metric("# of reserved containers") MutableGaugeInt reservedContainers;
@Metric("# of active users") MutableGaugeInt activeUsers; @Metric("# of active users") MutableGaugeInt activeUsers;
@Metric("# of active users") MutableGaugeInt activeApplications; @Metric("# of active users") MutableGaugeInt activeApplications;
static final Logger LOG = LoggerFactory.getLogger(QueueMetrics.class); static final Logger LOG = LoggerFactory.getLogger(QueueMetrics.class);
static final int GB = 1024; // resource.memory is in MB
static final MetricsInfo RECORD_INFO = info("QueueMetrics", static final MetricsInfo RECORD_INFO = info("QueueMetrics",
"Metrics for the resource scheduler"); "Metrics for the resource scheduler");
static final MetricsInfo QUEUE_INFO = info("Queue", "Metrics by queue"); static final MetricsInfo QUEUE_INFO = info("Queue", "Metrics by queue");
@ -183,7 +182,7 @@ public class QueueMetrics {
* @param limit resource limit * @param limit resource limit
*/ */
public void setAvailableResourcesToQueue(Resource limit) { public void setAvailableResourcesToQueue(Resource limit) {
availableGB.set(limit.getMemory()/GB); availableMB.set(limit.getMemory());
} }
/** /**
@ -219,7 +218,7 @@ public class QueueMetrics {
private void _incrPendingResources(int containers, Resource res) { private void _incrPendingResources(int containers, Resource res) {
pendingContainers.incr(containers); pendingContainers.incr(containers);
pendingGB.incr(res.getMemory()/GB); pendingMB.incr(res.getMemory());
} }
public void decrPendingResources(String user, int containers, Resource res) { public void decrPendingResources(String user, int containers, Resource res) {
@ -235,13 +234,13 @@ public class QueueMetrics {
private void _decrPendingResources(int containers, Resource res) { private void _decrPendingResources(int containers, Resource res) {
pendingContainers.decr(containers); pendingContainers.decr(containers);
pendingGB.decr(res.getMemory()/GB); pendingMB.decr(res.getMemory());
} }
public void allocateResources(String user, int containers, Resource res) { public void allocateResources(String user, int containers, Resource res) {
allocatedContainers.incr(containers); allocatedContainers.incr(containers);
aggregateContainersAllocated.incr(containers); aggregateContainersAllocated.incr(containers);
allocatedGB.incr(res.getMemory()/GB * containers); allocatedMB.incr(res.getMemory() * containers);
_decrPendingResources(containers, multiply(res, containers)); _decrPendingResources(containers, multiply(res, containers));
QueueMetrics userMetrics = getUserMetrics(user); QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) { if (userMetrics != null) {
@ -255,7 +254,7 @@ public class QueueMetrics {
public void releaseResources(String user, int containers, Resource res) { public void releaseResources(String user, int containers, Resource res) {
allocatedContainers.decr(containers); allocatedContainers.decr(containers);
aggregateContainersReleased.incr(containers); aggregateContainersReleased.incr(containers);
allocatedGB.decr(res.getMemory()/GB * containers); allocatedMB.decr(res.getMemory() * containers);
QueueMetrics userMetrics = getUserMetrics(user); QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) { if (userMetrics != null) {
userMetrics.releaseResources(user, containers, res); userMetrics.releaseResources(user, containers, res);
@ -267,7 +266,7 @@ public class QueueMetrics {
public void reserveResource(String user, Resource res) { public void reserveResource(String user, Resource res) {
reservedContainers.incr(); reservedContainers.incr();
reservedGB.incr(res.getMemory()/GB); reservedMB.incr(res.getMemory());
QueueMetrics userMetrics = getUserMetrics(user); QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) { if (userMetrics != null) {
userMetrics.reserveResource(user, res); userMetrics.reserveResource(user, res);
@ -279,7 +278,7 @@ public class QueueMetrics {
public void unreserveResource(String user, Resource res) { public void unreserveResource(String user, Resource res) {
reservedContainers.decr(); reservedContainers.decr();
reservedGB.decr(res.getMemory()/GB); reservedMB.decr(res.getMemory());
QueueMetrics userMetrics = getUserMetrics(user); QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) { if (userMetrics != null) {
userMetrics.unreserveResource(user, res); userMetrics.unreserveResource(user, res);
@ -343,28 +342,28 @@ public class QueueMetrics {
return appsFailed.value(); return appsFailed.value();
} }
public int getAllocatedGB() { public int getAllocatedMB() {
return allocatedGB.value(); return allocatedMB.value();
} }
public int getAllocatedContainers() { public int getAllocatedContainers() {
return allocatedContainers.value(); return allocatedContainers.value();
} }
public int getAvailableGB() { public int getAvailableMB() {
return availableGB.value(); return availableMB.value();
} }
public int getPendingGB() { public int getPendingMB() {
return pendingGB.value(); return pendingMB.value();
} }
public int getPendingContainers() { public int getPendingContainers() {
return pendingContainers.value(); return pendingContainers.value();
} }
public int getReservedGB() { public int getReservedMB() {
return reservedGB.value(); return reservedMB.value();
} }
public int getReservedContainers() { public int getReservedContainers() {

View File

@ -31,8 +31,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
@XmlAccessorType(XmlAccessType.FIELD) @XmlAccessorType(XmlAccessType.FIELD)
public class ClusterMetricsInfo { public class ClusterMetricsInfo {
private static final long MB_IN_GB = 1024;
protected int appsSubmitted; protected int appsSubmitted;
protected long reservedMB; protected long reservedMB;
protected long availableMB; protected long availableMB;
@ -55,9 +53,9 @@ public class ClusterMetricsInfo {
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
this.appsSubmitted = metrics.getAppsSubmitted(); this.appsSubmitted = metrics.getAppsSubmitted();
this.reservedMB = metrics.getReservedGB() * MB_IN_GB; this.reservedMB = metrics.getReservedMB();
this.availableMB = metrics.getAvailableGB() * MB_IN_GB; this.availableMB = metrics.getAvailableMB();
this.allocatedMB = metrics.getAllocatedGB() * MB_IN_GB; this.allocatedMB = metrics.getAllocatedMB();
this.containersAllocated = metrics.getAllocatedContainers(); this.containersAllocated = metrics.getAllocatedContainers();
this.totalMB = availableMB + reservedMB + allocatedMB; this.totalMB = availableMB + reservedMB + allocatedMB;
this.activeNodes = clusterMetrics.getNumActiveNMs(); this.activeNodes = clusterMetrics.getNumActiveNMs();

View File

@ -31,8 +31,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
@XmlAccessorType(XmlAccessType.FIELD) @XmlAccessorType(XmlAccessType.FIELD)
public class UserMetricsInfo { public class UserMetricsInfo {
private static final long MB_IN_GB = 1024;
protected int appsSubmitted; protected int appsSubmitted;
protected int runningContainers; protected int runningContainers;
protected int pendingContainers; protected int pendingContainers;
@ -60,9 +58,9 @@ public class UserMetricsInfo {
this.runningContainers = userMetrics.getAllocatedContainers(); this.runningContainers = userMetrics.getAllocatedContainers();
this.pendingContainers = userMetrics.getPendingContainers(); this.pendingContainers = userMetrics.getPendingContainers();
this.reservedContainers = userMetrics.getReservedContainers(); this.reservedContainers = userMetrics.getReservedContainers();
this.reservedMB = userMetrics.getReservedGB() * MB_IN_GB; this.reservedMB = userMetrics.getReservedMB();
this.pendingMB = userMetrics.getPendingGB() * MB_IN_GB; this.pendingMB = userMetrics.getPendingMB();
this.allocatedMB = userMetrics.getAllocatedGB() * MB_IN_GB; this.allocatedMB = userMetrics.getAllocatedMB();
} }
} }

View File

@ -57,16 +57,16 @@ public class TestQueueMetrics {
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB)); metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
// Available resources is set externally, as it depends on dynamic // Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources // configurable cluster/queue resources
checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0); checkResources(queueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
metrics.incrAppsRunning(user); metrics.incrAppsRunning(user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0); checkApps(queueSource, 1, 0, 1, 0, 0, 0);
metrics.allocateResources(user, 3, Resources.createResource(2*GB)); metrics.allocateResources(user, 3, Resources.createResource(2*GB));
checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 0, 0); checkResources(queueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 0, 0);
metrics.releaseResources(user, 1, Resources.createResource(2*GB)); metrics.releaseResources(user, 1, Resources.createResource(2*GB));
checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0); checkResources(queueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED); metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0); checkApps(queueSource, 1, 0, 0, 1, 0, 0);
@ -92,20 +92,20 @@ public class TestQueueMetrics {
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB)); metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
// Available resources is set externally, as it depends on dynamic // Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources // configurable cluster/queue resources
checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0); checkResources(queueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
checkResources(userSource, 0, 0, 0, 0, 10, 15, 5, 0, 0); checkResources(userSource, 0, 0, 0, 0, 10*GB, 15*GB, 5, 0, 0);
metrics.incrAppsRunning(user); metrics.incrAppsRunning(user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0); checkApps(queueSource, 1, 0, 1, 0, 0, 0);
checkApps(userSource, 1, 0, 1, 0, 0, 0); checkApps(userSource, 1, 0, 1, 0, 0, 0);
metrics.allocateResources(user, 3, Resources.createResource(2*GB)); metrics.allocateResources(user, 3, Resources.createResource(2*GB));
checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 0, 0); checkResources(queueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 0, 0);
checkResources(userSource, 6, 3, 3, 0, 10, 9, 2, 0, 0); checkResources(userSource, 6*GB, 3, 3, 0, 10*GB, 9*GB, 2, 0, 0);
metrics.releaseResources(user, 1, Resources.createResource(2*GB)); metrics.releaseResources(user, 1, Resources.createResource(2*GB));
checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0); checkResources(queueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
checkResources(userSource, 4, 2, 3, 1, 10, 9, 2, 0, 0); checkResources(userSource, 4*GB, 2, 3, 1, 10*GB, 9*GB, 2, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED); metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0); checkApps(queueSource, 1, 0, 0, 1, 0, 0);
@ -141,10 +141,10 @@ public class TestQueueMetrics {
parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB)); parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB));
metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB)); metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB));
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB)); metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0); checkResources(queueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
checkResources(parentQueueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0); checkResources(parentQueueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
checkResources(userSource, 0, 0, 0, 0, 10, 15, 5, 0, 0); checkResources(userSource, 0, 0, 0, 0, 10*GB, 15*GB, 5, 0, 0);
checkResources(parentUserSource, 0, 0, 0, 0, 10, 15, 5, 0, 0); checkResources(parentUserSource, 0, 0, 0, 0, 10*GB, 15*GB, 5, 0, 0);
metrics.incrAppsRunning(user); metrics.incrAppsRunning(user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0); checkApps(queueSource, 1, 0, 1, 0, 0, 0);
@ -154,17 +154,17 @@ public class TestQueueMetrics {
metrics.reserveResource(user, Resources.createResource(3*GB)); metrics.reserveResource(user, Resources.createResource(3*GB));
// Available resources is set externally, as it depends on dynamic // Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources // configurable cluster/queue resources
checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 3, 1); checkResources(queueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 3*GB, 1);
checkResources(parentQueueSource, 6, 3, 3, 0, 100, 9, 2, 3, 1); checkResources(parentQueueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 3*GB, 1);
checkResources(userSource, 6, 3, 3, 0, 10, 9, 2, 3, 1); checkResources(userSource, 6*GB, 3, 3, 0, 10*GB, 9*GB, 2, 3*GB, 1);
checkResources(parentUserSource, 6, 3, 3, 0, 10, 9, 2, 3, 1); checkResources(parentUserSource, 6*GB, 3, 3, 0, 10*GB, 9*GB, 2, 3*GB, 1);
metrics.releaseResources(user, 1, Resources.createResource(2*GB)); metrics.releaseResources(user, 1, Resources.createResource(2*GB));
metrics.unreserveResource(user, Resources.createResource(3*GB)); metrics.unreserveResource(user, Resources.createResource(3*GB));
checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0); checkResources(queueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
checkResources(parentQueueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0); checkResources(parentQueueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
checkResources(userSource, 4, 2, 3, 1, 10, 9, 2, 0, 0); checkResources(userSource, 4*GB, 2, 3, 1, 10*GB, 9*GB, 2, 0, 0);
checkResources(parentUserSource, 4, 2, 3, 1, 10, 9, 2, 0, 0); checkResources(parentUserSource, 4*GB, 2, 3, 1, 10*GB, 9*GB, 2, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED); metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0); checkApps(queueSource, 1, 0, 0, 1, 0, 0);
@ -184,18 +184,19 @@ public class TestQueueMetrics {
assertCounter("AppsKilled", killed, rb); assertCounter("AppsKilled", killed, rb);
} }
public static void checkResources(MetricsSource source, int allocGB, public static void checkResources(MetricsSource source, int allocatedMB,
int allocCtnrs, long aggreAllocCtnrs, long aggreReleasedCtnrs, int availGB, int pendingGB, int pendingCtnrs, int allocCtnrs, long aggreAllocCtnrs, long aggreReleasedCtnrs,
int reservedGB, int reservedCtnrs) { int availableMB, int pendingMB, int pendingCtnrs,
int reservedMB, int reservedCtnrs) {
MetricsRecordBuilder rb = getMetrics(source); MetricsRecordBuilder rb = getMetrics(source);
assertGauge("AllocatedGB", allocGB, rb); assertGauge("AllocatedMB", allocatedMB, rb);
assertGauge("AllocatedContainers", allocCtnrs, rb); assertGauge("AllocatedContainers", allocCtnrs, rb);
assertCounter("AggregateContainersAllocated", aggreAllocCtnrs, rb); assertCounter("AggregateContainersAllocated", aggreAllocCtnrs, rb);
assertCounter("AggregateContainersReleased", aggreReleasedCtnrs, rb); assertCounter("AggregateContainersReleased", aggreReleasedCtnrs, rb);
assertGauge("AvailableGB", availGB, rb); assertGauge("AvailableMB", availableMB, rb);
assertGauge("PendingGB", pendingGB, rb); assertGauge("PendingMB", pendingMB, rb);
assertGauge("PendingContainers", pendingCtnrs, rb); assertGauge("PendingContainers", pendingCtnrs, rb);
assertGauge("ReservedGB", reservedGB, rb); assertGauge("ReservedMB", reservedMB, rb);
assertGauge("ReservedContainers", reservedCtnrs, rb); assertGauge("ReservedContainers", reservedCtnrs, rb);
} }

View File

@ -251,7 +251,7 @@ public class TestLeafQueue {
// Only 1 container // Only 1 container
a.assignContainers(clusterResource, node_0); a.assignContainers(clusterResource, node_0);
assertEquals(7, a.getMetrics().getAvailableGB()); assertEquals(7*GB, a.getMetrics().getAvailableMB());
} }
@ -307,9 +307,9 @@ public class TestLeafQueue {
assertEquals(1*GB, a.getUsedResources().getMemory()); assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0, a.getMetrics().getReservedGB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(1, a.getMetrics().getAllocatedGB()); assertEquals(1*GB, a.getMetrics().getAllocatedMB());
assertEquals(0, a.getMetrics().getAvailableGB()); assertEquals(0*GB, a.getMetrics().getAvailableMB());
// Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also // Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also
// you can get one container more than user-limit // you can get one container more than user-limit
@ -317,16 +317,16 @@ public class TestLeafQueue {
assertEquals(2*GB, a.getUsedResources().getMemory()); assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0, a.getMetrics().getReservedGB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(2, a.getMetrics().getAllocatedGB()); assertEquals(2*GB, a.getMetrics().getAllocatedMB());
// Can't allocate 3rd due to user-limit // Can't allocate 3rd due to user-limit
a.assignContainers(clusterResource, node_0); a.assignContainers(clusterResource, node_0);
assertEquals(2*GB, a.getUsedResources().getMemory()); assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0, a.getMetrics().getReservedGB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(2, a.getMetrics().getAllocatedGB()); assertEquals(2*GB, a.getMetrics().getAllocatedMB());
// Bump up user-limit-factor, now allocate should work // Bump up user-limit-factor, now allocate should work
a.setUserLimitFactor(10); a.setUserLimitFactor(10);
@ -334,16 +334,16 @@ public class TestLeafQueue {
assertEquals(3*GB, a.getUsedResources().getMemory()); assertEquals(3*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0, a.getMetrics().getReservedGB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(3, a.getMetrics().getAllocatedGB()); assertEquals(3*GB, a.getMetrics().getAllocatedMB());
// One more should work, for app_1, due to user-limit-factor // One more should work, for app_1, due to user-limit-factor
a.assignContainers(clusterResource, node_0); a.assignContainers(clusterResource, node_0);
assertEquals(4*GB, a.getUsedResources().getMemory()); assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0, a.getMetrics().getReservedGB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(4, a.getMetrics().getAllocatedGB()); assertEquals(4*GB, a.getMetrics().getAllocatedMB());
// Test max-capacity // Test max-capacity
// Now - no more allocs since we are at max-cap // Now - no more allocs since we are at max-cap
@ -352,8 +352,8 @@ public class TestLeafQueue {
assertEquals(4*GB, a.getUsedResources().getMemory()); assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0, a.getMetrics().getReservedGB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(4, a.getMetrics().getAllocatedGB()); assertEquals(4*GB, a.getMetrics().getAllocatedMB());
// Release each container from app_0 // Release each container from app_0
for (RMContainer rmContainer : app_0.getLiveContainers()) { for (RMContainer rmContainer : app_0.getLiveContainers()) {
@ -363,8 +363,8 @@ public class TestLeafQueue {
assertEquals(1*GB, a.getUsedResources().getMemory()); assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0, a.getMetrics().getReservedGB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(1, a.getMetrics().getAllocatedGB()); assertEquals(1*GB, a.getMetrics().getAllocatedMB());
// Release each container from app_1 // Release each container from app_1
for (RMContainer rmContainer : app_1.getLiveContainers()) { for (RMContainer rmContainer : app_1.getLiveContainers()) {
@ -374,9 +374,9 @@ public class TestLeafQueue {
assertEquals(0*GB, a.getUsedResources().getMemory()); assertEquals(0*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0, a.getMetrics().getReservedGB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(0, a.getMetrics().getAllocatedGB()); assertEquals(0*GB, a.getMetrics().getAllocatedMB());
assertEquals(1, a.getMetrics().getAvailableGB()); assertEquals(1*GB, a.getMetrics().getAvailableMB());
} }
@Test @Test
@ -700,9 +700,9 @@ public class TestLeafQueue {
assertEquals(1*GB, a.getUsedResources().getMemory()); assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0, a.getMetrics().getReservedGB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(1, a.getMetrics().getAllocatedGB()); assertEquals(1*GB, a.getMetrics().getAllocatedMB());
assertEquals(0, a.getMetrics().getAvailableGB()); assertEquals(0*GB, a.getMetrics().getAvailableMB());
// Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also // Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also
// you can get one container more than user-limit // you can get one container more than user-limit
@ -710,8 +710,8 @@ public class TestLeafQueue {
assertEquals(2*GB, a.getUsedResources().getMemory()); assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0, a.getMetrics().getReservedGB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(2, a.getMetrics().getAllocatedGB()); assertEquals(2*GB, a.getMetrics().getAllocatedMB());
// Now, reservation should kick in for app_1 // Now, reservation should kick in for app_1
a.assignContainers(clusterResource, node_0); a.assignContainers(clusterResource, node_0);
@ -720,8 +720,8 @@ public class TestLeafQueue {
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory()); assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(2*GB, node_0.getUsedResource().getMemory()); assertEquals(2*GB, node_0.getUsedResource().getMemory());
assertEquals(4, a.getMetrics().getReservedGB()); assertEquals(4*GB, a.getMetrics().getReservedMB());
assertEquals(2, a.getMetrics().getAllocatedGB()); assertEquals(2*GB, a.getMetrics().getAllocatedMB());
// Now free 1 container from app_0 i.e. 1G // Now free 1 container from app_0 i.e. 1G
a.completedContainer(clusterResource, app_0, node_0, a.completedContainer(clusterResource, app_0, node_0,
@ -732,8 +732,8 @@ public class TestLeafQueue {
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory()); assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(1*GB, node_0.getUsedResource().getMemory()); assertEquals(1*GB, node_0.getUsedResource().getMemory());
assertEquals(4, a.getMetrics().getReservedGB()); assertEquals(4*GB, a.getMetrics().getReservedMB());
assertEquals(1, a.getMetrics().getAllocatedGB()); assertEquals(1*GB, a.getMetrics().getAllocatedMB());
// Now finish another container from app_0 and fulfill the reservation // Now finish another container from app_0 and fulfill the reservation
a.completedContainer(clusterResource, app_0, node_0, a.completedContainer(clusterResource, app_0, node_0,
@ -744,8 +744,8 @@ public class TestLeafQueue {
assertEquals(4*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(4*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentReservation().getMemory()); assertEquals(0*GB, app_1.getCurrentReservation().getMemory());
assertEquals(4*GB, node_0.getUsedResource().getMemory()); assertEquals(4*GB, node_0.getUsedResource().getMemory());
assertEquals(0, a.getMetrics().getReservedGB()); assertEquals(0*GB, a.getMetrics().getReservedMB());
assertEquals(4, a.getMetrics().getAllocatedGB()); assertEquals(4*GB, a.getMetrics().getAllocatedMB());
} }
@Test @Test

View File

@ -398,19 +398,19 @@ public class TestRMWebServices extends JerseyTest {
ResourceScheduler rs = rm.getResourceScheduler(); ResourceScheduler rs = rm.getResourceScheduler();
QueueMetrics metrics = rs.getRootQueueMetrics(); QueueMetrics metrics = rs.getRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
final long MB_IN_GB = 1024;
long totalMBExpect = (metrics.getReservedGB() * MB_IN_GB) long totalMBExpect =
+ (metrics.getAvailableGB() * MB_IN_GB) metrics.getReservedMB()+ metrics.getAvailableMB()
+ (metrics.getAllocatedGB() * MB_IN_GB); + metrics.getAllocatedMB();
assertEquals("appsSubmitted doesn't match", metrics.getAppsSubmitted(), sub); assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), sub);
assertEquals("reservedMB doesn't match", assertEquals("reservedMB doesn't match",
metrics.getReservedGB() * MB_IN_GB, reservedMB); metrics.getReservedMB(), reservedMB);
assertEquals("availableMB doesn't match", metrics.getAvailableGB() assertEquals("availableMB doesn't match",
* MB_IN_GB, availableMB); metrics.getAvailableMB(), availableMB);
assertEquals("allocatedMB doesn't match", metrics.getAllocatedGB() assertEquals("allocatedMB doesn't match",
* MB_IN_GB, allocMB); metrics.getAllocatedMB(), allocMB);
assertEquals("containersAllocated doesn't match", 0, containersAlloc); assertEquals("containersAllocated doesn't match", 0, containersAlloc);
assertEquals("totalMB doesn't match", totalMBExpect, totalMB); assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
assertEquals( assertEquals(