Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-3059. QueueMetrics do not have metrics for aggregate containers-allocated and aggregate containers-released. (Devaraj K via mahadev) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1183540 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mahadev Konar 2011-10-14 22:38:52 +00:00
parent 5f9eab403c
commit b3284c4607
3 changed files with 52 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/CHANGES.txt
View File

@ -385,6 +385,10 @@ Release 0.23.0 - Unreleased
MAPREDUCE-3133. Running a set of methods in a Single Test Class. MAPREDUCE-3133. Running a set of methods in a Single Test Class.
(Jonathan Eagles via mahadev) (Jonathan Eagles via mahadev)
MAPREDUCE-3059. QueueMetrics do not have metrics for aggregate
containers-allocated and aggregate containers-released.
(Devaraj K via mahadev)
OPTIMIZATIONS OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and MAPREDUCE-2026. Make JobTracker.getJobCounters() and

18
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
View File

@ -18,9 +18,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler; package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import com.google.common.base.Splitter; import static org.apache.hadoop.metrics2.lib.Interns.info;
import java.util.Map; import static org.apache.hadoop.yarn.server.resourcemanager.resource.Resources.multiply;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsInfo;
@ -28,16 +30,16 @@ import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import static org.apache.hadoop.metrics2.lib.Interns.info;
import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableCounterInt; import org.apache.hadoop.metrics2.lib.MutableCounterInt;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableGaugeInt; import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import static org.apache.hadoop.yarn.server.resourcemanager.resource.Resources.*;
import org.slf4j.LoggerFactory;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Splitter;
@InterfaceAudience.Private @InterfaceAudience.Private
@Metrics(context="yarn") @Metrics(context="yarn")
@ -51,6 +53,8 @@ public class QueueMetrics {
@Metric("Allocated memory in GiB") MutableGaugeInt allocatedGB; @Metric("Allocated memory in GiB") MutableGaugeInt allocatedGB;
@Metric("# of allocated containers") MutableGaugeInt allocatedContainers; @Metric("# of allocated containers") MutableGaugeInt allocatedContainers;
@Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated;
@Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased;
@Metric("Available memory in GiB") MutableGaugeInt availableGB; @Metric("Available memory in GiB") MutableGaugeInt availableGB;
@Metric("Pending memory allocation in GiB") MutableGaugeInt pendingGB; @Metric("Pending memory allocation in GiB") MutableGaugeInt pendingGB;
@Metric("# of pending containers") MutableGaugeInt pendingContainers; @Metric("# of pending containers") MutableGaugeInt pendingContainers;
@ -234,6 +238,7 @@ public class QueueMetrics {
public void allocateResources(String user, int containers, Resource res) { public void allocateResources(String user, int containers, Resource res) {
allocatedContainers.incr(containers); allocatedContainers.incr(containers);
aggregateContainersAllocated.incr(containers);
allocatedGB.incr(res.getMemory()/GB * containers); allocatedGB.incr(res.getMemory()/GB * containers);
_decrPendingResources(containers, multiply(res, containers)); _decrPendingResources(containers, multiply(res, containers));
QueueMetrics userMetrics = getUserMetrics(user); QueueMetrics userMetrics = getUserMetrics(user);
@ -247,6 +252,7 @@ public class QueueMetrics {
public void releaseResources(String user, int containers, Resource res) { public void releaseResources(String user, int containers, Resource res) {
allocatedContainers.decr(containers); allocatedContainers.decr(containers);
aggregateContainersReleased.incr(containers);
allocatedGB.decr(res.getMemory()/GB * containers); allocatedGB.decr(res.getMemory()/GB * containers);
QueueMetrics userMetrics = getUserMetrics(user); QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) { if (userMetrics != null) {

69
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
View File

@ -18,22 +18,23 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler; package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.metrics2.MetricsSource; import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
import org.apache.hadoop.metrics2.MetricsSystem; import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.apache.hadoop.test.MockitoMaker.make;
import static org.apache.hadoop.test.MockitoMaker.stub;
import static org.junit.Assert.assertNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.MetricsSource;
import static org.apache.hadoop.test.MetricsAsserts.*; import org.apache.hadoop.metrics2.MetricsSystem;
import static org.apache.hadoop.test.MockitoMaker.*; import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resource; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
public class TestQueueMetrics { public class TestQueueMetrics {
static final int GB = 1024; // MB static final int GB = 1024; // MB
@ -56,16 +57,16 @@ public class TestQueueMetrics {
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB)); metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
// Available resources is set externally, as it depends on dynamic // Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources // configurable cluster/queue resources
checkResources(queueSource, 0, 0, 100, 15, 5, 0, 0); checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
metrics.incrAppsRunning(user); metrics.incrAppsRunning(user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0); checkApps(queueSource, 1, 0, 1, 0, 0, 0);
metrics.allocateResources(user, 3, Resources.createResource(2*GB)); metrics.allocateResources(user, 3, Resources.createResource(2*GB));
checkResources(queueSource, 6, 3, 100, 9, 2, 0, 0); checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 0, 0);
metrics.releaseResources(user, 1, Resources.createResource(2*GB)); metrics.releaseResources(user, 1, Resources.createResource(2*GB));
checkResources(queueSource, 4, 2, 100, 9, 2, 0, 0); checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED); metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0); checkApps(queueSource, 1, 0, 0, 1, 0, 0);
@ -91,20 +92,20 @@ public class TestQueueMetrics {
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB)); metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
// Available resources is set externally, as it depends on dynamic // Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources // configurable cluster/queue resources
checkResources(queueSource, 0, 0, 100, 15, 5, 0, 0); checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
checkResources(userSource, 0, 0, 10, 15, 5, 0, 0); checkResources(userSource, 0, 0, 0, 0, 10, 15, 5, 0, 0);
metrics.incrAppsRunning(user); metrics.incrAppsRunning(user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0); checkApps(queueSource, 1, 0, 1, 0, 0, 0);
checkApps(userSource, 1, 0, 1, 0, 0, 0); checkApps(userSource, 1, 0, 1, 0, 0, 0);
metrics.allocateResources(user, 3, Resources.createResource(2*GB)); metrics.allocateResources(user, 3, Resources.createResource(2*GB));
checkResources(queueSource, 6, 3, 100, 9, 2, 0, 0); checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 0, 0);
checkResources(userSource, 6, 3, 10, 9, 2, 0, 0); checkResources(userSource, 6, 3, 3, 0, 10, 9, 2, 0, 0);
metrics.releaseResources(user, 1, Resources.createResource(2*GB)); metrics.releaseResources(user, 1, Resources.createResource(2*GB));
checkResources(queueSource, 4, 2, 100, 9, 2, 0, 0); checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
checkResources(userSource, 4, 2, 10, 9, 2, 0, 0); checkResources(userSource, 4, 2, 3, 1, 10, 9, 2, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED); metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0); checkApps(queueSource, 1, 0, 0, 1, 0, 0);
@ -140,10 +141,10 @@ public class TestQueueMetrics {
parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB)); parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB));
metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB)); metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB));
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB)); metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
checkResources(queueSource, 0, 0, 100, 15, 5, 0, 0); checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
checkResources(parentQueueSource, 0, 0, 100, 15, 5, 0, 0); checkResources(parentQueueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
checkResources(userSource, 0, 0, 10, 15, 5, 0, 0); checkResources(userSource, 0, 0, 0, 0, 10, 15, 5, 0, 0);
checkResources(parentUserSource, 0, 0, 10, 15, 5, 0, 0); checkResources(parentUserSource, 0, 0, 0, 0, 10, 15, 5, 0, 0);
metrics.incrAppsRunning(user); metrics.incrAppsRunning(user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0); checkApps(queueSource, 1, 0, 1, 0, 0, 0);
@ -153,17 +154,17 @@ public class TestQueueMetrics {
metrics.reserveResource(user, Resources.createResource(3*GB)); metrics.reserveResource(user, Resources.createResource(3*GB));
// Available resources is set externally, as it depends on dynamic // Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources // configurable cluster/queue resources
checkResources(queueSource, 6, 3, 100, 9, 2, 3, 1); checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 3, 1);
checkResources(parentQueueSource, 6, 3, 100, 9, 2, 3, 1); checkResources(parentQueueSource, 6, 3, 3, 0, 100, 9, 2, 3, 1);
checkResources(userSource, 6, 3, 10, 9, 2, 3, 1); checkResources(userSource, 6, 3, 3, 0, 10, 9, 2, 3, 1);
checkResources(parentUserSource, 6, 3, 10, 9, 2, 3, 1); checkResources(parentUserSource, 6, 3, 3, 0, 10, 9, 2, 3, 1);
metrics.releaseResources(user, 1, Resources.createResource(2*GB)); metrics.releaseResources(user, 1, Resources.createResource(2*GB));
metrics.unreserveResource(user, Resources.createResource(3*GB)); metrics.unreserveResource(user, Resources.createResource(3*GB));
checkResources(queueSource, 4, 2, 100, 9, 2, 0, 0); checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
checkResources(parentQueueSource, 4, 2, 100, 9, 2, 0, 0); checkResources(parentQueueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
checkResources(userSource, 4, 2, 10, 9, 2, 0, 0); checkResources(userSource, 4, 2, 3, 1, 10, 9, 2, 0, 0);
checkResources(parentUserSource, 4, 2, 10, 9, 2, 0, 0); checkResources(parentUserSource, 4, 2, 3, 1, 10, 9, 2, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED); metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0); checkApps(queueSource, 1, 0, 0, 1, 0, 0);
@ -184,11 +185,13 @@ public class TestQueueMetrics {
} }
public static void checkResources(MetricsSource source, int allocGB, public static void checkResources(MetricsSource source, int allocGB,
int allocCtnrs, int availGB, int pendingGB, int pendingCtnrs, int allocCtnrs, long aggreAllocCtnrs, long aggreReleasedCtnrs, int availGB, int pendingGB, int pendingCtnrs,
int reservedGB, int reservedCtnrs) { int reservedGB, int reservedCtnrs) {
MetricsRecordBuilder rb = getMetrics(source); MetricsRecordBuilder rb = getMetrics(source);
assertGauge("AllocatedGB", allocGB, rb); assertGauge("AllocatedGB", allocGB, rb);
assertGauge("AllocatedContainers", allocCtnrs, rb); assertGauge("AllocatedContainers", allocCtnrs, rb);
assertCounter("AggregateContainersAllocated", aggreAllocCtnrs, rb);
assertCounter("AggregateContainersReleased", aggreReleasedCtnrs, rb);
assertGauge("AvailableGB", availGB, rb); assertGauge("AvailableGB", availGB, rb);
assertGauge("PendingGB", pendingGB, rb); assertGauge("PendingGB", pendingGB, rb);
assertGauge("PendingContainers", pendingCtnrs, rb); assertGauge("PendingContainers", pendingCtnrs, rb);