diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 20a5a1ff790..4b705029f95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; @@ -76,6 +77,10 @@ public class QueueMetrics implements MetricsSource { @Metric("# of active applications") MutableGaugeInt activeApplications; @Metric("App Attempt First Container Allocation Delay") MutableRate appAttemptFirstContainerAllocationDelay; + @Metric("Aggregate total of preempted memory MB") + MutableCounterLong aggregateMemoryMBPreempted; + @Metric("Aggregate total of preempted vcores") + MutableCounterLong aggregateVcoresPreempted; //Metrics updated only for "default" partition @Metric("Allocated memory in MB") MutableGaugeLong allocatedMB; @@ -552,6 +557,14 @@ public class QueueMetrics implements MetricsSource { } } + public void updatePreemptedResources(Resource res) { + aggregateMemoryMBPreempted.incr(res.getMemorySize()); + aggregateVcoresPreempted.incr(res.getVirtualCores()); + if (parent != null) { + parent.updatePreemptedResources(res); + } + } + public void reserveResource(String partition, String user, Resource res) { if(partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) { reserveResource(user, res); @@ -732,4 +745,14 @@ public class QueueMetrics implements MetricsSource { public long getAggregatePreemptedContainers() { return aggregateContainersPreempted.value(); } + + @VisibleForTesting + public long getAggregateMemoryMBPreempted() { + return aggregateMemoryMBPreempted.value(); + } + + @VisibleForTesting + public long getAggregateVcoresPreempted() { + return aggregateVcoresPreempted.value(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 9b3a4530693..1173d40c34f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2115,6 +2115,7 @@ public class CapacityScheduler extends / DateUtils.MILLIS_PER_SECOND; qMetrics.updatePreemptedMemoryMBSeconds(mbSeconds); qMetrics.updatePreemptedVcoreSeconds(vcSeconds); + qMetrics.updatePreemptedResources(containerResource); } @Lock(Lock.NoLock.class) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java index 800789af72a..ad3f23d11d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java @@ -185,6 +185,11 @@ public class TestCapacitySchedulerSurgicalPreemption Assert.assertEquals("Number of preempted containers incorrectly recorded:", 4, cs.getQueue("root").getMetrics().getAggregatePreemptedContainers()); + Assert.assertEquals("Amount of preempted memory incorrectly recorded:", + 4 * GB, cs.getQueue("root").getMetrics().getAggregateMemoryMBPreempted()); + Assert.assertEquals("Number of preempted vcores incorrectly recorded:", + 4, cs.getQueue("root").getMetrics().getAggregateVcoresPreempted()); + rm1.close(); }