From 25946da4133a3784f1ec280cccac36b24db273a6 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Tue, 11 Jun 2013 22:15:02 +0000 Subject: [PATCH] YARN-767. Initialize application metrics at RM bootup. Contributed by Jian He. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1491989 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../server/resourcemanager/ResourceManager.java | 7 ++++--- .../resourcemanager/scheduler/QueueMetrics.java | 13 +++++++++++++ .../scheduler/TestQueueMetrics.java | 14 ++++++++++++++ .../scheduler/fair/TestFairScheduler.java | 2 ++ 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6bdbddc5333..984d0f750e1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -488,6 +488,9 @@ Release 2.1.0-beta - UNRELEASED YARN-760. NodeManager throws AvroRuntimeException on failed start. (Niranjan Singh via jlowe) + YARN-767. Initialize application metrics at RM bootup. (Jian He via + acmurthy) + BREAKDOWN OF HADOOP-8562 SUBTASKS AND RELATED JIRAS YARN-158. Yarn creating package-info.java must not depend on sh. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 47f83455b41..0efb174feca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -223,7 +223,10 @@ public synchronized void init(Configuration conf) { this.resourceTracker = createResourceTrackerService(); addService(resourceTracker); - + + DefaultMetricsSystem.initialize("ResourceManager"); + JvmMetrics.initSingleton("ResourceManager", null); + try { this.scheduler.reinitialize(conf, this.rmContext); } catch (IOException ioe) { @@ -598,8 +601,6 @@ public void start() { } startWepApp(); - DefaultMetricsSystem.initialize("ResourceManager"); - JvmMetrics.initSingleton("ResourceManager", null); try { rmDTSecretManager.startThreads(); } catch(IOException ie) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 5447845ebc3..58d9d4d9766 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -126,6 +126,18 @@ static QueueMetrics forQueue(String queueName, Queue parent, enableUserMetrics, conf); } + // this method is here because we want to make sure these metrics show up on + // queue registration. + private void initMetrics() { + appsSubmitted.incr(0); + appsRunning.incr(0); + appsPending.incr(0); + appsCompleted.incr(0); + appsKilled.incr(0); + appsFailed.incr(0); + reservedContainers.incr(0); + } + /** * Helper method to clear cache - used only for unit tests. */ @@ -156,6 +168,7 @@ static QueueMetrics forQueue(MetricsSystem ms, String queueName, ms.register( sourceName(queueName).toString(), "Metrics for queue: " + queueName, metrics); + metrics.initMetrics(); } queueMetrics.put(queueName, metrics); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index f91f9e6223b..11fc00ea99d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -32,10 +32,14 @@ import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; +import org.apache.hadoop.test.MetricsAsserts; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Assert; import org.junit.Before; @@ -264,6 +268,16 @@ public void testMetricsCache() { } } + @Test + public void testMetricsInitializedOnRMInit() { + YarnConfiguration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, + FifoScheduler.class, ResourceScheduler.class); + MockRM rm = new MockRM(conf); + QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics(); + checkApps(metrics, 0, 0, 0, 0, 0, 0); + MetricsAsserts.assertGauge("ReservedContainers", 0, metrics); + } public static void checkApps(MetricsSource source, int submitted, int pending, int running, int completed, int failed, int killed) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index bfda5e51f71..40c6be0e448 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -39,6 +39,7 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.Clock; import org.apache.hadoop.yarn.MockApps; @@ -133,6 +134,7 @@ public void tearDown() { scheduler = null; resourceManager = null; QueueMetrics.clearQueueMetrics(); + DefaultMetricsSystem.shutdown(); } private Configuration createConfiguration() {