From 5bd18c49bd5075fa20d24363dceea7828e3fa266 Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Fri, 2 Dec 2016 13:35:09 -0800 Subject: [PATCH] YARN-5929. Missing scheduling policy in the FS queue metric. (Contributed by Yufei Gu via Daniel Templeton) --- .../scheduler/fair/FSQueueMetrics.java | 32 +++++++-- .../scheduler/fair/TestFSQueueMetrics.java | 69 +++++++++++++++++++ 2 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueMetrics.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java index a970815f62a..ca375f2fe2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; @@ -169,6 +170,12 @@ public void setAMResourceUsage(Resource resource) { amResourceUsageVCores.set(resource.getVirtualCores()); } + /** + * Get the scheduling policy. + * + * @return the scheduling policy + */ + @Metric("Scheduling policy") public String getSchedulingPolicy() { return schedulingPolicy; } @@ -181,21 +188,38 @@ public void setSchedulingPolicy(String policy) { static FSQueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { MetricsSystem ms = DefaultMetricsSystem.instance(); + return forQueue(ms, queueName, parent, enableUserMetrics, conf); + } + + /** + * Get the FS queue metric for the given queue. Create one and register it to + * metrics system if there isn't one for the queue. + * + * @param ms the metric system + * @param queueName queue name + * @param parent parent queue + * @param enableUserMetrics if user metrics is needed + * @param conf configuration + * @return a FSQueueMetrics object + */ + @VisibleForTesting + public synchronized + static FSQueueMetrics forQueue(MetricsSystem ms, String queueName, + Queue parent, boolean enableUserMetrics, Configuration conf) { QueueMetrics metrics = queueMetrics.get(queueName); if (metrics == null) { metrics = new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf) .tag(QUEUE_INFO, queueName); - + // Register with the MetricsSystems if (ms != null) { metrics = ms.register( - sourceName(queueName).toString(), - "Metrics for queue: " + queueName, metrics); + sourceName(queueName).toString(), + "Metrics for queue: " + queueName, metrics); } queueMetrics.put(queueName, metrics); } return (FSQueueMetrics)metrics; } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueMetrics.java new file mode 100644 index 00000000000..7ccfbc3a9c8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueMetrics.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl; +import org.apache.hadoop.metrics2.impl.MetricsRecords; +import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestQueueMetrics; +import org.junit.Before; +import org.junit.Test; + +/** + * The test class for {@link FSQueueMetrics}. + */ +public class TestFSQueueMetrics { + private static final Configuration CONF = new Configuration(); + + private MetricsSystem ms; + + @Before public void setUp() { + ms = new MetricsSystemImpl(); + QueueMetrics.clearQueueMetrics(); + } + + /** + * Test if the metric scheduling policy is set correctly. + */ + @Test + public void testSchedulingPolicy() { + String queueName = "single"; + + FSQueueMetrics metrics = FSQueueMetrics.forQueue(ms, queueName, null, false, + CONF); + metrics.setSchedulingPolicy("drf"); + checkSchedulingPolicy(queueName, "drf"); + + // test resetting the scheduling policy + metrics.setSchedulingPolicy("fair"); + checkSchedulingPolicy(queueName, "fair"); + } + + private void checkSchedulingPolicy(String queueName, String policy) { + MetricsSource queueSource = TestQueueMetrics.queueSource(ms, queueName); + MetricsCollectorImpl collector = new MetricsCollectorImpl(); + queueSource.getMetrics(collector, true); + MetricsRecords.assertTag(collector.getRecords().get(0), "SchedulingPolicy", + policy); + } +}