From 46eb1033a86ca53e7b94202567aef8af2417bdf8 Mon Sep 17 00:00:00 2001 From: Haibo Chen Date: Tue, 17 Oct 2017 16:03:46 -0700 Subject: [PATCH] YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen) --- .../org/apache/hadoop/yarn/sls/SLSRunner.java | 1 + .../scheduler/CapacitySchedulerMetrics.java | 5 -- .../sls/scheduler/FairSchedulerMetrics.java | 17 +---- .../sls/scheduler/FifoSchedulerMetrics.java | 5 +- .../sls/scheduler/SLSCapacityScheduler.java | 19 ------ .../yarn/sls/scheduler/SLSFairScheduler.java | 19 ------ .../yarn/sls/scheduler/SchedulerMetrics.java | 66 ++++++++----------- 7 files changed, 36 insertions(+), 96 deletions(-) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index 59f9c1766d6..dfdf7c95fc4 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -731,6 +731,7 @@ public class SLSRunner extends Configured implements Tool { } queueAppNumMap.put(queueName, appNum); + wrapper.getSchedulerMetrics().trackQueue(queueName); } private void runNewAM(String jobType, String user, diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java index a73f48c4d7e..89b44c68d61 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java @@ -28,9 +28,4 @@ public class CapacitySchedulerMetrics extends SchedulerMetrics { public CapacitySchedulerMetrics() { super(); } - - @Override - public void trackQueue(String queueName) { - trackedQueues.add(queueName); - } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java index 7b306f0410c..a5aee7477c0 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java @@ -167,8 +167,9 @@ public class FairSchedulerMetrics extends SchedulerMetrics { } @Override - public void trackQueue(String queueName) { - trackedQueues.add(queueName); + protected void registerQueueMetrics(String queueName) { + super.registerQueueMetrics(queueName); + FairScheduler fair = (FairScheduler) scheduler; final FSQueue queue = fair.getQueueManager().getQueue(queueName); registerQueueMetrics(queue, Metric.DEMAND); @@ -209,16 +210,4 @@ public class FairSchedulerMetrics extends SchedulerMetrics { } ); } - - @Override - public void untrackQueue(String queueName) { - trackedQueues.remove(queueName); - - for (Metric metric: Metric.values()) { - metrics.remove("variable.queue." + queueName + "." + - metric.value + ".memory"); - metrics.remove("variable.queue." + queueName + "." + - metric.value + ".vcores"); - } - } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java index 6ab2e1d0107..427b5972699 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java @@ -35,8 +35,9 @@ public class FifoSchedulerMetrics extends SchedulerMetrics { } @Override - public void trackQueue(String queueName) { - trackedQueues.add(queueName); + protected void registerQueueMetrics(String queueName) { + super.registerQueueMetrics(queueName); + FifoScheduler fifo = (FifoScheduler) scheduler; // for FifoScheduler, only DEFAULT_QUEUE // here the three parameters doesn't affect results diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java index 108c2bc8b1e..6848b22b53e 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java @@ -323,25 +323,6 @@ public class SLSCapacityScheduler extends CapacityScheduler implements queueName); } - private void initQueueMetrics(CSQueue queue) { - if (queue instanceof LeafQueue) { - schedulerMetrics.initQueueMetric(queue.getQueueName()); - return; - } - - for (CSQueue child : queue.getChildQueues()) { - initQueueMetrics(child); - } - } - @Override - public void serviceInit(Configuration configuration) throws Exception { - super.serviceInit(configuration); - - if (metricsON) { - initQueueMetrics(getRootQueue()); - } - } - @Override public void serviceStop() throws Exception { try { diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java index 81f66488cdc..8e49c517ec2 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java @@ -306,25 +306,6 @@ public class SLSFairScheduler extends FairScheduler queueName); } - private void initQueueMetrics(FSQueue queue) { - if (queue instanceof FSLeafQueue) { - schedulerMetrics.initQueueMetric(queue.getQueueName()); - return; - } - - for (FSQueue child : queue.getChildQueues()) { - initQueueMetrics(child); - } - } - - @Override - public void serviceInit(Configuration conf) throws Exception { - super.serviceInit(conf); - if (metricsON) { - initQueueMetrics(getQueueManager().getRootQueue()); - } - } - @Override public void serviceStop() throws Exception { try { diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java index cabc94a4817..9c4d000994f 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java @@ -234,14 +234,35 @@ public abstract class SchedulerMetrics { } } - public abstract void trackQueue(String queueName); - - public void untrackQueue(String queueName) { - for (String m : queueTrackedMetrics) { - metrics.remove("variable.queue." + queueName + "." + m); + /** + * Track a queue by registering its metrics. + * + * @param queue queue name + */ + public void trackQueue(String queue) { + queueLock.lock(); + try { + if (!isTracked(queue)) { + trackedQueues.add(queue); + registerQueueMetrics(queue); + } + } finally { + queueLock.unlock(); } } - + + protected void registerQueueMetrics(String queueName) { + SortedMap counterMap = metrics.getCounters(); + + for (QueueMetric queueMetric : QueueMetric.values()) { + String metricName = getQueueMetricName(queueName, queueMetric); + if (!counterMap.containsKey(metricName)) { + metrics.counter(metricName); + queueTrackedMetrics.add(metricName); + } + } + } + public boolean isTracked(String queueName) { return trackedQueues.contains(queueName); } @@ -547,40 +568,13 @@ public abstract class SchedulerMetrics { return "counter.queue." + queue + "." + metric.value; } - private void traceQueueIfNotTraced(String queue) { - queueLock.lock(); - try { - if (!isTracked(queue)) { - trackQueue(queue); - } - } finally { - queueLock.unlock(); - } - } - - void initQueueMetric(String queueName){ - SortedMap counterMap = metrics.getCounters(); - - for (QueueMetric queueMetric : QueueMetric.values()) { - String metricName = getQueueMetricName(queueName, queueMetric); - if (!counterMap.containsKey(metricName)) { - metrics.counter(metricName); - counterMap = metrics.getCounters(); - } - } - - traceQueueIfNotTraced(queueName); - } - void updateQueueMetrics(Resource pendingResource, Resource allocatedResource, String queueName) { + trackQueue(queueName); + SortedMap counterMap = metrics.getCounters(); for(QueueMetric metric : QueueMetric.values()) { String metricName = getQueueMetricName(queueName, metric); - if (!counterMap.containsKey(metricName)) { - metrics.counter(metricName); - counterMap = metrics.getCounters(); - } if (metric == QueueMetric.PENDING_MEMORY) { counterMap.get(metricName).inc(pendingResource.getMemorySize()); @@ -592,8 +586,6 @@ public abstract class SchedulerMetrics { counterMap.get(metricName).inc(allocatedResource.getVirtualCores()); } } - - traceQueueIfNotTraced(queueName); } void updateQueueMetricsByRelease(Resource releaseResource, String queue) {