From f12c78120edf8679407a43857f5caa310bc556ca Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Tue, 1 May 2018 09:47:10 +0800 Subject: [PATCH] YARN-8212. Pending backlog for async allocation threads should be configurable. Contributed by Tao Yang. (Cherry picked from commit 2d319e37937c1e20c6a7dc4477ef88defd1f8464) --- .../scheduler/capacity/CapacityScheduler.java | 9 ++++++++- .../capacity/CapacitySchedulerConfiguration.java | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 5a29e5d115c..3587f3095e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -245,6 +245,7 @@ public class CapacityScheduler extends CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".scheduling-interval-ms"; private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5; + private long asyncMaxPendingBacklogs; public CapacityScheduler() { super(CapacityScheduler.class.getName()); @@ -354,6 +355,11 @@ public class CapacityScheduler extends asyncSchedulerThreads.add(new AsyncScheduleThread(this)); } resourceCommitterService = new ResourceCommitterService(this); + asyncMaxPendingBacklogs = this.conf.getInt( + CapacitySchedulerConfiguration. + SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS, + CapacitySchedulerConfiguration. + DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS); } // Setup how many containers we can allocate for each round @@ -541,7 +547,8 @@ public class CapacityScheduler extends Thread.sleep(100); } else { // Don't run schedule if we have some pending backlogs already - if (cs.getAsyncSchedulingPendingBacklogs() > 100) { + if (cs.getAsyncSchedulingPendingBacklogs() + > cs.asyncMaxPendingBacklogs) { Thread.sleep(1); } else{ schedule(cs); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 1e22e0b7ed4..c05b948b242 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -239,6 +239,14 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD = SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-threads"; + @Private + public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS = + SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-pending-backlogs"; + + @Private + public static final Integer + DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS = 100; + @Private public static final boolean DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE = false;