Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8212. Pending backlog for async allocation threads should be configurable. Contributed by Tao Yang. 

(cherry picked from commit 2d319e3793)
This commit is contained in:
Weiwei Yang 2018-05-01 09:47:10 +08:00
parent 0fc55a8f57
commit f34f4e9c11
2 changed files with 16 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
View File

@ -260,6 +260,7 @@ public class CapacityScheduler extends
CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX
+ ".scheduling-interval-ms"; + ".scheduling-interval-ms";
private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5; private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5;
private long asyncMaxPendingBacklogs;
public CapacityScheduler() { public CapacityScheduler() {
super(CapacityScheduler.class.getName()); super(CapacityScheduler.class.getName());
@ -377,6 +378,11 @@ public class CapacityScheduler extends
asyncSchedulerThreads.add(new AsyncScheduleThread(this)); asyncSchedulerThreads.add(new AsyncScheduleThread(this));
} }
resourceCommitterService = new ResourceCommitterService(this); resourceCommitterService = new ResourceCommitterService(this);
asyncMaxPendingBacklogs = this.conf.getInt(
CapacitySchedulerConfiguration.
SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS,
CapacitySchedulerConfiguration.
DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS);
} }
// Setup how many containers we can allocate for each round // Setup how many containers we can allocate for each round
@ -571,7 +577,8 @@ public class CapacityScheduler extends
Thread.sleep(100); Thread.sleep(100);
} else { } else {
// Don't run schedule if we have some pending backlogs already // Don't run schedule if we have some pending backlogs already
if (cs.getAsyncSchedulingPendingBacklogs() > 100) { if (cs.getAsyncSchedulingPendingBacklogs()
> cs.asyncMaxPendingBacklogs) {
Thread.sleep(1); Thread.sleep(1);
} else{ } else{
schedule(cs); schedule(cs);

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -244,6 +244,14 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD = public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD =
SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-threads"; SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-threads";
@Private
public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS =
SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-pending-backlogs";
@Private
public static final Integer
DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS = 100;
@Private @Private
public static final boolean DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE = false; public static final boolean DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE = false;