Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8212. Pending backlog for async allocation threads should be configurable. Contributed by Tao Yang.

Browse Source 
(Cherry picked from commit 2d319e37937c1e20c6a7dc4477ef88defd1f8464)
This commit is contained in:
Weiwei Yang 2018-05-01 09:47:10 +08:00
parent 0e31e014f3
commit f12c78120e
2 changed files with 16 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
View File

@ -245,6 +245,7 @@ public Configuration getConf() {
CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX
+ ".scheduling-interval-ms"; + ".scheduling-interval-ms";
private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5; private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5;
private long asyncMaxPendingBacklogs;
public CapacityScheduler() { public CapacityScheduler() {
super(CapacityScheduler.class.getName()); super(CapacityScheduler.class.getName());
@ -354,6 +355,11 @@ void initScheduler(Configuration configuration) throws
asyncSchedulerThreads.add(new AsyncScheduleThread(this)); asyncSchedulerThreads.add(new AsyncScheduleThread(this));
} }
resourceCommitterService = new ResourceCommitterService(this); resourceCommitterService = new ResourceCommitterService(this);
asyncMaxPendingBacklogs = this.conf.getInt(
CapacitySchedulerConfiguration.
SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS,
CapacitySchedulerConfiguration.
DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS);
} }
// Setup how many containers we can allocate for each round // Setup how many containers we can allocate for each round
@ -541,7 +547,8 @@ public void run() {
Thread.sleep(100); Thread.sleep(100);
} else { } else {
// Don't run schedule if we have some pending backlogs already // Don't run schedule if we have some pending backlogs already
if (cs.getAsyncSchedulingPendingBacklogs() > 100) { if (cs.getAsyncSchedulingPendingBacklogs()
> cs.asyncMaxPendingBacklogs) {
Thread.sleep(1); Thread.sleep(1);
} else{ } else{
schedule(cs); schedule(cs);

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -239,6 +239,14 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD = public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD =
SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-threads"; SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-threads";
@Private
public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS =
SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-pending-backlogs";
@Private
public static final Integer
DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS = 100;
@Private @Private
public static final boolean DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE = false; public static final boolean DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE = false;