Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8720. CapacityScheduler does not enforce max resource allocation check at queue level. Contributed by Tarun Parimi. 

(cherry picked from commit f1a893fdbc)
This commit is contained in:
Weiwei Yang 2018-09-14 16:33:51 +08:00
parent f918e4d894
commit ef17728c58
4 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
View File

@ -213,7 +213,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
} }
} }
Resource maximumCapacity = getScheduler().getMaximumResourceCapability(); Resource maximumCapacity =
getScheduler().getMaximumResourceCapability(app.getQueue());
// sanity check // sanity check
try { try {

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -515,10 +515,11 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
} }
// Normalize all requests // Normalize all requests
String queue = submissionContext.getQueue();
for (ResourceRequest amReq : amReqs) { for (ResourceRequest amReq : amReqs) {
SchedulerUtils.normalizeAndValidateRequest(amReq, SchedulerUtils.normalizeAndValidateRequest(amReq,
scheduler.getMaximumResourceCapability(), scheduler.getMaximumResourceCapability(queue),
submissionContext.getQueue(), scheduler, isRecovery, rmContext); queue, scheduler, isRecovery, rmContext);
amReq.setCapability( amReq.setCapability(
scheduler.getNormalizedResource(amReq.getCapability())); scheduler.getNormalizedResource(amReq.getCapability()));

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -898,6 +898,9 @@ public class TestAppManager{
when(scheduler.getMaximumResourceCapability()).thenReturn( when(scheduler.getMaximumResourceCapability()).thenReturn(
Resources.createResource( Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
when(scheduler.getMaximumResourceCapability(any(String.class))).thenReturn(
Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
ResourceCalculator rs = mock(ResourceCalculator.class); ResourceCalculator rs = mock(ResourceCalculator.class);
when(scheduler.getResourceCalculator()).thenReturn(rs); when(scheduler.getResourceCalculator()).thenReturn(rs);

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -1473,6 +1473,9 @@ public class TestClientRMService {
when(yarnScheduler.getMaximumResourceCapability()).thenReturn( when(yarnScheduler.getMaximumResourceCapability()).thenReturn(
Resources.createResource( Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
when(yarnScheduler.getMaximumResourceCapability(any(String.class)))
.thenReturn(Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn( when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn(
Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102))); Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102)));
when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn( when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(