Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6428. Queue AM limit is not honored in CS always. Contributed by Bibin A Chundatt.

This commit is contained in:
Naganarasimha 2017-07-09 17:26:09 +05:30
parent 9d27880442
commit 626d730bfc
2 changed files with 47 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
View File

@ -218,27 +218,19 @@ public class DominantResourceCalculator extends ResourceCalculator {
public Resource multiplyAndNormalizeUp(Resource r, double by,
Resource stepFactor) {
return Resources.createResource(
roundUp(
(int)Math.ceil(r.getMemorySize() * by), stepFactor.getMemorySize()),
roundUp(
(int)Math.ceil(r.getVirtualCores() * by),
stepFactor.getVirtualCores())
);
roundUp((long) Math.ceil((float) (r.getMemorySize() * by)),
stepFactor.getMemorySize()),
roundUp((int) Math.ceil((float) (r.getVirtualCores() * by)),
stepFactor.getVirtualCores()));
}
@Override
public Resource multiplyAndNormalizeDown(Resource r, double by,
Resource stepFactor) {
return Resources.createResource(
roundDown(
(int)(r.getMemorySize() * by),
stepFactor.getMemorySize()
),
roundDown(
(int)(r.getVirtualCores() * by),
stepFactor.getVirtualCores()
)
);
roundDown((long) (r.getMemorySize() * by), stepFactor.getMemorySize()),
roundDown((int) (r.getVirtualCores() * by),
stepFactor.getVirtualCores()));
}
@Override

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
View File

@ -4417,4 +4417,44 @@ public class TestCapacityScheduler {
Assert.assertEquals(b1.getState(), QueueState.RUNNING);
Assert.assertTrue(!b1.getChildQueues().isEmpty());
}
@Test(timeout = 30000)
public void testAMLimitDouble() throws Exception {
CapacitySchedulerConfiguration config =
new CapacitySchedulerConfiguration();
config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
DominantResourceCalculator.class.getName());
CapacitySchedulerConfiguration conf =
new CapacitySchedulerConfiguration(config);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
conf.setInt("yarn.scheduler.minimum-allocation-mb", 512);
conf.setInt("yarn.scheduler.minimum-allocation-vcores", 1);
MockRM rm = new MockRM(conf);
rm.start();
rm.registerNode("127.0.0.1:1234", 10 * GB);
rm.registerNode("127.0.0.1:1235", 10 * GB);
rm.registerNode("127.0.0.1:1236", 10 * GB);
rm.registerNode("127.0.0.1:1237", 10 * GB);
ResourceScheduler scheduler = rm.getRMContext().getScheduler();
waitforNMRegistered(scheduler, 4, 5);
LeafQueue queueA =
(LeafQueue) ((CapacityScheduler) scheduler).getQueue("default");
Resource amResourceLimit = queueA.getAMResourceLimit();
Assert.assertEquals(4096, amResourceLimit.getMemorySize());
Assert.assertEquals(4, amResourceLimit.getVirtualCores());
rm.stop();
}
private void waitforNMRegistered(ResourceScheduler scheduler, int nodecount,
int timesec) throws InterruptedException {
long start = System.currentTimeMillis();
while (System.currentTimeMillis() - start < timesec * 1000) {
if (scheduler.getNumClusterNodes() < nodecount) {
Thread.sleep(100);
} else {
break;
}
}
}
}