diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index b2f17c4e07e..d22ffd64f2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -242,6 +242,13 @@ public class Resources { out.setVirtualCores((int)(lhs.getVirtualCores() * by)); return out; } + + public static Resource multiplyAndRoundUp(Resource lhs, double by) { + Resource out = clone(lhs); + out.setMemorySize((long)Math.ceil(lhs.getMemorySize() * by)); + out.setVirtualCores((int)Math.ceil(lhs.getVirtualCores() * by)); + return out; + } public static Resource normalize( ResourceCalculator calculator, Resource lhs, Resource min, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index 057214badae..f8570a8a2f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.util.resource; import org.apache.hadoop.yarn.api.records.Resource; import org.junit.Test; + +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class TestResources { @@ -46,5 +48,25 @@ public class TestResources { assertTrue(Resources.none().compareTo( createResource(0, 1)) < 0); } - + + @Test + public void testMultipleRoundUp() { + final double by = 0.5; + final String memoryErrorMsg = "Invalid memory size."; + final String vcoreErrorMsg = "Invalid virtual core number."; + Resource resource = Resources.createResource(1, 1); + Resource result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 1); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); + + resource = Resources.createResource(2, 2); + result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 1); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); + + resource = Resources.createResource(0, 0); + result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 0); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 0d61bd69e73..48847e59a8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -517,7 +517,8 @@ public class FSLeafQueue extends FSQueue { getMaxShare().getVirtualCores())); } - return Resources.multiply(maxResource, maxAMShare); + // Round up to allow AM to run when there is only one vcore on the cluster + return Resources.multiplyAndRoundUp(maxResource, maxAMShare); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 0efe263cab0..4b134da098e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -660,15 +660,13 @@ public class TestFairScheduler extends FairSchedulerTestBase { // case, we use maxShare, since it is smaller than available resource. assertEquals("QueueFSZeroWithMax's fair share should be zero", 0, queueFSZeroWithMax.getFairShare().getMemorySize()); + Resource expectedAMResource = Resources.multiplyAndRoundUp( + queueFSZeroWithMax.getMaxShare(), queueFSZeroWithMax.getMaxAMShare()); assertEquals("QueueFSZeroWithMax's maximum AM resource should be " - + "maxShare * maxAMShare", - (long)(queueFSZeroWithMax.getMaxShare().getMemorySize() * - queueFSZeroWithMax.getMaxAMShare()), + + "maxShare * maxAMShare", expectedAMResource.getMemorySize(), queueFSZeroWithMax.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSZeroWithMax's maximum AM resource should be " - + "maxShare * maxAMShare", - (long)(queueFSZeroWithMax.getMaxShare().getVirtualCores() * - queueFSZeroWithMax.getMaxAMShare()), + + "maxShare * maxAMShare", expectedAMResource.getVirtualCores(), queueFSZeroWithMax.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to " + "AM resource request", @@ -690,17 +688,19 @@ public class TestFairScheduler extends FairSchedulerTestBase { // the min(maxShare, available resource) to compute maxAMShare, in this // case, we use available resource since it is smaller than the // default maxShare. + expectedAMResource = Resources.multiplyAndRoundUp( + Resources.createResource(memCapacity - amResource.getMemorySize(), + cpuCapacity - amResource.getVirtualCores()), + queueFSZeroWithAVL.getMaxAMShare()); assertEquals("QueueFSZeroWithAVL's fair share should be zero", 0, queueFSZeroWithAVL.getFairShare().getMemorySize()); assertEquals("QueueFSZeroWithAVL's maximum AM resource should be " + " available resource * maxAMShare", - (long) ((memCapacity - amResource.getMemorySize()) * - queueFSZeroWithAVL.getMaxAMShare()), + expectedAMResource.getMemorySize(), queueFSZeroWithAVL.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSZeroWithAVL's maximum AM resource should be " + " available resource * maxAMShare", - (long) ((cpuCapacity - amResource.getVirtualCores()) * - queueFSZeroWithAVL.getMaxAMShare()), + expectedAMResource.getVirtualCores(), queueFSZeroWithAVL.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to " + "AM resource request", @@ -722,13 +722,13 @@ public class TestFairScheduler extends FairSchedulerTestBase { // fair share to compute maxAMShare assertNotEquals("QueueFSNonZero's fair share shouldn't be zero", 0, queueFSNonZero.getFairShare().getMemorySize()); + expectedAMResource = Resources.multiplyAndRoundUp( + queueFSNonZero.getFairShare(), queueFSNonZero.getMaxAMShare()); assertEquals("QueueFSNonZero's maximum AM resource should be " - + " fair share * maxAMShare", - (long)(memCapacity * queueFSNonZero.getMaxAMShare()), + + " fair share * maxAMShare", expectedAMResource.getMemorySize(), queueFSNonZero.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSNonZero's maximum AM resource should be " - + " fair share * maxAMShare", - (long)(cpuCapacity * queueFSNonZero.getMaxAMShare()), + + " fair share * maxAMShare", expectedAMResource.getVirtualCores(), queueFSNonZero.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSNonZero's AM resource usage should be the same to " + "AM resource request",