diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index d8b51f76bed..6723fd9e749 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -43,7 +43,6 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.util.resource.Resources;
@Private
@@ -482,8 +481,7 @@ public ActiveUsersManager getActiveUsersManager() {
/**
* Check whether this queue can run this application master under the
- * maxAMShare limit. For FIFO and FAIR policies, check if the VCore usage
- * takes up the entire cluster or maxResources for the queue.
+ * maxAMShare limit.
* @param amResource
* @return true if this queue can run
*/
@@ -493,24 +491,25 @@ public boolean canRunAppAM(Resource amResource) {
if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
return true;
}
- Resource maxAMResource = Resources.multiply(getFairShare(), maxAMShare);
- Resource ifRunAMResource = Resources.add(amResourceUsage, amResource);
- boolean overMaxAMShareLimit = policy
- .checkIfAMResourceUsageOverLimit(ifRunAMResource, maxAMResource);
-
- // For fair policy and fifo policy which doesn't check VCore usages,
- // additionally check if the AM takes all available VCores or
- // over maxResource to avoid deadlock.
- if (!overMaxAMShareLimit && !policy.equals(
- SchedulingPolicy.getInstance(DominantResourceFairnessPolicy.class))) {
- overMaxAMShareLimit =
- isVCoresOverMaxResource(ifRunAMResource.getVirtualCores()) ||
- ifRunAMResource.getVirtualCores() >=
- scheduler.getRootQueueMetrics().getAvailableVirtualCores();
+ // If FairShare is zero, use min(maxShare, available resource) to compute
+ // maxAMResource
+ Resource maxResource = Resources.clone(getFairShare());
+ if (maxResource.getMemorySize() == 0) {
+ maxResource.setMemory(
+ Math.min(scheduler.getRootQueueMetrics().getAvailableMB(),
+ getMaxShare().getMemorySize()));
}
- return !overMaxAMShareLimit;
+ if (maxResource.getVirtualCoresSize() == 0) {
+ maxResource.setVirtualCores(Math.min(
+ scheduler.getRootQueueMetrics().getAvailableVirtualCores(),
+ getMaxShare().getVirtualCoresSize()));
+ }
+
+ Resource maxAMResource = Resources.multiply(maxResource, maxAMShare);
+ Resource ifRunAMResource = Resources.add(amResourceUsage, amResource);
+ return Resources.fitsIn(ifRunAMResource, maxAMResource);
}
public void addAMResourceUsage(Resource amResource) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index f50c35898ce..25554dd1f2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -310,25 +310,6 @@ protected boolean assignContainerPreCheck(FSSchedulerNode node) {
return true;
}
- /**
- * Helper method to check if requested VCores are over maxResource.
- * @param requestedVCores the number of VCores requested
- * @return true if the number of VCores requested is over the maxResource;
- * false otherwise
- */
- protected boolean isVCoresOverMaxResource(int requestedVCores) {
- if (requestedVCores >= scheduler.getAllocationConfiguration().
- getMaxResources(getName()).getVirtualCores()) {
- return true;
- }
-
- if (getParent() == null) {
- return false;
- }
-
- return getParent().isVCoresOverMaxResource(requestedVCores);
- }
-
/**
* Returns true if queue has at least one app running.
*/
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
index 160ba4b5d82..9eda46cd8c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
@@ -177,17 +177,6 @@ public abstract void computeSteadyShares(
public abstract boolean checkIfUsageOverFairShare(
Resource usage, Resource fairShare);
- /**
- * Check if a leaf queue's AM resource usage over its limit under this policy
- *
- * @param usage {@link Resource} the resource used by application masters
- * @param maxAMResource {@link Resource} the maximum allowed resource for
- * application masters
- * @return true if AM resource usage is over the limit
- */
- public abstract boolean checkIfAMResourceUsageOverLimit(
- Resource usage, Resource maxAMResource);
-
/**
* Get headroom by calculating the min of clusterAvailable
and
* (queueFairShare
- queueUsage
) resources that are
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
index 623437a3b1c..ad41b11f803 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
@@ -93,11 +93,6 @@ public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
return !Resources.fitsIn(usage, fairShare);
}
- @Override
- public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMResource) {
- return !Resources.fitsIn(usage, maxAMResource);
- }
-
@Override
public Resource getHeadroom(Resource queueFairShare, Resource queueUsage,
Resource maxAvailable) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index 42d0420325f..6aa8405ff43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -150,11 +150,6 @@ public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
return Resources.greaterThan(RESOURCE_CALCULATOR, null, usage, fairShare);
}
- @Override
- public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMResource) {
- return usage.getMemorySize() > maxAMResource.getMemorySize();
- }
-
@Override
public byte getApplicableDepth() {
return SchedulingPolicy.DEPTH_ANY;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
index c277df1d024..d3fdcf663d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
@@ -113,11 +113,6 @@ public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
"as FifoPolicy only works for FSLeafQueue.");
}
- @Override
- public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMResource) {
- return usage.getMemorySize() > maxAMResource.getMemorySize();
- }
-
@Override
public Resource getHeadroom(Resource queueFairShare,
Resource queueUsage, Resource maxAvailable) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index beb05411a75..61d93220aca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -1081,6 +1081,84 @@ public void testContainerReservationAttemptExceedingQueueMax()
getCurrentReservation().getMemorySize());
}
+ /**
+ * The test verifies that zero-FairShare queues (because of zero/tiny
+ * weight) can get resources for the AM.
+ */
+ @Test
+ public void testRequestAMResourceInZeroFairShareQueue() throws Exception {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("");
+ out.println("");
+ out.println("");
+ out.println("0.0");
+ out.println("4096mb,10vcores");
+ out.println("0.5");
+ out.println("");
+ out.println("");
+ out.println("2.0");
+ out.println("");
+ out.println("");
+ out.println("0.000001");
+ out.println("");
+ out.println("");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMNode node =
+ MockNodes.newNodeInfo(1, Resources.createResource(8192, 20),
+ 0, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+ scheduler.update();
+
+ //create request for non-zero weight queue
+ createSchedulingRequest(1024, "root.queue2", "user2", 1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // A managed AM which need 3G memory will not get resource,
+ // since it request more than the maxAMShare (4G * 0.5 = 2G).
+ Resource amResource = Resource.newInstance(1024, 1);
+ int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority();
+ ApplicationAttemptId attId1 = createAppAttemptId(1, 1);
+ createApplicationWithAMResource(attId1, "root.queue1", "user1", amResource);
+ createSchedulingRequestExistingApplication(3 * 1024, 1, amPriority, attId1);
+ FSAppAttempt app1 = scheduler.getSchedulerApp(attId1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application 1 should not be running",
+ 0, app1.getLiveContainers().size());
+
+ // A managed AM which need 2G memory will get resource,
+ // since it request no more than the maxAMShare (4G * 0.5 = 2G).
+ ApplicationAttemptId attId2 = createAppAttemptId(2, 1);
+ createApplicationWithAMResource(attId2, "root.queue1", "user1", amResource);
+ createSchedulingRequestExistingApplication(2 * 1024, 1, amPriority, attId2);
+ FSAppAttempt app2 = scheduler.getSchedulerApp(attId2);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application 2 should be running",
+ 1, app2.getLiveContainers().size());
+
+ // A managed AM which need 1G memory will get resource, even thought its
+ // fair share is 0 because its weight is tiny(0.000001).
+ ApplicationAttemptId attId3 = createAppAttemptId(3, 1);
+ createApplicationWithAMResource(attId3, "root.queue3", "user1", amResource);
+ createSchedulingRequestExistingApplication(1024, 1, amPriority, attId3);
+ FSAppAttempt app3 = scheduler.getSchedulerApp(attId3);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application 3 should be running",
+ 1, app3.getLiveContainers().size());
+ }
+
@Test (timeout = 500000)
public void testContainerReservationNotExceedingQueueMax() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);