diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 12ce05f2791..28e2d54c494 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -1633,4 +1633,13 @@ public abstract class AbstractCSQueue implements CSQueue { writeLock.unlock(); } } + + protected String getCapacityOrWeightString() { + if (queueCapacities.getWeight() != -1) { + return "weight=" + queueCapacities.getWeight() + ", " + + "normalizedWeight=" + queueCapacities.getNormalizedWeight(); + } else { + return "capacity=" + queueCapacities.getCapacity(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 6bf8d0a471a..104a89caee0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -268,9 +268,8 @@ public class LeafQueue extends AbstractCSQueue { usersManager.updateUserWeights(); LOG.info( - "Initializing " + getQueuePath() + "\n" + "capacity = " - + queueCapacities.getCapacity() - + " [= (float) configuredCapacity / 100 ]" + "\n" + "Initializing " + getQueuePath() + "\n" + + getExtendedCapacityOrWeightString() + "\n" + "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity() + " [= parentAbsoluteCapacity * capacity ]" + "\n" + "maxCapacity = " + queueCapacities.getMaximumCapacity() @@ -486,7 +485,7 @@ public class LeafQueue extends AbstractCSQueue { public String toString() { readLock.lock(); try { - return getQueuePath() + ": " + "capacity=" + queueCapacities.getCapacity() + return getQueuePath() + ": " + getCapacityOrWeightString() + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " + "usedResources=" + queueUsage.getUsed() + ", " + "usedCapacity=" + getUsedCapacity() + ", " + "absoluteUsedCapacity=" @@ -499,7 +498,19 @@ public class LeafQueue extends AbstractCSQueue { } finally { readLock.unlock(); } + } + protected String getExtendedCapacityOrWeightString() { + if (queueCapacities.getWeight() != -1) { + return "weight = " + queueCapacities.getWeight() + + " [= (float) configuredCapacity (with w suffix)] " + "\n" + + "normalizedWeight = " + queueCapacities.getNormalizedWeight() + + " [= (float) configuredCapacity / sum(configuredCapacity of " + + "all queues under the parent)]"; + } else { + return "capacity = " + queueCapacities.getCapacity() + + " [= (float) configuredCapacity / 100 ]"; + } } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index f79ffcbb33c..ce5e49040e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -173,7 +173,7 @@ public class ParentQueue extends AbstractCSQueue { ((ParentQueue) parent).getQueueOrderingPolicyConfigName()); queueOrderingPolicy.setQueues(childQueues); - LOG.info(queueName + ", capacity=" + this.queueCapacities.getCapacity() + LOG.info(queueName + ", " + getCapacityOrWeightString() + ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() + ", maxCapacity=" + this.queueCapacities.getMaximumCapacity() + ", absoluteMaxCapacity=" + this.queueCapacities @@ -462,8 +462,8 @@ public class ParentQueue extends AbstractCSQueue { public String toString() { return queueName + ": " + - "numChildQueue= " + childQueues.size() + ", " + - "capacity=" + queueCapacities.getCapacity() + ", " + + "numChildQueue= " + childQueues.size() + ", " + + getCapacityOrWeightString() + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " + "usedResources=" + queueUsage.getUsed() + "usedCapacity=" + getUsedCapacity() + ", " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java index bdf4d8d455e..1742ae45f35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java @@ -45,11 +45,16 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.IOException; import java.util.ArrayList; import java.util.Set; public class TestCapacitySchedulerWeightMode { - private final int GB = 1024; + private static final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + private static final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + private static final String A1 = A + ".a1"; + private static final String B1 = B + ".b1"; + private static final String B2 = B + ".b2"; private YarnConfiguration conf; @@ -91,14 +96,12 @@ public class TestCapacitySchedulerWeightMode { conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100); conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1); conf.setMaximumCapacity(A, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); conf.setLabeledQueueWeight(A, "x", 100); conf.setLabeledQueueWeight(A, "y", 50); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9); conf.setMaximumCapacity(B, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); @@ -106,7 +109,6 @@ public class TestCapacitySchedulerWeightMode { conf.setLabeledQueueWeight(B, "z", 100); // Define 2nd-level queues - final String A1 = A + ".a1"; conf.setQueues(A, new String[] { "a1" }); conf.setLabeledQueueWeight(A1, RMNodeLabelsManager.NO_LABEL, 100); conf.setMaximumCapacity(A1, 100); @@ -116,12 +118,10 @@ public class TestCapacitySchedulerWeightMode { conf.setLabeledQueueWeight(A1, "y", 100); conf.setQueues(B, new String[] { "b1", "b2" }); - final String B1 = B + ".b1"; conf.setLabeledQueueWeight(B1, RMNodeLabelsManager.NO_LABEL, 50); conf.setMaximumCapacity(B1, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); - final String B2 = B + ".b2"; conf.setLabeledQueueWeight(B2, RMNodeLabelsManager.NO_LABEL, 50); conf.setMaximumCapacity(B2, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); @@ -155,14 +155,12 @@ public class TestCapacitySchedulerWeightMode { conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100); conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1); conf.setMaximumCapacity(A, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); conf.setLabeledQueueWeight(A, "x", 100); conf.setLabeledQueueWeight(A, "y", 50); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9); conf.setMaximumCapacity(B, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); @@ -170,7 +168,6 @@ public class TestCapacitySchedulerWeightMode { conf.setLabeledQueueWeight(B, "z", 100); // Define 2nd-level queues - final String A1 = A + ".a1"; conf.setQueues(A, new String[] { "a1" }); conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100); conf.setMaximumCapacity(A1, 100); @@ -180,12 +177,10 @@ public class TestCapacitySchedulerWeightMode { conf.setCapacityByLabel(A1, "y", 100); conf.setQueues(B, new String[] { "b1", "b2" }); - final String B1 = B + ".b1"; conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50); conf.setMaximumCapacity(B1, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); - final String B2 = B + ".b2"; conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50); conf.setMaximumCapacity(B2, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); @@ -219,14 +214,12 @@ public class TestCapacitySchedulerWeightMode { conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100); conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 10); conf.setMaximumCapacity(A, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); conf.setCapacityByLabel(A, "x", 100); conf.setCapacityByLabel(A, "y", 50); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacityByLabel(B, RMNodeLabelsManager.NO_LABEL, 90); conf.setMaximumCapacity(B, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); @@ -234,7 +227,6 @@ public class TestCapacitySchedulerWeightMode { conf.setCapacityByLabel(B, "z", 100); // Define 2nd-level queues - final String A1 = A + ".a1"; conf.setQueues(A, new String[] { "a1" }); conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100); conf.setMaximumCapacity(A1, 100); @@ -244,12 +236,10 @@ public class TestCapacitySchedulerWeightMode { conf.setCapacityByLabel(A1, "y", 100); conf.setQueues(B, new String[] { "b1", "b2" }); - final String B1 = B + ".b1"; conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50); conf.setMaximumCapacity(B1, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); - final String B2 = B + ".b2"; conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50); conf.setMaximumCapacity(B2, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); @@ -297,6 +287,57 @@ public class TestCapacitySchedulerWeightMode { getCSConfWithLabelsParentUsePctChildUseWeight(conf)); } + /** + * This checks whether the parent prints the correct log about the + * configured mode. + */ + @Test(timeout = 300000) + public void testGetCapacityOrWeightStringUsingWeights() throws IOException { + try (MockRM rm = new MockRM( + getCSConfWithQueueLabelsWeightOnly(conf))) { + rm.start(); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + String capacityOrWeightString = ((ParentQueue) cs.getQueue(A)) + .getCapacityOrWeightString(); + validateCapacityOrWeightString(capacityOrWeightString, true); + + capacityOrWeightString = ((LeafQueue) cs.getQueue(A1)) + .getCapacityOrWeightString(); + validateCapacityOrWeightString(capacityOrWeightString, true); + + capacityOrWeightString = ((LeafQueue) cs.getQueue(A1)) + .getExtendedCapacityOrWeightString(); + validateCapacityOrWeightString(capacityOrWeightString, true); + } + } + + /** + * This checks whether the parent prints the correct log about the + * configured mode. + */ + @Test(timeout = 300000) + public void testGetCapacityOrWeightStringParentPctLeafWeights() + throws IOException { + try (MockRM rm = new MockRM( + getCSConfWithLabelsParentUseWeightChildUsePct(conf))) { + rm.start(); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + String capacityOrWeightString = ((ParentQueue) cs.getQueue(A)) + .getCapacityOrWeightString(); + validateCapacityOrWeightString(capacityOrWeightString, true); + + capacityOrWeightString = ((LeafQueue) cs.getQueue(A1)) + .getCapacityOrWeightString(); + validateCapacityOrWeightString(capacityOrWeightString, false); + + capacityOrWeightString = ((LeafQueue) cs.getQueue(A1)) + .getExtendedCapacityOrWeightString(); + validateCapacityOrWeightString(capacityOrWeightString, false); + } + } + private void internalTestContainerAlloationWithNodeLabel(Configuration csConf) throws Exception { /* @@ -449,4 +490,15 @@ public class TestCapacitySchedulerWeightMode { } } } + + private void validateCapacityOrWeightString(String capacityOrWeightString, + boolean shouldContainWeight) { + Assert.assertEquals(shouldContainWeight, + capacityOrWeightString.contains("weight")); + Assert.assertEquals(shouldContainWeight, + capacityOrWeightString.contains("normalizedWeight")); + Assert.assertEquals(!shouldContainWeight, + capacityOrWeightString.contains("capacity")); + + } }