Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-10627. Extend logging to give more information about weight mode. Contributed by Benjamin Teke.

This commit is contained in:
Peter Bacsko 2021-02-26 21:57:54 +01:00
parent e04bcb3a06
commit 947b50489d
4 changed files with 95 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
View File

@ -1633,4 +1633,13 @@ public abstract class AbstractCSQueue implements CSQueue {
writeLock.unlock();
}
}
protected String getCapacityOrWeightString() {
if (queueCapacities.getWeight() != -1) {
return "weight=" + queueCapacities.getWeight() + ", " +
"normalizedWeight=" + queueCapacities.getNormalizedWeight();
} else {
return "capacity=" + queueCapacities.getCapacity();
}
}
}

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -268,9 +268,8 @@ public class LeafQueue extends AbstractCSQueue {
usersManager.updateUserWeights();
LOG.info(
"Initializing " + getQueuePath() + "\n" + "capacity = "
+ queueCapacities.getCapacity()
+ " [= (float) configuredCapacity / 100 ]" + "\n"
"Initializing " + getQueuePath() + "\n" +
getExtendedCapacityOrWeightString() + "\n"
+ "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity()
+ " [= parentAbsoluteCapacity * capacity ]" + "\n"
+ "maxCapacity = " + queueCapacities.getMaximumCapacity()
@ -486,7 +485,7 @@ public class LeafQueue extends AbstractCSQueue {
public String toString() {
readLock.lock();
try {
return getQueuePath() + ": " + "capacity=" + queueCapacities.getCapacity()
return getQueuePath() + ": " + getCapacityOrWeightString()
+ ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity()
+ ", " + "usedResources=" + queueUsage.getUsed() + ", "
+ "usedCapacity=" + getUsedCapacity() + ", " + "absoluteUsedCapacity="
@ -499,7 +498,19 @@ public class LeafQueue extends AbstractCSQueue {
} finally {
readLock.unlock();
}
}
protected String getExtendedCapacityOrWeightString() {
if (queueCapacities.getWeight() != -1) {
return "weight = " + queueCapacities.getWeight()
+ " [= (float) configuredCapacity (with w suffix)] " + "\n"
+ "normalizedWeight = " + queueCapacities.getNormalizedWeight()
+ " [= (float) configuredCapacity / sum(configuredCapacity of " +
"all queues under the parent)]";
} else {
return "capacity = " + queueCapacities.getCapacity()
+ " [= (float) configuredCapacity / 100 ]";
}
}
@VisibleForTesting

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
View File

@ -173,7 +173,7 @@ public class ParentQueue extends AbstractCSQueue {
((ParentQueue) parent).getQueueOrderingPolicyConfigName());
queueOrderingPolicy.setQueues(childQueues);
LOG.info(queueName + ", capacity=" + this.queueCapacities.getCapacity()
LOG.info(queueName + ", " + getCapacityOrWeightString()
+ ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity()
+ ", maxCapacity=" + this.queueCapacities.getMaximumCapacity()
+ ", absoluteMaxCapacity=" + this.queueCapacities
@ -462,8 +462,8 @@ public class ParentQueue extends AbstractCSQueue {
public String toString() {
return queueName + ": " +
"numChildQueue= " + childQueues.size() + ", " +
"capacity=" + queueCapacities.getCapacity() + ", " +
"numChildQueue= " + childQueues.size() + ", " +
getCapacityOrWeightString() + ", " +
"absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " +
"usedResources=" + queueUsage.getUsed() +
"usedCapacity=" + getUsedCapacity() + ", " +

84
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
View File

@ -45,11 +45,16 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Set;
public class TestCapacitySchedulerWeightMode {
private final int GB = 1024;
private static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
private static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
private static final String A1 = A + ".a1";
private static final String B1 = B + ".b1";
private static final String B2 = B + ".b2";
private YarnConfiguration conf;
@ -91,14 +96,12 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100);
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100);
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1);
conf.setMaximumCapacity(A, 10);
conf.setAccessibleNodeLabels(A, toSet("x", "y"));
conf.setLabeledQueueWeight(A, "x", 100);
conf.setLabeledQueueWeight(A, "y", 50);
final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9);
conf.setMaximumCapacity(B, 100);
conf.setAccessibleNodeLabels(B, toSet("y", "z"));
@ -106,7 +109,6 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(B, "z", 100);
// Define 2nd-level queues
final String A1 = A + ".a1";
conf.setQueues(A, new String[] { "a1" });
conf.setLabeledQueueWeight(A1, RMNodeLabelsManager.NO_LABEL, 100);
conf.setMaximumCapacity(A1, 100);
@ -116,12 +118,10 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(A1, "y", 100);
conf.setQueues(B, new String[] { "b1", "b2" });
final String B1 = B + ".b1";
conf.setLabeledQueueWeight(B1, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B1, 50);
conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET);
final String B2 = B + ".b2";
conf.setLabeledQueueWeight(B2, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B2, 50);
conf.setAccessibleNodeLabels(B2, toSet("y", "z"));
@ -155,14 +155,12 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100);
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100);
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1);
conf.setMaximumCapacity(A, 10);
conf.setAccessibleNodeLabels(A, toSet("x", "y"));
conf.setLabeledQueueWeight(A, "x", 100);
conf.setLabeledQueueWeight(A, "y", 50);
final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9);
conf.setMaximumCapacity(B, 100);
conf.setAccessibleNodeLabels(B, toSet("y", "z"));
@ -170,7 +168,6 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(B, "z", 100);
// Define 2nd-level queues
final String A1 = A + ".a1";
conf.setQueues(A, new String[] { "a1" });
conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100);
conf.setMaximumCapacity(A1, 100);
@ -180,12 +177,10 @@ public class TestCapacitySchedulerWeightMode {
conf.setCapacityByLabel(A1, "y", 100);
conf.setQueues(B, new String[] { "b1", "b2" });
final String B1 = B + ".b1";
conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B1, 50);
conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET);
final String B2 = B + ".b2";
conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B2, 50);
conf.setAccessibleNodeLabels(B2, toSet("y", "z"));
@ -219,14 +214,12 @@ public class TestCapacitySchedulerWeightMode {
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100);
conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100);
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 10);
conf.setMaximumCapacity(A, 10);
conf.setAccessibleNodeLabels(A, toSet("x", "y"));
conf.setCapacityByLabel(A, "x", 100);
conf.setCapacityByLabel(A, "y", 50);
final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacityByLabel(B, RMNodeLabelsManager.NO_LABEL, 90);
conf.setMaximumCapacity(B, 100);
conf.setAccessibleNodeLabels(B, toSet("y", "z"));
@ -234,7 +227,6 @@ public class TestCapacitySchedulerWeightMode {
conf.setCapacityByLabel(B, "z", 100);
// Define 2nd-level queues
final String A1 = A + ".a1";
conf.setQueues(A, new String[] { "a1" });
conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100);
conf.setMaximumCapacity(A1, 100);
@ -244,12 +236,10 @@ public class TestCapacitySchedulerWeightMode {
conf.setCapacityByLabel(A1, "y", 100);
conf.setQueues(B, new String[] { "b1", "b2" });
final String B1 = B + ".b1";
conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B1, 50);
conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET);
final String B2 = B + ".b2";
conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50);
conf.setMaximumCapacity(B2, 50);
conf.setAccessibleNodeLabels(B2, toSet("y", "z"));
@ -297,6 +287,57 @@ public class TestCapacitySchedulerWeightMode {
getCSConfWithLabelsParentUsePctChildUseWeight(conf));
}
/**
* This checks whether the parent prints the correct log about the
* configured mode.
*/
@Test(timeout = 300000)
public void testGetCapacityOrWeightStringUsingWeights() throws IOException {
try (MockRM rm = new MockRM(
getCSConfWithQueueLabelsWeightOnly(conf))) {
rm.start();
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
String capacityOrWeightString = ((ParentQueue) cs.getQueue(A))
.getCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, true);
capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
.getCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, true);
capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
.getExtendedCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, true);
}
}
/**
* This checks whether the parent prints the correct log about the
* configured mode.
*/
@Test(timeout = 300000)
public void testGetCapacityOrWeightStringParentPctLeafWeights()
throws IOException {
try (MockRM rm = new MockRM(
getCSConfWithLabelsParentUseWeightChildUsePct(conf))) {
rm.start();
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
String capacityOrWeightString = ((ParentQueue) cs.getQueue(A))
.getCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, true);
capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
.getCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, false);
capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
.getExtendedCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, false);
}
}
private void internalTestContainerAlloationWithNodeLabel(Configuration csConf)
throws Exception {
/*
@ -449,4 +490,15 @@ public class TestCapacitySchedulerWeightMode {
}
}
}
private void validateCapacityOrWeightString(String capacityOrWeightString,
boolean shouldContainWeight) {
Assert.assertEquals(shouldContainWeight,
capacityOrWeightString.contains("weight"));
Assert.assertEquals(shouldContainWeight,
capacityOrWeightString.contains("normalizedWeight"));
Assert.assertEquals(!shouldContainWeight,
capacityOrWeightString.contains("capacity"));
}
}