Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9785. Fix DominantResourceCalculator when one resource is zero. Contributed by Bibin A Chundatt, Sunil Govindan, Bilwa S T.

Browse Source 
(cherry picked from commit bb26514ba9e8a038749c4acd592a47d5cd5bf92a)
This commit is contained in:
Zhankun Tang 2019-09-03 14:55:15 +08:00
parent 1e6095f16b
commit 269aa7ebfe
2 changed files with 48 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
View File

@ -102,7 +102,7 @@ public int compare(Resource clusterResource, Resource lhs, Resource rhs,
return 0;
}
if (isInvalidDivisor(clusterResource)) {
if (isAllInvalidDivisor(clusterResource)) {
return this.compare(lhs, rhs);
}
@ -279,6 +279,11 @@ private void calculateShares(ResourceInformation[] clusterRes, Resource first,
firstShares[i] = calculateShare(clusterRes[i], firstRes[i]);
secondShares[i] = calculateShare(clusterRes[i], secondRes[i]);
if (firstShares[i] == Float.POSITIVE_INFINITY ||
secondShares[i] == Float.POSITIVE_INFINITY) {
continue;
}
if (firstShares[i] > max[0]) {
max[0] = firstShares[i];
}
@ -297,6 +302,9 @@ private void calculateShares(ResourceInformation[] clusterRes, Resource first,
*/
private double calculateShare(ResourceInformation clusterRes,
ResourceInformation res) {
if(clusterRes.getValue() == 0) {
return Float.POSITIVE_INFINITY;
}
return (double) res.getValue() / clusterRes.getValue();
}
@ -316,6 +324,10 @@ private double compareShares(double[] lhsShares, double[] rhsShares) {
// lhsShares and rhsShares must necessarily have the same length, because
// everyone uses the same master resource list.
for (int i = lhsShares.length - 1; i >= 0; i--) {
if (lhsShares[i] == Float.POSITIVE_INFINITY ||
rhsShares[i] == Float.POSITIVE_INFINITY) {
continue;
}
diff = lhsShares[i] - rhsShares[i];
if (diff != 0.0) {
@ -372,6 +384,18 @@ public boolean isInvalidDivisor(Resource r) {
return false;
}
public boolean isAllInvalidDivisor(Resource r) {
boolean flag = true;
for (ResourceInformation res : r.getResources()) {
if (flag == true && res.getValue() == 0L) {
flag = true;
continue;
}
flag = false;
}
return flag;
}
@Override
public float ratio(Resource a, Resource b) {
float ratio = 0.0f;

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
View File

@ -186,6 +186,7 @@ public void testCompare() {
testCompareDefault(cluster);
} else if (resourceCalculator instanceof DominantResourceCalculator) {
testCompareDominant(cluster);
testCompareDominantZeroValueResource();
}
}
@ -201,6 +202,28 @@ private void testCompareDefault(Resource cluster) {
assertComparison(cluster, newResource(2, 1, 1), newResource(1, 0, 0), 1);
}
/**
* Verify compare when one or all the resource are zero.
*/
private void testCompareDominantZeroValueResource(){
Resource cluster = newResource(4L, 4, 0);
assertComparison(cluster, newResource(2, 1, 1), newResource(1, 1, 2), 1);
assertComparison(cluster, newResource(2, 2, 1), newResource(1, 2, 2), 1);
assertComparison(cluster, newResource(2, 2, 1), newResource(2, 2, 2), 0);
assertComparison(cluster, newResource(0, 2, 1), newResource(0, 2, 2), 0);
assertComparison(cluster, newResource(0, 1, 2), newResource(1, 1, 2), -1);
assertComparison(cluster, newResource(1, 1, 2), newResource(2, 1, 2), -1);
// cluster resource zero
cluster = newResource(0, 0, 0);
assertComparison(cluster, newResource(2, 1, 1), newResource(1, 1, 1), 1);
assertComparison(cluster, newResource(2, 2, 2), newResource(1, 1, 1), 1);
assertComparison(cluster, newResource(2, 1, 1), newResource(1, 2, 1), 0);
assertComparison(cluster, newResource(1, 1, 1), newResource(1, 1, 1), 0);
assertComparison(cluster, newResource(1, 1, 1), newResource(1, 1, 2), -1);
assertComparison(cluster, newResource(1, 1, 1), newResource(1, 2, 1), -1);
}
private void testCompareDominant(Resource cluster) {
assertComparison(cluster, newResource(2, 1, 1), newResource(2, 1, 1), 0);
assertComparison(cluster, newResource(2, 1, 1), newResource(1, 2, 1), 0);