Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6445. [YARN-3926] Performance improvements in resource profile branch with respect to SLS. Contributed by Varun Vasudev. 

(cherry picked from commit 58da54640e)
This commit is contained in:
Sunil G 2017-04-25 11:53:11 +05:30 committed by Daniel Templeton
parent bda5486dad
commit 48ddb8f5b4
11 changed files with 158 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
View File

@ -107,12 +107,23 @@ public abstract class Resource implements Comparable<Resource> {
@InterfaceStability.Unstable
public static Resource newInstance(Resource resource) {
Resource ret = Resource.newInstance(0, 0);
for (Map.Entry<String, ResourceInformation> entry : resource.getResources()
Resource.copy(resource, ret);
return ret;
}
@InterfaceAudience.Private
@InterfaceStability.Unstable
public static void copy(Resource source, Resource dest) {
for (Map.Entry<String, ResourceInformation> entry : source.getResources()
.entrySet()) {
ret.setResourceInformation(entry.getKey(),
try {
ResourceInformation.copy(entry.getValue(),
dest.getResourceInformation(entry.getKey()));
} catch (YarnException ye) {
dest.setResourceInformation(entry.getKey(),
ResourceInformation.newInstance(entry.getValue()));
}
return ret;
}
}
/**

51
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
View File

@ -30,9 +30,9 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
private String name;
private String units;
private ResourceTypes resourceType;
private Long value;
private Long minimumAllocation;
private Long maximumAllocation;
private long value;
private long minimumAllocation;
private long maximumAllocation;
private static final String MEMORY_URI = "memory-mb";
private static final String VCORES_URI = "vcores";
@ -106,7 +106,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
*
* @return the resource value
*/
public Long getValue() {
public long getValue() {
return value;
}
@ -115,7 +115,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
*
* @param rValue the resource value
*/
public void setValue(Long rValue) {
public void setValue(long rValue) {
this.value = rValue;
}
@ -124,7 +124,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
*
* @return the minimum allocation for the resource
*/
public Long getMinimumAllocation() {
public long getMinimumAllocation() {
return minimumAllocation;
}
@ -133,7 +133,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
*
* @param minimumAllocation the minimum allocation for the resource
*/
public void setMinimumAllocation(Long minimumAllocation) {
public void setMinimumAllocation(long minimumAllocation) {
this.minimumAllocation = minimumAllocation;
}
@ -142,7 +142,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
*
* @return the maximum allocation for the resource
*/
public Long getMaximumAllocation() {
public long getMaximumAllocation() {
return maximumAllocation;
}
@ -151,7 +151,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
*
* @param maximumAllocation the maximum allocation for the resource
*/
public void setMaximumAllocation(Long maximumAllocation) {
public void setMaximumAllocation(long maximumAllocation) {
this.maximumAllocation = maximumAllocation;
}
@ -163,18 +163,13 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
*/
public static ResourceInformation newInstance(ResourceInformation other) {
ResourceInformation ret = new ResourceInformation();
ret.setName(other.getName());
ret.setResourceType(other.getResourceType());
ret.setUnits(other.getUnits());
ret.setValue(other.getValue());
ret.setMinimumAllocation(other.getMinimumAllocation());
ret.setMaximumAllocation(other.getMaximumAllocation());
copy(other, ret);
return ret;
}
public static ResourceInformation newInstance(String name, String units,
Long value, ResourceTypes type, Long minimumAllocation,
Long maximumAllocation) {
long value, ResourceTypes type, long minimumAllocation,
long maximumAllocation) {
ResourceInformation ret = new ResourceInformation();
ret.setName(name);
ret.setResourceType(type);
@ -186,7 +181,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
}
public static ResourceInformation newInstance(String name, String units,
Long value) {
long value) {
return ResourceInformation
.newInstance(name, units, value, ResourceTypes.COUNTABLE, 0L,
Long.MAX_VALUE);
@ -198,7 +193,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
Long.MAX_VALUE);
}
public static ResourceInformation newInstance(String name, Long value) {
public static ResourceInformation newInstance(String name, long value) {
return ResourceInformation
.newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,
Long.MAX_VALUE);
@ -208,6 +203,22 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
return ResourceInformation.newInstance(name, "");
}
/**
* Copies the content of the source ResourceInformation object to the
* destination object, overwriting all properties of the destination object.
* @param src Source ResourceInformation object
* @param dst Destination ResourceInformation object
*/
public static void copy(ResourceInformation src, ResourceInformation dst) {
dst.setName(src.getName());
dst.setResourceType(src.getResourceType());
dst.setUnits(src.getUnits());
dst.setValue(src.getValue());
dst.setMinimumAllocation(src.getMinimumAllocation());
dst.setMaximumAllocation(src.getMaximumAllocation());
}
@Override
public String toString() {
return "name: " + this.name + ", units: " + this.units + ", type: "
@ -244,7 +255,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
939769357 + name.hashCode(); // prime * result = 939769357 initially
result = prime * result + resourceType.hashCode();
result = prime * result + units.hashCode();
result = prime * result + value.hashCode();
result = prime * result + Long.hashCode(value);
return result;
}

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
View File

@ -128,23 +128,24 @@ public class UnitsConversionUtil {
* @param fromValue the value you wish to convert
* @return the value in toUnit
*/
public static Long convert(String fromUnit, String toUnit, Long fromValue) {
if (toUnit == null || fromUnit == null || fromValue == null) {
public static long convert(String fromUnit, String toUnit, long fromValue) {
if (toUnit == null || fromUnit == null) {
throw new IllegalArgumentException("One or more arguments are null");
}
String overflowMsg =
"Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit
+ "' will result in an overflow of Long";
if (fromUnit.equals(toUnit)) {
return fromValue;
}
Converter fc = getConverter(fromUnit);
Converter tc = getConverter(toUnit);
Long numerator = fc.numerator * tc.denominator;
Long denominator = fc.denominator * tc.numerator;
Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator;
long numerator = fc.numerator * tc.denominator;
long denominator = fc.denominator * tc.numerator;
long numeratorMultiplierLimit = Long.MAX_VALUE / numerator;
if (numerator < denominator) {
if (numeratorMultiplierLimit < fromValue) {
String overflowMsg =
"Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit
+ "' will result in an overflow of Long";
throw new IllegalArgumentException(overflowMsg);
}
return (fromValue * numerator) / denominator;
@ -152,8 +153,11 @@ public class UnitsConversionUtil {
if (numeratorMultiplierLimit > fromValue) {
return (numerator * fromValue) / denominator;
}
Long tmp = numerator / denominator;
long tmp = numerator / denominator;
if ((Long.MAX_VALUE / tmp) < fromValue) {
String overflowMsg =
"Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit
+ "' will result in an overflow of Long";
throw new IllegalArgumentException(overflowMsg);
}
return fromValue * tmp;
@ -170,8 +174,8 @@ public class UnitsConversionUtil {
* @return +1, 0 or -1 depending on whether the relationship is greater than,
* equal to or lesser than
*/
public static int compare(String unitA, Long valueA, String unitB,
Long valueB) {
public static int compare(String unitA, long valueA, String unitB,
long valueB) {
if (unitA == null || unitB == null || !KNOWN_UNITS.contains(unitA)
|| !KNOWN_UNITS.contains(unitB)) {
throw new IllegalArgumentException("Units cannot be null");
@ -185,19 +189,19 @@ public class UnitsConversionUtil {
Converter unitAC = getConverter(unitA);
Converter unitBC = getConverter(unitB);
if (unitA.equals(unitB)) {
return valueA.compareTo(valueB);
return Long.valueOf(valueA).compareTo(valueB);
}
int unitAPos = SORTED_UNITS.indexOf(unitA);
int unitBPos = SORTED_UNITS.indexOf(unitB);
try {
Long tmpA = valueA;
Long tmpB = valueB;
long tmpA = valueA;
long tmpB = valueB;
if (unitAPos < unitBPos) {
tmpB = convert(unitB, unitA, valueB);
} else {
tmpA = convert(unitA, unitB, valueA);
}
return tmpA.compareTo(tmpB);
return Long.valueOf(tmpA).compareTo(tmpB);
} catch (IllegalArgumentException ie) {
BigInteger tmpA = BigInteger.valueOf(valueA);
BigInteger tmpB = BigInteger.valueOf(valueB);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java
View File

@ -50,7 +50,7 @@ public class TestResourceInformation {
@Test
public void testValue() {
String name = "yarn.io/test";
Long value = 1l;
long value = 1L;
ResourceInformation ri = ResourceInformation.newInstance(name, value);
Assert.assertEquals("Resource name incorrect", name, ri.getName());
Assert.assertEquals("Resource value incorrect", value, ri.getValue());
@ -59,7 +59,7 @@ public class TestResourceInformation {
@Test
public void testResourceInformation() {
String name = "yarn.io/test";
Long value = 1l;
long value = 1L;
String units = "m";
ResourceInformation ri =
ResourceInformation.newInstance(name, units, value);

60
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java
View File

@ -27,62 +27,62 @@ public class TestUnitsConversionUtil {
public void testUnitsConversion() {
int value = 5;
String fromUnit = "";
Long test = Long.valueOf(value);
long test = value;
Assert.assertEquals("pico test failed",
Long.valueOf(value * 1000l * 1000l * 1000l * 1000l),
value * 1000L * 1000L * 1000L * 1000L,
UnitsConversionUtil.convert(fromUnit, "p", test));
Assert.assertEquals("nano test failed",
Long.valueOf(value * 1000l * 1000l * 1000l),
value * 1000L * 1000L * 1000L,
UnitsConversionUtil.convert(fromUnit, "n", test));
Assert
.assertEquals("micro test failed", Long.valueOf(value * 1000l * 1000l),
.assertEquals("micro test failed", value * 1000L * 1000L,
UnitsConversionUtil.convert(fromUnit, "u", test));
Assert.assertEquals("milli test failed", Long.valueOf(value * 1000l),
Assert.assertEquals("milli test failed", value * 1000L,
UnitsConversionUtil.convert(fromUnit, "m", test));
test = Long.valueOf(value * 1000l * 1000l * 1000l * 1000l * 1000l);
test = value * 1000L * 1000L * 1000L * 1000L * 1000L;
fromUnit = "";
Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l),
Assert.assertEquals("kilo test failed", test / 1000L,
UnitsConversionUtil.convert(fromUnit, "k", test));
Assert
.assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)),
.assertEquals("mega test failed", test / (1000L * 1000L),
UnitsConversionUtil.convert(fromUnit, "M", test));
Assert.assertEquals("giga test failed",
Long.valueOf(test / (1000l * 1000l * 1000l)),
test / (1000L * 1000L * 1000L),
UnitsConversionUtil.convert(fromUnit, "G", test));
Assert.assertEquals("tera test failed",
Long.valueOf(test / (1000l * 1000l * 1000l * 1000l)),
test / (1000L * 1000L * 1000L * 1000L),
UnitsConversionUtil.convert(fromUnit, "T", test));
Assert.assertEquals("peta test failed",
Long.valueOf(test / (1000l * 1000l * 1000l * 1000l * 1000l)),
test / (1000L * 1000L * 1000L * 1000L * 1000L),
UnitsConversionUtil.convert(fromUnit, "P", test));
Assert.assertEquals("nano to pico test failed", Long.valueOf(value * 1000l),
UnitsConversionUtil.convert("n", "p", Long.valueOf(value)));
Assert.assertEquals("nano to pico test failed", value * 1000L,
UnitsConversionUtil.convert("n", "p", value));
Assert.assertEquals("mega to giga test failed", Long.valueOf(value),
UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l)));
Assert.assertEquals("mega to giga test failed", value,
UnitsConversionUtil.convert("M", "G", value * 1000L));
Assert.assertEquals("Mi to Gi test failed", Long.valueOf(value),
UnitsConversionUtil.convert("Mi", "Gi", Long.valueOf(value * 1024l)));
Assert.assertEquals("Mi to Gi test failed", value,
UnitsConversionUtil.convert("Mi", "Gi", value * 1024L));
Assert.assertEquals("Mi to Ki test failed", Long.valueOf(value * 1024),
UnitsConversionUtil.convert("Mi", "Ki", Long.valueOf(value)));
Assert.assertEquals("Mi to Ki test failed", value * 1024,
UnitsConversionUtil.convert("Mi", "Ki", value));
Assert.assertEquals("Ki to base units test failed", Long.valueOf(5 * 1024),
UnitsConversionUtil.convert("Ki", "", Long.valueOf(5)));
Assert.assertEquals("Ki to base units test failed", 5 * 1024,
UnitsConversionUtil.convert("Ki", "", 5));
Assert.assertEquals("Mi to k test failed", Long.valueOf(1073741),
UnitsConversionUtil.convert("Mi", "k", Long.valueOf(1024)));
Assert.assertEquals("Mi to k test failed", 1073741,
UnitsConversionUtil.convert("Mi", "k", 1024));
Assert.assertEquals("M to Mi test failed", Long.valueOf(953),
UnitsConversionUtil.convert("M", "Mi", Long.valueOf(1000)));
Assert.assertEquals("M to Mi test failed", 953,
UnitsConversionUtil.convert("M", "Mi", 1000));
}
@Test
public void testOverflow() {
Long test = Long.valueOf(5 * 1000l * 1000l * 1000l * 1000l * 1000l);
long test = 5 * 1000L * 1000L * 1000L * 1000L * 1000L;
try {
UnitsConversionUtil.convert("P", "p", test);
Assert.fail("this operation should result in an overflow");
@ -100,9 +100,9 @@ public class TestUnitsConversionUtil {
@Test
public void testCompare() {
String unitA = "P";
Long valueA = Long.valueOf(1);
long valueA = 1;
String unitB = "p";
Long valueB = Long.valueOf(2);
long valueB = 2;
Assert.assertEquals(1,
UnitsConversionUtil.compare(unitA, valueA, unitB, valueB));
Assert.assertEquals(-1,
@ -120,7 +120,7 @@ public class TestUnitsConversionUtil {
Assert.assertEquals(-1,
UnitsConversionUtil.compare(unitB, valueB, unitA, valueA));
Assert.assertEquals(0,
UnitsConversionUtil.compare(unitA, valueA, unitB, 1000l));
UnitsConversionUtil.compare(unitA, valueA, unitB, 1000L));
unitA = "p";
unitB = "n";
@ -129,7 +129,7 @@ public class TestUnitsConversionUtil {
Assert.assertEquals(1,
UnitsConversionUtil.compare(unitB, valueB, unitA, valueA));
Assert.assertEquals(0,
UnitsConversionUtil.compare(unitA, 1000l, unitB, valueA));
UnitsConversionUtil.compare(unitA, 1000L, unitB, valueA));
}
}

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
import org.apache.commons.collections.map.UnmodifiableMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@ -46,6 +47,7 @@ public class ResourcePBImpl extends Resource {
boolean viaProto = false;
private Map<String, ResourceInformation> resources;
private Map<String, ResourceInformation> readOnlyResources;
// call via ProtoUtils.convertToProtoFormat(Resource)
@ -68,6 +70,7 @@ public class ResourcePBImpl extends Resource {
public ResourcePBImpl(ResourceProto proto) {
this.proto = proto;
viaProto = true;
this.readOnlyResources = null;
this.resources = null;
initResources();
}
@ -111,10 +114,9 @@ public class ResourcePBImpl extends Resource {
@Override
public void setMemorySize(long memory) {
setResourceInformation(ResourceInformation.MEMORY_MB.getName(),
ResourceInformation.newInstance(ResourceInformation.MEMORY_MB.getName(),
ResourceInformation.MEMORY_MB.getUnits(), memory));
maybeInitBuilder();
getResourceInformation(ResourceInformation.MEMORY_MB.getName())
.setValue(memory);
}
@Override
@ -127,9 +129,9 @@ public class ResourcePBImpl extends Resource {
@Override
public void setVirtualCores(int vCores) {
setResourceInformation(ResourceInformation.VCORES.getName(),
ResourceInformation.newInstance(ResourceInformation.VCORES.getName(),
ResourceInformation.VCORES.getUnits(), (long) vCores));
maybeInitBuilder();
getResourceInformation(ResourceInformation.VCORES.getName())
.setValue(vCores);
}
private void initResources() {
@ -143,7 +145,7 @@ public class ResourcePBImpl extends Resource {
entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) :
ResourceTypes.COUNTABLE;
String units = entry.hasUnits() ? entry.getUnits() : "";
Long value = entry.hasValue() ? entry.getValue() : 0L;
long value = entry.hasValue() ? entry.getValue() : 0L;
ResourceInformation ri = ResourceInformation
.newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE);
if (resources.containsKey(ri.getName())) {
@ -171,7 +173,7 @@ public class ResourcePBImpl extends Resource {
}
initResources();
if (resources.containsKey(resource)) {
resources.put(resource, resourceInformation);
ResourceInformation.copy(resourceInformation, resources.get(resource));
}
}
@ -193,7 +195,7 @@ public class ResourcePBImpl extends Resource {
@Override
public Map<String, ResourceInformation> getResources() {
initResources();
return Collections.unmodifiableMap(this.resources);
return readOnlyResources;
}
@Override
@ -226,6 +228,7 @@ public class ResourcePBImpl extends Resource {
resources.put(entry.getKey(),
ResourceInformation.newInstance(entry.getValue()));
}
readOnlyResources = Collections.unmodifiableMap(resources);
}
}

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
View File

@ -154,7 +154,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
clusterResource.getResourceInformation(rName);
ResourceInformation resourceInformation =
resource.getResourceInformation(rName);
Long resourceValue = UnitsConversionUtil
long resourceValue = UnitsConversionUtil
.convert(resourceInformation.getUnits(),
clusterResourceResourceInformation.getUnits(),
resourceInformation.getValue());
@ -180,11 +180,11 @@ public class DominantResourceCalculator extends ResourceCalculator {
available.getResourceInformation(resource);
ResourceInformation requiredResource =
required.getResourceInformation(resource);
Long requiredResourceValue = UnitsConversionUtil
long requiredResourceValue = UnitsConversionUtil
.convert(requiredResource.getUnits(), availableResource.getUnits(),
requiredResource.getValue());
if (requiredResourceValue != 0) {
Long tmp = availableResource.getValue() / requiredResourceValue;
long tmp = availableResource.getValue() / requiredResourceValue;
min = min < tmp ? min : tmp;
}
} catch (YarnException ye) {
@ -228,7 +228,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
a.getResourceInformation(resource);
ResourceInformation bResourceInformation =
b.getResourceInformation(resource);
Long bResourceValue = UnitsConversionUtil
long bResourceValue = UnitsConversionUtil
.convert(bResourceInformation.getUnits(),
aResourceInformation.getUnits(),
bResourceInformation.getValue());
@ -249,14 +249,13 @@ public class DominantResourceCalculator extends ResourceCalculator {
}
public Resource divideAndCeil(Resource numerator, long denominator) {
Resource ret = Resources.createResource(0, 0);
Resource ret = Resource.newInstance(numerator);
for (String resource : resourceNames) {
try {
ResourceInformation resourceInformation = ResourceInformation
.newInstance(numerator.getResourceInformation(resource));
ResourceInformation resourceInformation =
ret.getResourceInformation(resource);
resourceInformation.setValue(
divideAndCeil(resourceInformation.getValue(), denominator));
ret.setResourceInformation(resource, resourceInformation);
} catch (YarnException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
@ -276,7 +275,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource normalize(Resource r, Resource minimumResource,
Resource maximumResource, Resource stepFactor) {
Resource ret = Resources.createResource(0, 0);
Resource ret = Resource.newInstance(r);
for (String resource : resourceNames) {
try {
ResourceInformation rResourceInformation =
@ -287,28 +286,26 @@ public class DominantResourceCalculator extends ResourceCalculator {
maximumResource.getResourceInformation(resource);
ResourceInformation stepFactorResourceInformation =
stepFactor.getResourceInformation(resource);
ResourceInformation tmp =
ResourceInformation.newInstance(rResourceInformation);
ResourceInformation tmp = ret.getResourceInformation(resource);
Long rValue = rResourceInformation.getValue();
Long minimumValue = UnitsConversionUtil
long rValue = rResourceInformation.getValue();
long minimumValue = UnitsConversionUtil
.convert(minimumResourceInformation.getUnits(),
rResourceInformation.getUnits(),
minimumResourceInformation.getValue());
Long maximumValue = UnitsConversionUtil
long maximumValue = UnitsConversionUtil
.convert(maximumResourceInformation.getUnits(),
rResourceInformation.getUnits(),
maximumResourceInformation.getValue());
Long stepFactorValue = UnitsConversionUtil
long stepFactorValue = UnitsConversionUtil
.convert(stepFactorResourceInformation.getUnits(),
rResourceInformation.getUnits(),
stepFactorResourceInformation.getValue());
Long value = Math.max(rValue, minimumValue);
long value = Math.max(rValue, minimumValue);
if (stepFactorValue != 0) {
value = roundUp(value, stepFactorValue);
}
tmp.setValue(Math.min(value, maximumValue));
ret.setResourceInformation(resource, tmp);
} catch (YarnException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
@ -328,28 +325,27 @@ public class DominantResourceCalculator extends ResourceCalculator {
}
private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) {
Resource ret = Resources.createResource(0, 0);
Resource ret = Resource.newInstance(r);
for (String resource : resourceNames) {
try {
ResourceInformation rResourceInformation =
r.getResourceInformation(resource);
ResourceInformation stepFactorResourceInformation =
stepFactor.getResourceInformation(resource);
ResourceInformation tmp =
ResourceInformation.newInstance(rResourceInformation);
Long rValue = rResourceInformation.getValue();
Long stepFactorValue = UnitsConversionUtil
long rValue = rResourceInformation.getValue();
long stepFactorValue = UnitsConversionUtil
.convert(stepFactorResourceInformation.getUnits(),
rResourceInformation.getUnits(),
stepFactorResourceInformation.getValue());
Long value = rValue;
long value = rValue;
if (stepFactorValue != 0) {
value = roundUp ? roundUp(rValue, stepFactorValue) :
roundDown(rValue, stepFactorValue);
}
tmp.setValue(value);
ret.setResourceInformation(resource, tmp);
ResourceInformation
.copy(rResourceInformation, ret.getResourceInformation(resource));
ret.getResourceInformation(resource).setValue(value);
} catch (YarnException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
@ -372,15 +368,14 @@ public class DominantResourceCalculator extends ResourceCalculator {
private Resource multiplyAndNormalize(Resource r, double by,
Resource stepFactor, boolean roundUp) {
Resource ret = Resources.createResource(0, 0);
Resource ret = Resource.newInstance(r);
for (String resource : resourceNames) {
try {
ResourceInformation rResourceInformation =
r.getResourceInformation(resource);
ResourceInformation stepFactorResourceInformation =
stepFactor.getResourceInformation(resource);
ResourceInformation tmp =
ResourceInformation.newInstance(rResourceInformation);
ResourceInformation tmp = ret.getResourceInformation(resource);
Long rValue = rResourceInformation.getValue();
Long stepFactorValue = UnitsConversionUtil
@ -397,7 +392,6 @@ public class DominantResourceCalculator extends ResourceCalculator {
roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by);
}
tmp.setValue(value);
ret.setResourceInformation(resource, tmp);
} catch (YarnException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
@ -414,7 +408,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
smaller.getResourceInformation(resource);
ResourceInformation bResourceInformation =
bigger.getResourceInformation(resource);
Long sResourceValue = UnitsConversionUtil
long sResourceValue = UnitsConversionUtil
.convert(sResourceInformation.getUnits(),
bResourceInformation.getUnits(),
sResourceInformation.getValue());

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
View File

@ -42,7 +42,7 @@ public class Resources {
static class FixedValueResource extends Resource {
private Map<String, ResourceInformation> resources;
private Long resourceValue;
private long resourceValue;
private String name;
/**
@ -50,7 +50,7 @@ public class Resources {
* @param rName the name of the resource
* @param value the fixed value to be returned for all resource types
*/
FixedValueResource(String rName, Long value) {
FixedValueResource(String rName, long value) {
this.resourceValue = value;
this.name = rName;
resources = initResourceMap();
@ -60,7 +60,7 @@ public class Resources {
if(this.resourceValue > Integer.MAX_VALUE) {
return Integer.MAX_VALUE;
}
return this.resourceValue.intValue();
return Long.valueOf(this.resourceValue).intValue();
}
@Override
@ -203,7 +203,7 @@ public class Resources {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
ResourceInformation lhsValue = entry.getValue();
Long convertedRhs = UnitsConversionUtil
long convertedRhs = UnitsConversionUtil
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
rhsValue.getValue());
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
@ -225,7 +225,7 @@ public class Resources {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
ResourceInformation lhsValue = entry.getValue();
Long convertedRhs = UnitsConversionUtil
long convertedRhs = UnitsConversionUtil
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
rhsValue.getValue());
lhs.setResourceValue(name, lhsValue.getValue() - convertedRhs);
@ -288,7 +288,7 @@ public class Resources {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
ResourceInformation lhsValue = entry.getValue();
Long convertedRhs = (long) (UnitsConversionUtil
long convertedRhs = (long) (UnitsConversionUtil
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
rhsValue.getValue()) * by);
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
@ -422,7 +422,7 @@ public class Resources {
try {
ResourceInformation rhsValue = bigger.getResourceInformation(name);
ResourceInformation lhsValue = entry.getValue();
Long convertedRhs = UnitsConversionUtil
long convertedRhs = UnitsConversionUtil
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
rhsValue.getValue());
if(lhsValue.getValue() > convertedRhs) {
@ -448,7 +448,7 @@ public class Resources {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
ResourceInformation lhsValue = entry.getValue();
Long convertedRhs = UnitsConversionUtil
long convertedRhs = UnitsConversionUtil
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
rhsValue.getValue());
ResourceInformation outInfo =
@ -469,7 +469,7 @@ public class Resources {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
ResourceInformation lhsValue = entry.getValue();
Long convertedRhs = UnitsConversionUtil
long convertedRhs = UnitsConversionUtil
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
rhsValue.getValue());
ResourceInformation outInfo =

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
View File

@ -283,6 +283,7 @@ public class TestResourceUtils {
Map<String, ResourceInformation> actual =
ResourceUtils.getNodeResourceInformation(conf);
Assert.assertEquals(entry.getValue().getResources(), actual);
dest.delete();
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
View File

@ -42,6 +42,16 @@ import static org.junit.Assert.assertTrue;
public class TestResources {
static class ExtendedResources extends Resources {
public static Resource unbounded() {
return new FixedValueResource("UNBOUNDED", Long.MAX_VALUE);
}
public static Resource none() {
return new FixedValueResource("NONE", 0L);
}
}
private static final String EXTRA_RESOURCE_TYPE = "resource2";
private String resourceTypesFile;
@ -87,7 +97,7 @@ public class TestResources {
@Test(timeout = 10000)
public void testCompareToWithUnboundedResource() {
unsetExtraResourceType();
Resource unboundedClone = Resources.clone(Resources.unbounded());
Resource unboundedClone = Resources.clone(ExtendedResources.unbounded());
assertTrue(unboundedClone
.compareTo(createResource(Long.MAX_VALUE, Integer.MAX_VALUE)) == 0);
assertTrue(unboundedClone.compareTo(createResource(Long.MAX_VALUE, 0)) > 0);

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
View File

@ -1334,15 +1334,7 @@ public abstract class AbstractYarnScheduler
* @return a Resource object with the minimum allocation for the scheduler
*/
public Resource getMinimumAllocation() {
boolean profilesEnabled = getConfig()
.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
Resource ret;
if (!profilesEnabled) {
ret = ResourceUtils.getResourceTypesMinimumAllocation();
} else {
ret = rmContext.getResourceProfilesManager().getMinimumProfile();
}
Resource ret = ResourceUtils.getResourceTypesMinimumAllocation();
LOG.info("Minimum allocation = " + ret);
return ret;
}
@ -1356,15 +1348,7 @@ public abstract class AbstractYarnScheduler
*/
public Resource getMaximumAllocation() {
boolean profilesEnabled = getConfig()
.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
Resource ret;
if (!profilesEnabled) {
ret = ResourceUtils.getResourceTypesMaximumAllocation();
} else {
ret = rmContext.getResourceProfilesManager().getMaximumProfile();
}
Resource ret = ResourceUtils.getResourceTypesMaximumAllocation();
LOG.info("Maximum allocation = " + ret);
return ret;
}