YARN-6445. [YARN-3926] Performance improvements in resource profile branch with respect to SLS. Contributed by Varun Vasudev.
This commit is contained in:
parent
dae65f3bef
commit
58da54640e
|
@ -107,14 +107,25 @@ public abstract class Resource implements Comparable<Resource> {
|
|||
@InterfaceStability.Unstable
|
||||
public static Resource newInstance(Resource resource) {
|
||||
Resource ret = Resource.newInstance(0, 0);
|
||||
for (Map.Entry<String, ResourceInformation> entry : resource.getResources()
|
||||
.entrySet()) {
|
||||
ret.setResourceInformation(entry.getKey(),
|
||||
ResourceInformation.newInstance(entry.getValue()));
|
||||
}
|
||||
Resource.copy(resource, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Unstable
|
||||
public static void copy(Resource source, Resource dest) {
|
||||
for (Map.Entry<String, ResourceInformation> entry : source.getResources()
|
||||
.entrySet()) {
|
||||
try {
|
||||
ResourceInformation.copy(entry.getValue(),
|
||||
dest.getResourceInformation(entry.getKey()));
|
||||
} catch (YarnException ye) {
|
||||
dest.setResourceInformation(entry.getKey(),
|
||||
ResourceInformation.newInstance(entry.getValue()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is DEPRECATED:
|
||||
* Use {@link Resource#getMemorySize()} instead
|
||||
|
|
|
@ -30,9 +30,9 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
private String name;
|
||||
private String units;
|
||||
private ResourceTypes resourceType;
|
||||
private Long value;
|
||||
private Long minimumAllocation;
|
||||
private Long maximumAllocation;
|
||||
private long value;
|
||||
private long minimumAllocation;
|
||||
private long maximumAllocation;
|
||||
|
||||
private static final String MEMORY_URI = "memory-mb";
|
||||
private static final String VCORES_URI = "vcores";
|
||||
|
@ -106,7 +106,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
*
|
||||
* @return the resource value
|
||||
*/
|
||||
public Long getValue() {
|
||||
public long getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
|
@ -115,7 +115,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
*
|
||||
* @param rValue the resource value
|
||||
*/
|
||||
public void setValue(Long rValue) {
|
||||
public void setValue(long rValue) {
|
||||
this.value = rValue;
|
||||
}
|
||||
|
||||
|
@ -124,7 +124,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
*
|
||||
* @return the minimum allocation for the resource
|
||||
*/
|
||||
public Long getMinimumAllocation() {
|
||||
public long getMinimumAllocation() {
|
||||
return minimumAllocation;
|
||||
}
|
||||
|
||||
|
@ -133,7 +133,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
*
|
||||
* @param minimumAllocation the minimum allocation for the resource
|
||||
*/
|
||||
public void setMinimumAllocation(Long minimumAllocation) {
|
||||
public void setMinimumAllocation(long minimumAllocation) {
|
||||
this.minimumAllocation = minimumAllocation;
|
||||
}
|
||||
|
||||
|
@ -142,7 +142,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
*
|
||||
* @return the maximum allocation for the resource
|
||||
*/
|
||||
public Long getMaximumAllocation() {
|
||||
public long getMaximumAllocation() {
|
||||
return maximumAllocation;
|
||||
}
|
||||
|
||||
|
@ -151,7 +151,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
*
|
||||
* @param maximumAllocation the maximum allocation for the resource
|
||||
*/
|
||||
public void setMaximumAllocation(Long maximumAllocation) {
|
||||
public void setMaximumAllocation(long maximumAllocation) {
|
||||
this.maximumAllocation = maximumAllocation;
|
||||
}
|
||||
|
||||
|
@ -163,18 +163,13 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
*/
|
||||
public static ResourceInformation newInstance(ResourceInformation other) {
|
||||
ResourceInformation ret = new ResourceInformation();
|
||||
ret.setName(other.getName());
|
||||
ret.setResourceType(other.getResourceType());
|
||||
ret.setUnits(other.getUnits());
|
||||
ret.setValue(other.getValue());
|
||||
ret.setMinimumAllocation(other.getMinimumAllocation());
|
||||
ret.setMaximumAllocation(other.getMaximumAllocation());
|
||||
copy(other, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static ResourceInformation newInstance(String name, String units,
|
||||
Long value, ResourceTypes type, Long minimumAllocation,
|
||||
Long maximumAllocation) {
|
||||
long value, ResourceTypes type, long minimumAllocation,
|
||||
long maximumAllocation) {
|
||||
ResourceInformation ret = new ResourceInformation();
|
||||
ret.setName(name);
|
||||
ret.setResourceType(type);
|
||||
|
@ -186,7 +181,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
}
|
||||
|
||||
public static ResourceInformation newInstance(String name, String units,
|
||||
Long value) {
|
||||
long value) {
|
||||
return ResourceInformation
|
||||
.newInstance(name, units, value, ResourceTypes.COUNTABLE, 0L,
|
||||
Long.MAX_VALUE);
|
||||
|
@ -198,7 +193,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
Long.MAX_VALUE);
|
||||
}
|
||||
|
||||
public static ResourceInformation newInstance(String name, Long value) {
|
||||
public static ResourceInformation newInstance(String name, long value) {
|
||||
return ResourceInformation
|
||||
.newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,
|
||||
Long.MAX_VALUE);
|
||||
|
@ -208,6 +203,22 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
return ResourceInformation.newInstance(name, "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies the content of the source ResourceInformation object to the
|
||||
* destination object, overwriting all properties of the destination object.
|
||||
* @param src Source ResourceInformation object
|
||||
* @param dst Destination ResourceInformation object
|
||||
*/
|
||||
|
||||
public static void copy(ResourceInformation src, ResourceInformation dst) {
|
||||
dst.setName(src.getName());
|
||||
dst.setResourceType(src.getResourceType());
|
||||
dst.setUnits(src.getUnits());
|
||||
dst.setValue(src.getValue());
|
||||
dst.setMinimumAllocation(src.getMinimumAllocation());
|
||||
dst.setMaximumAllocation(src.getMaximumAllocation());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "name: " + this.name + ", units: " + this.units + ", type: "
|
||||
|
@ -244,7 +255,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
939769357 + name.hashCode(); // prime * result = 939769357 initially
|
||||
result = prime * result + resourceType.hashCode();
|
||||
result = prime * result + units.hashCode();
|
||||
result = prime * result + value.hashCode();
|
||||
result = prime * result + Long.hashCode(value);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
|
@ -128,23 +128,24 @@ public class UnitsConversionUtil {
|
|||
* @param fromValue the value you wish to convert
|
||||
* @return the value in toUnit
|
||||
*/
|
||||
public static Long convert(String fromUnit, String toUnit, Long fromValue) {
|
||||
if (toUnit == null || fromUnit == null || fromValue == null) {
|
||||
public static long convert(String fromUnit, String toUnit, long fromValue) {
|
||||
if (toUnit == null || fromUnit == null) {
|
||||
throw new IllegalArgumentException("One or more arguments are null");
|
||||
}
|
||||
String overflowMsg =
|
||||
"Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit
|
||||
+ "' will result in an overflow of Long";
|
||||
|
||||
if (fromUnit.equals(toUnit)) {
|
||||
return fromValue;
|
||||
}
|
||||
Converter fc = getConverter(fromUnit);
|
||||
Converter tc = getConverter(toUnit);
|
||||
Long numerator = fc.numerator * tc.denominator;
|
||||
Long denominator = fc.denominator * tc.numerator;
|
||||
Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator;
|
||||
long numerator = fc.numerator * tc.denominator;
|
||||
long denominator = fc.denominator * tc.numerator;
|
||||
long numeratorMultiplierLimit = Long.MAX_VALUE / numerator;
|
||||
if (numerator < denominator) {
|
||||
if (numeratorMultiplierLimit < fromValue) {
|
||||
String overflowMsg =
|
||||
"Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit
|
||||
+ "' will result in an overflow of Long";
|
||||
throw new IllegalArgumentException(overflowMsg);
|
||||
}
|
||||
return (fromValue * numerator) / denominator;
|
||||
|
@ -152,8 +153,11 @@ public class UnitsConversionUtil {
|
|||
if (numeratorMultiplierLimit > fromValue) {
|
||||
return (numerator * fromValue) / denominator;
|
||||
}
|
||||
Long tmp = numerator / denominator;
|
||||
long tmp = numerator / denominator;
|
||||
if ((Long.MAX_VALUE / tmp) < fromValue) {
|
||||
String overflowMsg =
|
||||
"Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit
|
||||
+ "' will result in an overflow of Long";
|
||||
throw new IllegalArgumentException(overflowMsg);
|
||||
}
|
||||
return fromValue * tmp;
|
||||
|
@ -170,8 +174,8 @@ public class UnitsConversionUtil {
|
|||
* @return +1, 0 or -1 depending on whether the relationship is greater than,
|
||||
* equal to or lesser than
|
||||
*/
|
||||
public static int compare(String unitA, Long valueA, String unitB,
|
||||
Long valueB) {
|
||||
public static int compare(String unitA, long valueA, String unitB,
|
||||
long valueB) {
|
||||
if (unitA == null || unitB == null || !KNOWN_UNITS.contains(unitA)
|
||||
|| !KNOWN_UNITS.contains(unitB)) {
|
||||
throw new IllegalArgumentException("Units cannot be null");
|
||||
|
@ -185,19 +189,19 @@ public class UnitsConversionUtil {
|
|||
Converter unitAC = getConverter(unitA);
|
||||
Converter unitBC = getConverter(unitB);
|
||||
if (unitA.equals(unitB)) {
|
||||
return valueA.compareTo(valueB);
|
||||
return Long.valueOf(valueA).compareTo(valueB);
|
||||
}
|
||||
int unitAPos = SORTED_UNITS.indexOf(unitA);
|
||||
int unitBPos = SORTED_UNITS.indexOf(unitB);
|
||||
try {
|
||||
Long tmpA = valueA;
|
||||
Long tmpB = valueB;
|
||||
long tmpA = valueA;
|
||||
long tmpB = valueB;
|
||||
if (unitAPos < unitBPos) {
|
||||
tmpB = convert(unitB, unitA, valueB);
|
||||
} else {
|
||||
tmpA = convert(unitA, unitB, valueA);
|
||||
}
|
||||
return tmpA.compareTo(tmpB);
|
||||
return Long.valueOf(tmpA).compareTo(tmpB);
|
||||
} catch (IllegalArgumentException ie) {
|
||||
BigInteger tmpA = BigInteger.valueOf(valueA);
|
||||
BigInteger tmpB = BigInteger.valueOf(valueB);
|
||||
|
|
|
@ -50,7 +50,7 @@ public class TestResourceInformation {
|
|||
@Test
|
||||
public void testValue() {
|
||||
String name = "yarn.io/test";
|
||||
Long value = 1l;
|
||||
long value = 1L;
|
||||
ResourceInformation ri = ResourceInformation.newInstance(name, value);
|
||||
Assert.assertEquals("Resource name incorrect", name, ri.getName());
|
||||
Assert.assertEquals("Resource value incorrect", value, ri.getValue());
|
||||
|
@ -59,7 +59,7 @@ public class TestResourceInformation {
|
|||
@Test
|
||||
public void testResourceInformation() {
|
||||
String name = "yarn.io/test";
|
||||
Long value = 1l;
|
||||
long value = 1L;
|
||||
String units = "m";
|
||||
ResourceInformation ri =
|
||||
ResourceInformation.newInstance(name, units, value);
|
||||
|
|
|
@ -27,62 +27,62 @@ public class TestUnitsConversionUtil {
|
|||
public void testUnitsConversion() {
|
||||
int value = 5;
|
||||
String fromUnit = "";
|
||||
Long test = Long.valueOf(value);
|
||||
long test = value;
|
||||
Assert.assertEquals("pico test failed",
|
||||
Long.valueOf(value * 1000l * 1000l * 1000l * 1000l),
|
||||
value * 1000L * 1000L * 1000L * 1000L,
|
||||
UnitsConversionUtil.convert(fromUnit, "p", test));
|
||||
Assert.assertEquals("nano test failed",
|
||||
Long.valueOf(value * 1000l * 1000l * 1000l),
|
||||
value * 1000L * 1000L * 1000L,
|
||||
UnitsConversionUtil.convert(fromUnit, "n", test));
|
||||
Assert
|
||||
.assertEquals("micro test failed", Long.valueOf(value * 1000l * 1000l),
|
||||
.assertEquals("micro test failed", value * 1000L * 1000L,
|
||||
UnitsConversionUtil.convert(fromUnit, "u", test));
|
||||
Assert.assertEquals("milli test failed", Long.valueOf(value * 1000l),
|
||||
Assert.assertEquals("milli test failed", value * 1000L,
|
||||
UnitsConversionUtil.convert(fromUnit, "m", test));
|
||||
|
||||
test = Long.valueOf(value * 1000l * 1000l * 1000l * 1000l * 1000l);
|
||||
test = value * 1000L * 1000L * 1000L * 1000L * 1000L;
|
||||
fromUnit = "";
|
||||
Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l),
|
||||
Assert.assertEquals("kilo test failed", test / 1000L,
|
||||
UnitsConversionUtil.convert(fromUnit, "k", test));
|
||||
|
||||
Assert
|
||||
.assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)),
|
||||
.assertEquals("mega test failed", test / (1000L * 1000L),
|
||||
UnitsConversionUtil.convert(fromUnit, "M", test));
|
||||
Assert.assertEquals("giga test failed",
|
||||
Long.valueOf(test / (1000l * 1000l * 1000l)),
|
||||
test / (1000L * 1000L * 1000L),
|
||||
UnitsConversionUtil.convert(fromUnit, "G", test));
|
||||
Assert.assertEquals("tera test failed",
|
||||
Long.valueOf(test / (1000l * 1000l * 1000l * 1000l)),
|
||||
test / (1000L * 1000L * 1000L * 1000L),
|
||||
UnitsConversionUtil.convert(fromUnit, "T", test));
|
||||
Assert.assertEquals("peta test failed",
|
||||
Long.valueOf(test / (1000l * 1000l * 1000l * 1000l * 1000l)),
|
||||
test / (1000L * 1000L * 1000L * 1000L * 1000L),
|
||||
UnitsConversionUtil.convert(fromUnit, "P", test));
|
||||
|
||||
Assert.assertEquals("nano to pico test failed", Long.valueOf(value * 1000l),
|
||||
UnitsConversionUtil.convert("n", "p", Long.valueOf(value)));
|
||||
Assert.assertEquals("nano to pico test failed", value * 1000L,
|
||||
UnitsConversionUtil.convert("n", "p", value));
|
||||
|
||||
Assert.assertEquals("mega to giga test failed", Long.valueOf(value),
|
||||
UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l)));
|
||||
Assert.assertEquals("mega to giga test failed", value,
|
||||
UnitsConversionUtil.convert("M", "G", value * 1000L));
|
||||
|
||||
Assert.assertEquals("Mi to Gi test failed", Long.valueOf(value),
|
||||
UnitsConversionUtil.convert("Mi", "Gi", Long.valueOf(value * 1024l)));
|
||||
Assert.assertEquals("Mi to Gi test failed", value,
|
||||
UnitsConversionUtil.convert("Mi", "Gi", value * 1024L));
|
||||
|
||||
Assert.assertEquals("Mi to Ki test failed", Long.valueOf(value * 1024),
|
||||
UnitsConversionUtil.convert("Mi", "Ki", Long.valueOf(value)));
|
||||
Assert.assertEquals("Mi to Ki test failed", value * 1024,
|
||||
UnitsConversionUtil.convert("Mi", "Ki", value));
|
||||
|
||||
Assert.assertEquals("Ki to base units test failed", Long.valueOf(5 * 1024),
|
||||
UnitsConversionUtil.convert("Ki", "", Long.valueOf(5)));
|
||||
Assert.assertEquals("Ki to base units test failed", 5 * 1024,
|
||||
UnitsConversionUtil.convert("Ki", "", 5));
|
||||
|
||||
Assert.assertEquals("Mi to k test failed", Long.valueOf(1073741),
|
||||
UnitsConversionUtil.convert("Mi", "k", Long.valueOf(1024)));
|
||||
Assert.assertEquals("Mi to k test failed", 1073741,
|
||||
UnitsConversionUtil.convert("Mi", "k", 1024));
|
||||
|
||||
Assert.assertEquals("M to Mi test failed", Long.valueOf(953),
|
||||
UnitsConversionUtil.convert("M", "Mi", Long.valueOf(1000)));
|
||||
Assert.assertEquals("M to Mi test failed", 953,
|
||||
UnitsConversionUtil.convert("M", "Mi", 1000));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOverflow() {
|
||||
Long test = Long.valueOf(5 * 1000l * 1000l * 1000l * 1000l * 1000l);
|
||||
long test = 5 * 1000L * 1000L * 1000L * 1000L * 1000L;
|
||||
try {
|
||||
UnitsConversionUtil.convert("P", "p", test);
|
||||
Assert.fail("this operation should result in an overflow");
|
||||
|
@ -100,9 +100,9 @@ public class TestUnitsConversionUtil {
|
|||
@Test
|
||||
public void testCompare() {
|
||||
String unitA = "P";
|
||||
Long valueA = Long.valueOf(1);
|
||||
long valueA = 1;
|
||||
String unitB = "p";
|
||||
Long valueB = Long.valueOf(2);
|
||||
long valueB = 2;
|
||||
Assert.assertEquals(1,
|
||||
UnitsConversionUtil.compare(unitA, valueA, unitB, valueB));
|
||||
Assert.assertEquals(-1,
|
||||
|
@ -120,7 +120,7 @@ public class TestUnitsConversionUtil {
|
|||
Assert.assertEquals(-1,
|
||||
UnitsConversionUtil.compare(unitB, valueB, unitA, valueA));
|
||||
Assert.assertEquals(0,
|
||||
UnitsConversionUtil.compare(unitA, valueA, unitB, 1000l));
|
||||
UnitsConversionUtil.compare(unitA, valueA, unitB, 1000L));
|
||||
|
||||
unitA = "p";
|
||||
unitB = "n";
|
||||
|
@ -129,7 +129,7 @@ public class TestUnitsConversionUtil {
|
|||
Assert.assertEquals(1,
|
||||
UnitsConversionUtil.compare(unitB, valueB, unitA, valueA));
|
||||
Assert.assertEquals(0,
|
||||
UnitsConversionUtil.compare(unitA, 1000l, unitB, valueA));
|
||||
UnitsConversionUtil.compare(unitA, 1000L, unitB, valueA));
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
package org.apache.hadoop.yarn.api.records.impl.pb;
|
||||
|
||||
import org.apache.commons.collections.map.UnmodifiableMap;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
|
@ -46,6 +47,7 @@ public class ResourcePBImpl extends Resource {
|
|||
boolean viaProto = false;
|
||||
|
||||
private Map<String, ResourceInformation> resources;
|
||||
private Map<String, ResourceInformation> readOnlyResources;
|
||||
|
||||
|
||||
// call via ProtoUtils.convertToProtoFormat(Resource)
|
||||
|
@ -68,6 +70,7 @@ public class ResourcePBImpl extends Resource {
|
|||
public ResourcePBImpl(ResourceProto proto) {
|
||||
this.proto = proto;
|
||||
viaProto = true;
|
||||
this.readOnlyResources = null;
|
||||
this.resources = null;
|
||||
initResources();
|
||||
}
|
||||
|
@ -111,10 +114,9 @@ public class ResourcePBImpl extends Resource {
|
|||
|
||||
@Override
|
||||
public void setMemorySize(long memory) {
|
||||
setResourceInformation(ResourceInformation.MEMORY_MB.getName(),
|
||||
ResourceInformation.newInstance(ResourceInformation.MEMORY_MB.getName(),
|
||||
ResourceInformation.MEMORY_MB.getUnits(), memory));
|
||||
|
||||
maybeInitBuilder();
|
||||
getResourceInformation(ResourceInformation.MEMORY_MB.getName())
|
||||
.setValue(memory);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -127,9 +129,9 @@ public class ResourcePBImpl extends Resource {
|
|||
|
||||
@Override
|
||||
public void setVirtualCores(int vCores) {
|
||||
setResourceInformation(ResourceInformation.VCORES.getName(),
|
||||
ResourceInformation.newInstance(ResourceInformation.VCORES.getName(),
|
||||
ResourceInformation.VCORES.getUnits(), (long) vCores));
|
||||
maybeInitBuilder();
|
||||
getResourceInformation(ResourceInformation.VCORES.getName())
|
||||
.setValue(vCores);
|
||||
}
|
||||
|
||||
private void initResources() {
|
||||
|
@ -143,7 +145,7 @@ public class ResourcePBImpl extends Resource {
|
|||
entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) :
|
||||
ResourceTypes.COUNTABLE;
|
||||
String units = entry.hasUnits() ? entry.getUnits() : "";
|
||||
Long value = entry.hasValue() ? entry.getValue() : 0L;
|
||||
long value = entry.hasValue() ? entry.getValue() : 0L;
|
||||
ResourceInformation ri = ResourceInformation
|
||||
.newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE);
|
||||
if (resources.containsKey(ri.getName())) {
|
||||
|
@ -171,7 +173,7 @@ public class ResourcePBImpl extends Resource {
|
|||
}
|
||||
initResources();
|
||||
if (resources.containsKey(resource)) {
|
||||
resources.put(resource, resourceInformation);
|
||||
ResourceInformation.copy(resourceInformation, resources.get(resource));
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -193,7 +195,7 @@ public class ResourcePBImpl extends Resource {
|
|||
@Override
|
||||
public Map<String, ResourceInformation> getResources() {
|
||||
initResources();
|
||||
return Collections.unmodifiableMap(this.resources);
|
||||
return readOnlyResources;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -226,6 +228,7 @@ public class ResourcePBImpl extends Resource {
|
|||
resources.put(entry.getKey(),
|
||||
ResourceInformation.newInstance(entry.getValue()));
|
||||
}
|
||||
readOnlyResources = Collections.unmodifiableMap(resources);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -154,7 +154,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
clusterResource.getResourceInformation(rName);
|
||||
ResourceInformation resourceInformation =
|
||||
resource.getResourceInformation(rName);
|
||||
Long resourceValue = UnitsConversionUtil
|
||||
long resourceValue = UnitsConversionUtil
|
||||
.convert(resourceInformation.getUnits(),
|
||||
clusterResourceResourceInformation.getUnits(),
|
||||
resourceInformation.getValue());
|
||||
|
@ -180,11 +180,11 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
available.getResourceInformation(resource);
|
||||
ResourceInformation requiredResource =
|
||||
required.getResourceInformation(resource);
|
||||
Long requiredResourceValue = UnitsConversionUtil
|
||||
long requiredResourceValue = UnitsConversionUtil
|
||||
.convert(requiredResource.getUnits(), availableResource.getUnits(),
|
||||
requiredResource.getValue());
|
||||
if (requiredResourceValue != 0) {
|
||||
Long tmp = availableResource.getValue() / requiredResourceValue;
|
||||
long tmp = availableResource.getValue() / requiredResourceValue;
|
||||
min = min < tmp ? min : tmp;
|
||||
}
|
||||
} catch (YarnException ye) {
|
||||
|
@ -228,7 +228,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
a.getResourceInformation(resource);
|
||||
ResourceInformation bResourceInformation =
|
||||
b.getResourceInformation(resource);
|
||||
Long bResourceValue = UnitsConversionUtil
|
||||
long bResourceValue = UnitsConversionUtil
|
||||
.convert(bResourceInformation.getUnits(),
|
||||
aResourceInformation.getUnits(),
|
||||
bResourceInformation.getValue());
|
||||
|
@ -249,14 +249,13 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
}
|
||||
|
||||
public Resource divideAndCeil(Resource numerator, long denominator) {
|
||||
Resource ret = Resources.createResource(0, 0);
|
||||
Resource ret = Resource.newInstance(numerator);
|
||||
for (String resource : resourceNames) {
|
||||
try {
|
||||
ResourceInformation resourceInformation = ResourceInformation
|
||||
.newInstance(numerator.getResourceInformation(resource));
|
||||
ResourceInformation resourceInformation =
|
||||
ret.getResourceInformation(resource);
|
||||
resourceInformation.setValue(
|
||||
divideAndCeil(resourceInformation.getValue(), denominator));
|
||||
ret.setResourceInformation(resource, resourceInformation);
|
||||
} catch (YarnException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + resource, ye);
|
||||
|
@ -276,7 +275,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
@Override
|
||||
public Resource normalize(Resource r, Resource minimumResource,
|
||||
Resource maximumResource, Resource stepFactor) {
|
||||
Resource ret = Resources.createResource(0, 0);
|
||||
Resource ret = Resource.newInstance(r);
|
||||
for (String resource : resourceNames) {
|
||||
try {
|
||||
ResourceInformation rResourceInformation =
|
||||
|
@ -287,28 +286,26 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
maximumResource.getResourceInformation(resource);
|
||||
ResourceInformation stepFactorResourceInformation =
|
||||
stepFactor.getResourceInformation(resource);
|
||||
ResourceInformation tmp =
|
||||
ResourceInformation.newInstance(rResourceInformation);
|
||||
ResourceInformation tmp = ret.getResourceInformation(resource);
|
||||
|
||||
Long rValue = rResourceInformation.getValue();
|
||||
Long minimumValue = UnitsConversionUtil
|
||||
long rValue = rResourceInformation.getValue();
|
||||
long minimumValue = UnitsConversionUtil
|
||||
.convert(minimumResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
minimumResourceInformation.getValue());
|
||||
Long maximumValue = UnitsConversionUtil
|
||||
long maximumValue = UnitsConversionUtil
|
||||
.convert(maximumResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
maximumResourceInformation.getValue());
|
||||
Long stepFactorValue = UnitsConversionUtil
|
||||
long stepFactorValue = UnitsConversionUtil
|
||||
.convert(stepFactorResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
stepFactorResourceInformation.getValue());
|
||||
Long value = Math.max(rValue, minimumValue);
|
||||
long value = Math.max(rValue, minimumValue);
|
||||
if (stepFactorValue != 0) {
|
||||
value = roundUp(value, stepFactorValue);
|
||||
}
|
||||
tmp.setValue(Math.min(value, maximumValue));
|
||||
ret.setResourceInformation(resource, tmp);
|
||||
} catch (YarnException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + resource, ye);
|
||||
|
@ -328,28 +325,27 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
}
|
||||
|
||||
private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) {
|
||||
Resource ret = Resources.createResource(0, 0);
|
||||
Resource ret = Resource.newInstance(r);
|
||||
for (String resource : resourceNames) {
|
||||
try {
|
||||
ResourceInformation rResourceInformation =
|
||||
r.getResourceInformation(resource);
|
||||
ResourceInformation stepFactorResourceInformation =
|
||||
stepFactor.getResourceInformation(resource);
|
||||
ResourceInformation tmp =
|
||||
ResourceInformation.newInstance(rResourceInformation);
|
||||
|
||||
Long rValue = rResourceInformation.getValue();
|
||||
Long stepFactorValue = UnitsConversionUtil
|
||||
long rValue = rResourceInformation.getValue();
|
||||
long stepFactorValue = UnitsConversionUtil
|
||||
.convert(stepFactorResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
stepFactorResourceInformation.getValue());
|
||||
Long value = rValue;
|
||||
long value = rValue;
|
||||
if (stepFactorValue != 0) {
|
||||
value = roundUp ? roundUp(rValue, stepFactorValue) :
|
||||
roundDown(rValue, stepFactorValue);
|
||||
}
|
||||
tmp.setValue(value);
|
||||
ret.setResourceInformation(resource, tmp);
|
||||
ResourceInformation
|
||||
.copy(rResourceInformation, ret.getResourceInformation(resource));
|
||||
ret.getResourceInformation(resource).setValue(value);
|
||||
} catch (YarnException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + resource, ye);
|
||||
|
@ -372,15 +368,14 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
|
||||
private Resource multiplyAndNormalize(Resource r, double by,
|
||||
Resource stepFactor, boolean roundUp) {
|
||||
Resource ret = Resources.createResource(0, 0);
|
||||
Resource ret = Resource.newInstance(r);
|
||||
for (String resource : resourceNames) {
|
||||
try {
|
||||
ResourceInformation rResourceInformation =
|
||||
r.getResourceInformation(resource);
|
||||
ResourceInformation stepFactorResourceInformation =
|
||||
stepFactor.getResourceInformation(resource);
|
||||
ResourceInformation tmp =
|
||||
ResourceInformation.newInstance(rResourceInformation);
|
||||
ResourceInformation tmp = ret.getResourceInformation(resource);
|
||||
|
||||
Long rValue = rResourceInformation.getValue();
|
||||
Long stepFactorValue = UnitsConversionUtil
|
||||
|
@ -397,7 +392,6 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by);
|
||||
}
|
||||
tmp.setValue(value);
|
||||
ret.setResourceInformation(resource, tmp);
|
||||
} catch (YarnException ye) {
|
||||
throw new IllegalArgumentException(
|
||||
"Error getting resource information for " + resource, ye);
|
||||
|
@ -414,7 +408,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
|
|||
smaller.getResourceInformation(resource);
|
||||
ResourceInformation bResourceInformation =
|
||||
bigger.getResourceInformation(resource);
|
||||
Long sResourceValue = UnitsConversionUtil
|
||||
long sResourceValue = UnitsConversionUtil
|
||||
.convert(sResourceInformation.getUnits(),
|
||||
bResourceInformation.getUnits(),
|
||||
sResourceInformation.getValue());
|
||||
|
|
|
@ -42,7 +42,7 @@ public class Resources {
|
|||
static class FixedValueResource extends Resource {
|
||||
|
||||
private Map<String, ResourceInformation> resources;
|
||||
private Long resourceValue;
|
||||
private long resourceValue;
|
||||
private String name;
|
||||
|
||||
/**
|
||||
|
@ -50,7 +50,7 @@ public class Resources {
|
|||
* @param rName the name of the resource
|
||||
* @param value the fixed value to be returned for all resource types
|
||||
*/
|
||||
FixedValueResource(String rName, Long value) {
|
||||
FixedValueResource(String rName, long value) {
|
||||
this.resourceValue = value;
|
||||
this.name = rName;
|
||||
resources = initResourceMap();
|
||||
|
@ -60,7 +60,7 @@ public class Resources {
|
|||
if(this.resourceValue > Integer.MAX_VALUE) {
|
||||
return Integer.MAX_VALUE;
|
||||
}
|
||||
return this.resourceValue.intValue();
|
||||
return Long.valueOf(this.resourceValue).intValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -203,7 +203,7 @@ public class Resources {
|
|||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
Long convertedRhs = UnitsConversionUtil
|
||||
long convertedRhs = UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue());
|
||||
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
|
||||
|
@ -225,7 +225,7 @@ public class Resources {
|
|||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
Long convertedRhs = UnitsConversionUtil
|
||||
long convertedRhs = UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue());
|
||||
lhs.setResourceValue(name, lhsValue.getValue() - convertedRhs);
|
||||
|
@ -288,7 +288,7 @@ public class Resources {
|
|||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
Long convertedRhs = (long) (UnitsConversionUtil
|
||||
long convertedRhs = (long) (UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue()) * by);
|
||||
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
|
||||
|
@ -422,7 +422,7 @@ public class Resources {
|
|||
try {
|
||||
ResourceInformation rhsValue = bigger.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
Long convertedRhs = UnitsConversionUtil
|
||||
long convertedRhs = UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue());
|
||||
if(lhsValue.getValue() > convertedRhs) {
|
||||
|
@ -448,7 +448,7 @@ public class Resources {
|
|||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
Long convertedRhs = UnitsConversionUtil
|
||||
long convertedRhs = UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue());
|
||||
ResourceInformation outInfo =
|
||||
|
@ -469,7 +469,7 @@ public class Resources {
|
|||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(name);
|
||||
ResourceInformation lhsValue = entry.getValue();
|
||||
Long convertedRhs = UnitsConversionUtil
|
||||
long convertedRhs = UnitsConversionUtil
|
||||
.convert(rhsValue.getUnits(), lhsValue.getUnits(),
|
||||
rhsValue.getValue());
|
||||
ResourceInformation outInfo =
|
||||
|
|
|
@ -283,6 +283,7 @@ public class TestResourceUtils {
|
|||
Map<String, ResourceInformation> actual =
|
||||
ResourceUtils.getNodeResourceInformation(conf);
|
||||
Assert.assertEquals(entry.getValue().getResources(), actual);
|
||||
dest.delete();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -42,6 +42,16 @@ import static org.junit.Assert.assertTrue;
|
|||
|
||||
public class TestResources {
|
||||
|
||||
static class ExtendedResources extends Resources {
|
||||
public static Resource unbounded() {
|
||||
return new FixedValueResource("UNBOUNDED", Long.MAX_VALUE);
|
||||
}
|
||||
|
||||
public static Resource none() {
|
||||
return new FixedValueResource("NONE", 0L);
|
||||
}
|
||||
}
|
||||
|
||||
private static final String EXTRA_RESOURCE_TYPE = "resource2";
|
||||
private String resourceTypesFile;
|
||||
|
||||
|
@ -87,7 +97,7 @@ public class TestResources {
|
|||
@Test(timeout = 10000)
|
||||
public void testCompareToWithUnboundedResource() {
|
||||
unsetExtraResourceType();
|
||||
Resource unboundedClone = Resources.clone(Resources.unbounded());
|
||||
Resource unboundedClone = Resources.clone(ExtendedResources.unbounded());
|
||||
assertTrue(unboundedClone
|
||||
.compareTo(createResource(Long.MAX_VALUE, Integer.MAX_VALUE)) == 0);
|
||||
assertTrue(unboundedClone.compareTo(createResource(Long.MAX_VALUE, 0)) > 0);
|
||||
|
|
|
@ -131,7 +131,8 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
|
|||
continue;
|
||||
}
|
||||
if (resourceName.equals(VCORES)) {
|
||||
resource.setVirtualCores(resourceValue.getValue().intValue());
|
||||
resource
|
||||
.setVirtualCores(Long.valueOf(resourceValue.getValue()).intValue());
|
||||
continue;
|
||||
}
|
||||
if (resourceTypes.containsKey(resourceName)) {
|
||||
|
|
Loading…
Reference in New Issue