YARN-4829. Add support for binary units in Resource class.(vvasudev via asuresh)

This commit is contained in:
Arun Suresh 2016-03-17 23:50:22 -07:00 committed by Wangda Tan
parent ceb12c59a3
commit 7ba698997b
5 changed files with 52 additions and 16 deletions

View File

@ -36,7 +36,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
private static final String VCORES_URI = "vcores";
public static final ResourceInformation MEMORY_MB =
ResourceInformation.newInstance(MEMORY_URI, "M");
ResourceInformation.newInstance(MEMORY_URI, "Mi");
public static final ResourceInformation VCORES =
ResourceInformation.newInstance(VCORES_URI);

View File

@ -46,7 +46,8 @@ public class UnitsConversionUtil {
}
private static final String[] UNITS =
{"p", "n", "u", "m", "", "k", "M", "G", "T", "P"};
{ "p", "n", "u", "m", "", "k", "M", "G", "T", "P", "Ki", "Mi", "Gi", "Ti",
"Pi" };
private static final List<String> SORTED_UNITS = Arrays.asList(UNITS);
public static final Set<String> KNOWN_UNITS = createKnownUnitsSet();
private static final Converter PICO =
@ -65,6 +66,15 @@ public class UnitsConversionUtil {
private static final Converter PETA =
new Converter(1000L * 1000L * 1000L * 1000L * 1000L, 1L);
private static final Converter KILO_BINARY = new Converter(1024L, 1L);
private static final Converter MEGA_BINARY = new Converter(1024L * 1024L, 1L);
private static final Converter GIGA_BINARY =
new Converter(1024L * 1024L * 1024L, 1L);
private static final Converter TERA_BINARY =
new Converter(1024L * 1024L * 1024L * 1024L, 1L);
private static final Converter PETA_BINARY =
new Converter(1024L * 1024L * 1024L * 1024L * 1024L, 1L);
private static Set<String> createKnownUnitsSet() {
Set<String> ret = new HashSet<>();
ret.addAll(Arrays.asList(UNITS));
@ -93,6 +103,16 @@ public class UnitsConversionUtil {
return TERA;
case "P":
return PETA;
case "Ki":
return KILO_BINARY;
case "Mi":
return MEGA_BINARY;
case "Gi":
return GIGA_BINARY;
case "Ti":
return TERA_BINARY;
case "Pi":
return PETA_BINARY;
default:
throw new IllegalArgumentException(
"Unknown unit '" + unit + "'. Known units are " + KNOWN_UNITS);
@ -112,28 +132,29 @@ public class UnitsConversionUtil {
if (toUnit == null || fromUnit == null || fromValue == null) {
throw new IllegalArgumentException("One or more arguments are null");
}
Long tmp;
String overflowMsg =
"Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit
+ "' will result in an overflow of Long";
if (fromUnit.equals(toUnit)) {
return fromValue;
}
Converter fc = getConverter(fromUnit);
Converter tc = getConverter(toUnit);
Long numerator = fc.numerator * tc.denominator;
Long denominator = fc.denominator * tc.numerator;
Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator;
if (numerator < denominator) {
if (!toUnit.equals(fromUnit)) {
tmp = Long.MAX_VALUE / numerator;
if (tmp < fromValue) {
throw new IllegalArgumentException(overflowMsg);
}
if (numeratorMultiplierLimit < fromValue) {
throw new IllegalArgumentException(overflowMsg);
}
return (fromValue * numerator) / denominator;
}
tmp = numerator / denominator;
if (!toUnit.equals(fromUnit)) {
if ((Long.MAX_VALUE / tmp) < fromValue) {
throw new IllegalArgumentException(overflowMsg);
}
if (numeratorMultiplierLimit > fromValue) {
return (numerator * fromValue) / denominator;
}
Long tmp = numerator / denominator;
if ((Long.MAX_VALUE / tmp) < fromValue) {
throw new IllegalArgumentException(overflowMsg);
}
return fromValue * tmp;
}

View File

@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.util;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.junit.Assert;
import org.junit.Test;
@ -45,6 +44,7 @@ public class TestUnitsConversionUtil {
fromUnit = "";
Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l),
UnitsConversionUtil.convert(fromUnit, "k", test));
Assert
.assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)),
UnitsConversionUtil.convert(fromUnit, "M", test));
@ -63,6 +63,21 @@ public class TestUnitsConversionUtil {
Assert.assertEquals("mega to giga test failed", Long.valueOf(value),
UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l)));
Assert.assertEquals("Mi to Gi test failed", Long.valueOf(value),
UnitsConversionUtil.convert("Mi", "Gi", Long.valueOf(value * 1024l)));
Assert.assertEquals("Mi to Ki test failed", Long.valueOf(value * 1024),
UnitsConversionUtil.convert("Mi", "Ki", Long.valueOf(value)));
Assert.assertEquals("Ki to base units test failed", Long.valueOf(5 * 1024),
UnitsConversionUtil.convert("Ki", "", Long.valueOf(5)));
Assert.assertEquals("Mi to k test failed", Long.valueOf(1073741),
UnitsConversionUtil.convert("Mi", "k", Long.valueOf(1024)));
Assert.assertEquals("M to Mi test failed", Long.valueOf(953),
UnitsConversionUtil.convert("M", "Mi", Long.valueOf(1000)));
}
@Test

View File

@ -95,7 +95,7 @@ public class ResourcePBImpl extends Resource {
initResourcesMap();
ResourceInformation ri =
this.getResourceInformation(ResourceInformation.MEMORY_MB.getName());
return UnitsConversionUtil.convert(ri.getUnits(), "M", ri.getValue());
return UnitsConversionUtil.convert(ri.getUnits(), "Mi", ri.getValue());
}
@Override

View File

@ -195,7 +195,7 @@ public class TestResourceUtils {
// we must always have memory and vcores with their fixed units
Assert.assertTrue(ret.containsKey("memory-mb"));
ResourceInformation memInfo = ret.get("memory-mb");
Assert.assertEquals("M", memInfo.getUnits());
Assert.assertEquals("Mi", memInfo.getUnits());
Assert.assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType());
Assert.assertTrue(ret.containsKey("vcores"));
ResourceInformation vcoresInfo = ret.get("vcores");