YARN-4829. Add support for binary units in Resource class.(vvasudev via asuresh)
This commit is contained in:
parent
ceb12c59a3
commit
7ba698997b
|
@ -36,7 +36,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
|
|||
private static final String VCORES_URI = "vcores";
|
||||
|
||||
public static final ResourceInformation MEMORY_MB =
|
||||
ResourceInformation.newInstance(MEMORY_URI, "M");
|
||||
ResourceInformation.newInstance(MEMORY_URI, "Mi");
|
||||
public static final ResourceInformation VCORES =
|
||||
ResourceInformation.newInstance(VCORES_URI);
|
||||
|
||||
|
|
|
@ -46,7 +46,8 @@ public class UnitsConversionUtil {
|
|||
}
|
||||
|
||||
private static final String[] UNITS =
|
||||
{"p", "n", "u", "m", "", "k", "M", "G", "T", "P"};
|
||||
{ "p", "n", "u", "m", "", "k", "M", "G", "T", "P", "Ki", "Mi", "Gi", "Ti",
|
||||
"Pi" };
|
||||
private static final List<String> SORTED_UNITS = Arrays.asList(UNITS);
|
||||
public static final Set<String> KNOWN_UNITS = createKnownUnitsSet();
|
||||
private static final Converter PICO =
|
||||
|
@ -65,6 +66,15 @@ public class UnitsConversionUtil {
|
|||
private static final Converter PETA =
|
||||
new Converter(1000L * 1000L * 1000L * 1000L * 1000L, 1L);
|
||||
|
||||
private static final Converter KILO_BINARY = new Converter(1024L, 1L);
|
||||
private static final Converter MEGA_BINARY = new Converter(1024L * 1024L, 1L);
|
||||
private static final Converter GIGA_BINARY =
|
||||
new Converter(1024L * 1024L * 1024L, 1L);
|
||||
private static final Converter TERA_BINARY =
|
||||
new Converter(1024L * 1024L * 1024L * 1024L, 1L);
|
||||
private static final Converter PETA_BINARY =
|
||||
new Converter(1024L * 1024L * 1024L * 1024L * 1024L, 1L);
|
||||
|
||||
private static Set<String> createKnownUnitsSet() {
|
||||
Set<String> ret = new HashSet<>();
|
||||
ret.addAll(Arrays.asList(UNITS));
|
||||
|
@ -93,6 +103,16 @@ public class UnitsConversionUtil {
|
|||
return TERA;
|
||||
case "P":
|
||||
return PETA;
|
||||
case "Ki":
|
||||
return KILO_BINARY;
|
||||
case "Mi":
|
||||
return MEGA_BINARY;
|
||||
case "Gi":
|
||||
return GIGA_BINARY;
|
||||
case "Ti":
|
||||
return TERA_BINARY;
|
||||
case "Pi":
|
||||
return PETA_BINARY;
|
||||
default:
|
||||
throw new IllegalArgumentException(
|
||||
"Unknown unit '" + unit + "'. Known units are " + KNOWN_UNITS);
|
||||
|
@ -112,28 +132,29 @@ public class UnitsConversionUtil {
|
|||
if (toUnit == null || fromUnit == null || fromValue == null) {
|
||||
throw new IllegalArgumentException("One or more arguments are null");
|
||||
}
|
||||
Long tmp;
|
||||
String overflowMsg =
|
||||
"Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit
|
||||
+ "' will result in an overflow of Long";
|
||||
if (fromUnit.equals(toUnit)) {
|
||||
return fromValue;
|
||||
}
|
||||
Converter fc = getConverter(fromUnit);
|
||||
Converter tc = getConverter(toUnit);
|
||||
Long numerator = fc.numerator * tc.denominator;
|
||||
Long denominator = fc.denominator * tc.numerator;
|
||||
Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator;
|
||||
if (numerator < denominator) {
|
||||
if (!toUnit.equals(fromUnit)) {
|
||||
tmp = Long.MAX_VALUE / numerator;
|
||||
if (tmp < fromValue) {
|
||||
throw new IllegalArgumentException(overflowMsg);
|
||||
}
|
||||
if (numeratorMultiplierLimit < fromValue) {
|
||||
throw new IllegalArgumentException(overflowMsg);
|
||||
}
|
||||
return (fromValue * numerator) / denominator;
|
||||
}
|
||||
tmp = numerator / denominator;
|
||||
if (!toUnit.equals(fromUnit)) {
|
||||
if ((Long.MAX_VALUE / tmp) < fromValue) {
|
||||
throw new IllegalArgumentException(overflowMsg);
|
||||
}
|
||||
if (numeratorMultiplierLimit > fromValue) {
|
||||
return (numerator * fromValue) / denominator;
|
||||
}
|
||||
Long tmp = numerator / denominator;
|
||||
if ((Long.MAX_VALUE / tmp) < fromValue) {
|
||||
throw new IllegalArgumentException(overflowMsg);
|
||||
}
|
||||
return fromValue * tmp;
|
||||
}
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
|
||||
package org.apache.hadoop.yarn.util;
|
||||
|
||||
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
|
@ -45,6 +44,7 @@ public class TestUnitsConversionUtil {
|
|||
fromUnit = "";
|
||||
Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l),
|
||||
UnitsConversionUtil.convert(fromUnit, "k", test));
|
||||
|
||||
Assert
|
||||
.assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)),
|
||||
UnitsConversionUtil.convert(fromUnit, "M", test));
|
||||
|
@ -63,6 +63,21 @@ public class TestUnitsConversionUtil {
|
|||
|
||||
Assert.assertEquals("mega to giga test failed", Long.valueOf(value),
|
||||
UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l)));
|
||||
|
||||
Assert.assertEquals("Mi to Gi test failed", Long.valueOf(value),
|
||||
UnitsConversionUtil.convert("Mi", "Gi", Long.valueOf(value * 1024l)));
|
||||
|
||||
Assert.assertEquals("Mi to Ki test failed", Long.valueOf(value * 1024),
|
||||
UnitsConversionUtil.convert("Mi", "Ki", Long.valueOf(value)));
|
||||
|
||||
Assert.assertEquals("Ki to base units test failed", Long.valueOf(5 * 1024),
|
||||
UnitsConversionUtil.convert("Ki", "", Long.valueOf(5)));
|
||||
|
||||
Assert.assertEquals("Mi to k test failed", Long.valueOf(1073741),
|
||||
UnitsConversionUtil.convert("Mi", "k", Long.valueOf(1024)));
|
||||
|
||||
Assert.assertEquals("M to Mi test failed", Long.valueOf(953),
|
||||
UnitsConversionUtil.convert("M", "Mi", Long.valueOf(1000)));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
@ -95,7 +95,7 @@ public class ResourcePBImpl extends Resource {
|
|||
initResourcesMap();
|
||||
ResourceInformation ri =
|
||||
this.getResourceInformation(ResourceInformation.MEMORY_MB.getName());
|
||||
return UnitsConversionUtil.convert(ri.getUnits(), "M", ri.getValue());
|
||||
return UnitsConversionUtil.convert(ri.getUnits(), "Mi", ri.getValue());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -195,7 +195,7 @@ public class TestResourceUtils {
|
|||
// we must always have memory and vcores with their fixed units
|
||||
Assert.assertTrue(ret.containsKey("memory-mb"));
|
||||
ResourceInformation memInfo = ret.get("memory-mb");
|
||||
Assert.assertEquals("M", memInfo.getUnits());
|
||||
Assert.assertEquals("Mi", memInfo.getUnits());
|
||||
Assert.assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType());
|
||||
Assert.assertTrue(ret.containsKey("vcores"));
|
||||
ResourceInformation vcoresInfo = ret.get("vcores");
|
||||
|
|
Loading…
Reference in New Issue