diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 80e3192537e..a17e81b5348 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -36,7 +36,7 @@ public class ResourceInformation implements Comparable { private static final String VCORES_URI = "vcores"; public static final ResourceInformation MEMORY_MB = - ResourceInformation.newInstance(MEMORY_URI, "M"); + ResourceInformation.newInstance(MEMORY_URI, "Mi"); public static final ResourceInformation VCORES = ResourceInformation.newInstance(VCORES_URI); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java index 77852633f3f..47bb3df2fed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java @@ -46,7 +46,8 @@ public static class Converter { } private static final String[] UNITS = - {"p", "n", "u", "m", "", "k", "M", "G", "T", "P"}; + { "p", "n", "u", "m", "", "k", "M", "G", "T", "P", "Ki", "Mi", "Gi", "Ti", + "Pi" }; private static final List SORTED_UNITS = Arrays.asList(UNITS); public static final Set KNOWN_UNITS = createKnownUnitsSet(); private static final Converter PICO = @@ -65,6 +66,15 @@ public static class Converter { private static final Converter PETA = new Converter(1000L * 1000L * 1000L * 1000L * 1000L, 1L); + private static final Converter KILO_BINARY = new Converter(1024L, 1L); + private static final Converter MEGA_BINARY = new Converter(1024L * 1024L, 1L); + private static final Converter GIGA_BINARY = + new Converter(1024L * 1024L * 1024L, 1L); + private static final Converter TERA_BINARY = + new Converter(1024L * 1024L * 1024L * 1024L, 1L); + private static final Converter PETA_BINARY = + new Converter(1024L * 1024L * 1024L * 1024L * 1024L, 1L); + private static Set createKnownUnitsSet() { Set ret = new HashSet<>(); ret.addAll(Arrays.asList(UNITS)); @@ -93,6 +103,16 @@ private static Converter getConverter(String unit) { return TERA; case "P": return PETA; + case "Ki": + return KILO_BINARY; + case "Mi": + return MEGA_BINARY; + case "Gi": + return GIGA_BINARY; + case "Ti": + return TERA_BINARY; + case "Pi": + return PETA_BINARY; default: throw new IllegalArgumentException( "Unknown unit '" + unit + "'. Known units are " + KNOWN_UNITS); @@ -112,28 +132,29 @@ public static Long convert(String fromUnit, String toUnit, Long fromValue) { if (toUnit == null || fromUnit == null || fromValue == null) { throw new IllegalArgumentException("One or more arguments are null"); } - Long tmp; String overflowMsg = "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit + "' will result in an overflow of Long"; + if (fromUnit.equals(toUnit)) { + return fromValue; + } Converter fc = getConverter(fromUnit); Converter tc = getConverter(toUnit); Long numerator = fc.numerator * tc.denominator; Long denominator = fc.denominator * tc.numerator; + Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator; if (numerator < denominator) { - if (!toUnit.equals(fromUnit)) { - tmp = Long.MAX_VALUE / numerator; - if (tmp < fromValue) { - throw new IllegalArgumentException(overflowMsg); - } + if (numeratorMultiplierLimit < fromValue) { + throw new IllegalArgumentException(overflowMsg); } return (fromValue * numerator) / denominator; } - tmp = numerator / denominator; - if (!toUnit.equals(fromUnit)) { - if ((Long.MAX_VALUE / tmp) < fromValue) { - throw new IllegalArgumentException(overflowMsg); - } + if (numeratorMultiplierLimit > fromValue) { + return (numerator * fromValue) / denominator; + } + Long tmp = numerator / denominator; + if ((Long.MAX_VALUE / tmp) < fromValue) { + throw new IllegalArgumentException(overflowMsg); } return fromValue * tmp; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java index 421768f83c9..0f999e8a76f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.util; -import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.junit.Assert; import org.junit.Test; @@ -45,6 +44,7 @@ public void testUnitsConversion() { fromUnit = ""; Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l), UnitsConversionUtil.convert(fromUnit, "k", test)); + Assert .assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)), UnitsConversionUtil.convert(fromUnit, "M", test)); @@ -63,6 +63,21 @@ public void testUnitsConversion() { Assert.assertEquals("mega to giga test failed", Long.valueOf(value), UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l))); + + Assert.assertEquals("Mi to Gi test failed", Long.valueOf(value), + UnitsConversionUtil.convert("Mi", "Gi", Long.valueOf(value * 1024l))); + + Assert.assertEquals("Mi to Ki test failed", Long.valueOf(value * 1024), + UnitsConversionUtil.convert("Mi", "Ki", Long.valueOf(value))); + + Assert.assertEquals("Ki to base units test failed", Long.valueOf(5 * 1024), + UnitsConversionUtil.convert("Ki", "", Long.valueOf(5))); + + Assert.assertEquals("Mi to k test failed", Long.valueOf(1073741), + UnitsConversionUtil.convert("Mi", "k", Long.valueOf(1024))); + + Assert.assertEquals("M to Mi test failed", Long.valueOf(953), + UnitsConversionUtil.convert("M", "Mi", Long.valueOf(1000))); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index ad1839a6dd5..0b93442d7b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -95,7 +95,7 @@ public long getMemorySize() { initResourcesMap(); ResourceInformation ri = this.getResourceInformation(ResourceInformation.MEMORY_MB.getName()); - return UnitsConversionUtil.convert(ri.getUnits(), "M", ri.getValue()); + return UnitsConversionUtil.convert(ri.getUnits(), "Mi", ri.getValue()); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 606e89cf2a2..4bb68c002a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -195,7 +195,7 @@ public void testInitializeResourcesMap() throws Exception { // we must always have memory and vcores with their fixed units Assert.assertTrue(ret.containsKey("memory-mb")); ResourceInformation memInfo = ret.get("memory-mb"); - Assert.assertEquals("M", memInfo.getUnits()); + Assert.assertEquals("Mi", memInfo.getUnits()); Assert.assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType()); Assert.assertTrue(ret.containsKey("vcores")); ResourceInformation vcoresInfo = ret.get("vcores");