diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 904ff4b4083..c83c3a22fda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -202,7 +202,7 @@ public class ResourceInformation implements Comparable { ResourceInformation ret = new ResourceInformation(); ret.setName(name); ret.setResourceType(type); - ret.setUnits(units); + ret.setUnitsWithoutValidation(units); ret.setValue(value); ret.setMinimumAllocation(minimumAllocation); ret.setMaximumAllocation(maximumAllocation); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 3dbd609b534..c2d720147b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -527,6 +528,12 @@ public class ResourceUtils { String units = getUnits(value); Long resourceValue = Long.valueOf(value.substring(0, value.length() - units.length())); + String destUnit = getDefaultUnit(resourceType); + if(!units.equals(destUnit)) { + resourceValue = UnitsConversionUtil.convert( + units, destUnit, resourceValue); + units = destUnit; + } nodeResources.get(resourceType).setValue(resourceValue); nodeResources.get(resourceType).setUnits(units); if (LOG.isDebugEnabled()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java index 66bf3204bf6..c342dbe40a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java @@ -43,7 +43,7 @@ public class TestResourceInformation { Assert.assertEquals("Resource units incorrect", units, ri.getUnits()); units = "z"; try { - ResourceInformation.newInstance(name, units); + ResourceInformation.newInstance(name, units).setUnits(units); Assert.fail(units + "is not a valid unit"); } catch (IllegalArgumentException ie) { // do nothing diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 15d2470840c..144f48ff1d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -173,9 +173,17 @@ public class ResourcePBImpl extends Resource { ri.setResourceType(entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) : ResourceTypes.COUNTABLE); - ri.setUnits( - entry.hasUnits() ? entry.getUnits() : resourceInformation.getUnits()); - ri.setValue(entry.hasValue() ? entry.getValue() : 0L); + String units = entry.hasUnits() ? entry.getUnits() : + ResourceUtils.getDefaultUnit(entry.getKey()); + long value = entry.hasValue() ? entry.getValue() : 0L; + String destUnit = ResourceUtils.getDefaultUnit(entry.getKey()); + if(!units.equals(destUnit)) { + ri.setValue(UnitsConversionUtil.convert(units, destUnit, value)); + ri.setUnits(destUnit); + } else { + ri.setUnits(units); + ri.setValue(value); + } return ri; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 2e85ebca860..9aeb51cc2cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -24,7 +24,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.util.UnitsConversionUtil; import java.io.PrintWriter; import java.io.StringWriter; @@ -298,11 +297,7 @@ public class DominantResourceCalculator extends ResourceCalculator { */ private double calculateShare(ResourceInformation clusterRes, ResourceInformation res) { - // Convert the resources' units into the cluster resource's units - long value = UnitsConversionUtil.convert(res.getUnits(), - clusterRes.getUnits(), res.getValue()); - - return (double) value / clusterRes.getValue(); + return (double) res.getValue() / clusterRes.getValue(); } /** @@ -340,11 +335,8 @@ public class DominantResourceCalculator extends ResourceCalculator { ResourceInformation availableResource = available .getResourceInformation(i); ResourceInformation requiredResource = required.getResourceInformation(i); - long requiredResourceValue = UnitsConversionUtil.convert( - requiredResource.getUnits(), availableResource.getUnits(), - requiredResource.getValue()); - if (requiredResourceValue != 0) { - long tmp = availableResource.getValue() / requiredResourceValue; + if (requiredResource.getValue() != 0) { + long tmp = availableResource.getValue() / requiredResource.getValue(); min = min < tmp ? min : tmp; } } @@ -387,11 +379,8 @@ public class DominantResourceCalculator extends ResourceCalculator { for (int i = 0; i < maxLength; i++) { ResourceInformation aResourceInformation = a.getResourceInformation(i); ResourceInformation bResourceInformation = b.getResourceInformation(i); - long bResourceValue = UnitsConversionUtil.convert( - bResourceInformation.getUnits(), aResourceInformation.getUnits(), - bResourceInformation.getValue()); float tmp = (float) aResourceInformation.getValue() - / (float) bResourceValue; + / (float) bResourceInformation.getValue(); ratio = ratio > tmp ? ratio : tmp; } return ratio; @@ -437,23 +426,11 @@ public class DominantResourceCalculator extends ResourceCalculator { ResourceInformation tmp = ret.getResourceInformation(i); long rValue = rResourceInformation.getValue(); - long minimumValue = UnitsConversionUtil.convert( - minimumResourceInformation.getUnits(), - rResourceInformation.getUnits(), - minimumResourceInformation.getValue()); - long maximumValue = UnitsConversionUtil.convert( - maximumResourceInformation.getUnits(), - rResourceInformation.getUnits(), - maximumResourceInformation.getValue()); - long stepFactorValue = UnitsConversionUtil.convert( - stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); - long value = Math.max(rValue, minimumValue); - if (stepFactorValue != 0) { - value = roundUp(value, stepFactorValue); + long value = Math.max(rValue, minimumResourceInformation.getValue()); + if (stepFactorResourceInformation.getValue() != 0) { + value = roundUp(value, stepFactorResourceInformation.getValue()); } - tmp.setValue(Math.min(value, maximumValue)); + tmp.setValue(Math.min(value, maximumResourceInformation.getValue())); ret.setResourceInformation(i, tmp); } return ret; @@ -478,10 +455,7 @@ public class DominantResourceCalculator extends ResourceCalculator { .getResourceInformation(i); long rValue = rResourceInformation.getValue(); - long stepFactorValue = UnitsConversionUtil.convert( - stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); + long stepFactorValue = stepFactorResourceInformation.getValue(); long value = rValue; if (stepFactorValue != 0) { value = roundUp @@ -506,10 +480,7 @@ public class DominantResourceCalculator extends ResourceCalculator { .getResourceInformation(i); long rValue = rResourceInformation.getValue(); - long stepFactorValue = UnitsConversionUtil.convert( - stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); + long stepFactorValue = stepFactorResourceInformation.getValue(); ret.setResourceValue(i, ResourceCalculator .roundUp((long) Math.ceil(rValue * by[i]), stepFactorValue)); } @@ -539,10 +510,7 @@ public class DominantResourceCalculator extends ResourceCalculator { ResourceInformation tmp = ret.getResourceInformation(i); long rValue = rResourceInformation.getValue(); - long stepFactorValue = UnitsConversionUtil.convert( - stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); + long stepFactorValue = stepFactorResourceInformation.getValue(); long value; if (stepFactorValue != 0) { value = roundUp @@ -566,10 +534,7 @@ public class DominantResourceCalculator extends ResourceCalculator { .getResourceInformation(i); ResourceInformation bResourceInformation = bigger .getResourceInformation(i); - long sResourceValue = UnitsConversionUtil.convert( - sResourceInformation.getUnits(), bResourceInformation.getUnits(), - sResourceInformation.getValue()); - if (sResourceValue > bResourceInformation.getValue()) { + if (sResourceInformation.getValue() > bResourceInformation.getValue()) { return false; } } @@ -587,11 +552,7 @@ public class DominantResourceCalculator extends ResourceCalculator { ResourceInformation tmp = ret.getResourceInformation(i); long rValue = rResourceInformation.getValue(); - long stepFactorValue = UnitsConversionUtil.convert( - stepFactorResourceInformation.getUnits(), - rResourceInformation.getUnits(), - stepFactorResourceInformation.getValue()); - + long stepFactorValue = stepFactorResourceInformation.getValue(); long value = rValue; if (stepFactorValue != 0) { value = roundDown(rValue, stepFactorValue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 86365775f35..48c2c364ae9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.impl.LightWeightResource; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; -import org.apache.hadoop.yarn.util.UnitsConversionUtil; /** * Resources is a computation class which provides a set of apis to do @@ -257,12 +256,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); - - long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) - ? rhsValue.getValue() - : UnitsConversionUtil.convert(rhsValue.getUnits(), - lhsValue.getUnits(), rhsValue.getValue()); - lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs); + lhs.setResourceValue(i, lhsValue.getValue() + rhsValue.getValue()); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); continue; @@ -281,12 +275,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); - - long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) - ? rhsValue.getValue() - : UnitsConversionUtil.convert(rhsValue.getUnits(), - lhsValue.getUnits(), rhsValue.getValue()); - lhs.setResourceValue(i, lhsValue.getValue() - convertedRhs); + lhs.setResourceValue(i, lhsValue.getValue() - rhsValue.getValue()); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); continue; @@ -365,12 +354,7 @@ public class Resources { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); - long convertedRhs = (long) (((rhsValue.getUnits() - .equals(lhsValue.getUnits())) - ? rhsValue.getValue() - : UnitsConversionUtil.convert(rhsValue.getUnits(), - lhsValue.getUnits(), rhsValue.getValue())) - * by); + long convertedRhs = (long) (rhsValue.getValue() * by); lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); @@ -511,12 +495,7 @@ public class Resources { try { ResourceInformation rhsValue = bigger.getResourceInformation(i); ResourceInformation lhsValue = smaller.getResourceInformation(i); - - long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) - ? rhsValue.getValue() - : UnitsConversionUtil.convert(rhsValue.getUnits(), - lhsValue.getUnits(), rhsValue.getValue()); - if (lhsValue.getValue() > convertedRhs) { + if (lhsValue.getValue() > rhsValue.getValue()) { return false; } } catch (ResourceNotFoundException ye) { @@ -539,12 +518,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); - - long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) - ? rhsValue.getValue() - : UnitsConversionUtil.convert(rhsValue.getUnits(), - lhsValue.getUnits(), rhsValue.getValue()); - ResourceInformation outInfo = lhsValue.getValue() < convertedRhs + ResourceInformation outInfo = lhsValue.getValue() < rhsValue.getValue() ? lhsValue : rhsValue; ret.setResourceInformation(i, outInfo); @@ -563,12 +537,7 @@ public class Resources { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); - - long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) - ? rhsValue.getValue() - : UnitsConversionUtil.convert(rhsValue.getUnits(), - lhsValue.getUnits(), rhsValue.getValue()); - ResourceInformation outInfo = lhsValue.getValue() > convertedRhs + ResourceInformation outInfo = lhsValue.getValue() > rhsValue.getValue() ? lhsValue : rhsValue; ret.setResourceInformation(i, outInfo); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java index 4887b50e2c2..5ab528bdd8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java @@ -18,11 +18,19 @@ package org.apache.hadoop.yarn.api; +import java.io.File; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.proto.YarnProtos; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.resource.TestResourceUtils; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -31,6 +39,27 @@ import static org.junit.Assert.assertEquals; * Test class to handle various proto related tests for resources. */ public class TestResourcePBImpl { + + @Before + public void setup() throws Exception { + ResourceUtils.resetResourceTypes(); + + String resourceTypesFile = "resource-types-4.xml"; + Configuration conf = new YarnConfiguration(); + TestResourceUtils.setupResourceTypes(conf, resourceTypesFile); + } + + @After + public void teardown() { + Configuration conf = new YarnConfiguration(); + File source = new File( + conf.getClassLoader().getResource("resource-types-4.xml").getFile()); + File dest = new File(source.getParent(), "resource-types.xml"); + if (dest.exists()) { + dest.delete(); + } + } + @Test public void testEmptyResourcePBInit() throws Exception { Resource res = new ResourcePBImpl(); @@ -85,4 +114,65 @@ public class TestResourcePBImpl { assertEquals("Cast to Integer.MAX_VALUE if the long is greater than " + "Integer.MAX_VALUE", Integer.MAX_VALUE, res.getVirtualCores()); } + + @Test + public void testResourcePBWithExtraResources() throws Exception { + + //Resource 'resource1' has been passed as 4T + //4T should be converted to 4000G + YarnProtos.ResourceInformationProto riProto = + YarnProtos.ResourceInformationProto.newBuilder().setType( + YarnProtos.ResourceTypeInfoProto.newBuilder(). + setName("resource1").setType( + YarnProtos.ResourceTypesProto.COUNTABLE).getType()). + setValue(4).setUnits("T").setKey("resource1").build(); + + YarnProtos.ResourceProto proto = + YarnProtos.ResourceProto.newBuilder().setMemory(1024). + setVirtualCores(3).addResourceValueMap(riProto).build(); + Resource res = new ResourcePBImpl(proto); + + Assert.assertEquals(4000, + res.getResourceInformation("resource1").getValue()); + Assert.assertEquals("G", + res.getResourceInformation("resource1").getUnits()); + + //Resource 'resource2' has been passed as 4M + //4M should be converted to 4000000000m + YarnProtos.ResourceInformationProto riProto1 = + YarnProtos.ResourceInformationProto.newBuilder().setType( + YarnProtos.ResourceTypeInfoProto.newBuilder(). + setName("resource2").setType( + YarnProtos.ResourceTypesProto.COUNTABLE).getType()). + setValue(4).setUnits("M").setKey("resource2").build(); + + YarnProtos.ResourceProto proto1 = + YarnProtos.ResourceProto.newBuilder().setMemory(1024). + setVirtualCores(3).addResourceValueMap(riProto1).build(); + Resource res1 = new ResourcePBImpl(proto1); + + Assert.assertEquals(4000000000L, + res1.getResourceInformation("resource2").getValue()); + Assert.assertEquals("m", + res1.getResourceInformation("resource2").getUnits()); + + //Resource 'resource1' has been passed as 3M + //3M should be converted to 0G + YarnProtos.ResourceInformationProto riProto2 = + YarnProtos.ResourceInformationProto.newBuilder().setType( + YarnProtos.ResourceTypeInfoProto.newBuilder(). + setName("resource1").setType( + YarnProtos.ResourceTypesProto.COUNTABLE).getType()). + setValue(3).setUnits("M").setKey("resource1").build(); + + YarnProtos.ResourceProto proto2 = + YarnProtos.ResourceProto.newBuilder().setMemory(1024). + setVirtualCores(3).addResourceValueMap(riProto2).build(); + Resource res2 = new ResourcePBImpl(proto2); + + Assert.assertEquals(0, + res2.getResourceInformation("resource1").getValue()); + Assert.assertEquals("G", + res2.getResourceInformation("resource1").getUnits()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 2671de85057..9b480177075 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -377,6 +377,46 @@ public class TestResourceUtils { } } + @Test + public void testGetResourceInformationWithDiffUnits() throws Exception { + + Configuration conf = new YarnConfiguration(); + Map testRun = new HashMap<>(); + setupResourceTypes(conf, "resource-types-4.xml"); + Resource test3Resources = Resource.newInstance(0, 0); + + //Resource 'resource1' has been passed as 5T + //5T should be converted to 5000G + test3Resources.setResourceInformation("resource1", + ResourceInformation.newInstance("resource1", "T", 5L)); + + //Resource 'resource2' has been passed as 2M + //2M should be converted to 2000000000m + test3Resources.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "M", 2L)); + test3Resources.setResourceInformation("yarn.io/gpu", + ResourceInformation.newInstance("yarn.io/gpu", "", 1)); + testRun.put("node-resources-3.xml", test3Resources); + + for (Map.Entry entry : testRun.entrySet()) { + String resourceFile = entry.getKey(); + ResourceUtils.resetNodeResources(); + File dest; + File source = new File( + conf.getClassLoader().getResource(resourceFile).getFile()); + dest = new File(source.getParent(), "node-resources.xml"); + FileUtils.copyFile(source, dest); + Map actual = ResourceUtils + .getNodeResourceInformation(conf); + Assert.assertEquals(actual.size(), + entry.getValue().getResources().length); + for (ResourceInformation resInfo : entry.getValue().getResources()) { + Assert.assertEquals(resInfo, actual.get(resInfo.getName())); + } + dest.delete(); + } + } + public static String setupResourceTypes(Configuration conf, String filename) throws Exception { File source = new File( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-3.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-3.xml new file mode 100644 index 00000000000..23f6a6f5cdb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-3.xml @@ -0,0 +1,33 @@ + + + + + + + + yarn.nodemanager.resource-type.resource1 + 5T + + + + yarn.nodemanager.resource-type.resource2 + 2M + + + + yarn.nodemanager.resource-type.yarn.io/gpu + 1 + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index d66a8663c6b..6644e44453d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -33,6 +33,7 @@ import static org.mockito.Mockito.when; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; import java.security.AccessControlException; import java.security.PrivilegedExceptionAction; @@ -49,6 +50,7 @@ import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CyclicBarrier; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -157,7 +159,9 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.UTCClock; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; import org.junit.Assume; @@ -2243,4 +2247,88 @@ public class TestClientRMService { rmService.getApplications(request).getApplicationList().size()); rmService.setDisplayPerUserApps(false); } + + @Test + public void testRegisterNMWithDiffUnits() throws Exception { + ResourceUtils.resetResourceTypes(); + Configuration yarnConf = new YarnConfiguration(); + String resourceTypesFile = "resource-types-4.xml"; + InputStream source = + yarnConf.getClassLoader().getResourceAsStream(resourceTypesFile); + File dest = new File(yarnConf.getClassLoader(). + getResource(".").getPath(), "resource-types.xml"); + FileUtils.copyInputStreamToFile(source, dest); + ResourceUtils.getResourceTypes(); + + yarnConf.setClass( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class, ResourceCalculator.class); + + MockRM rm = new MockRM(yarnConf) { + protected ClientRMService createClientRMService() { + return new ClientRMService(this.rmContext, scheduler, + this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, + this.getRMContext().getRMDelegationTokenSecretManager()); + }; + }; + rm.start(); + + Resource resource = BuilderUtils.newResource(1024, 1); + resource.setResourceInformation("memory-mb", + ResourceInformation.newInstance("memory-mb", "G", 1024)); + resource.setResourceInformation("resource1", + ResourceInformation.newInstance("resource1", "T", 1)); + resource.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "M", 1)); + + MockNM node = rm.registerNode("host1:1234", resource); + node.nodeHeartbeat(true); + + // Create a client. + Configuration conf = new Configuration(); + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); + LOG.info("Connecting to ResourceManager at " + rmAddress); + ApplicationClientProtocol client = + (ApplicationClientProtocol) rpc + .getProxy(ApplicationClientProtocol.class, rmAddress, conf); + + // Make call + GetClusterNodesRequest request = + GetClusterNodesRequest.newInstance(EnumSet.of(NodeState.RUNNING)); + List nodeReports = + client.getClusterNodes(request).getNodeReports(); + Assert.assertEquals(1, nodeReports.size()); + Assert.assertNotSame("Node is expected to be healthy!", NodeState.UNHEALTHY, + nodeReports.get(0).getNodeState()); + Assert.assertEquals(1, nodeReports.size()); + + //Resource 'resource1' has been passed as 1T while registering NM. + //1T should be converted to 1000G + Assert.assertEquals("G", nodeReports.get(0).getCapability(). + getResourceInformation("resource1").getUnits()); + Assert.assertEquals(1000, nodeReports.get(0).getCapability(). + getResourceInformation("resource1").getValue()); + + //Resource 'resource2' has been passed as 1M while registering NM + //1M should be converted to 1000000000M + Assert.assertEquals("m", nodeReports.get(0).getCapability(). + getResourceInformation("resource2").getUnits()); + Assert.assertEquals(1000000000, nodeReports.get(0).getCapability(). + getResourceInformation("resource2").getValue()); + + //Resource 'memory-mb' has been passed as 1024G while registering NM + //1024G should be converted to 976562Mi + Assert.assertEquals("Mi", nodeReports.get(0).getCapability(). + getResourceInformation("memory-mb").getUnits()); + Assert.assertEquals(976562, nodeReports.get(0).getCapability(). + getResourceInformation("memory-mb").getValue()); + + rpc.stopProxy(client, conf); + rm.close(); + + if (dest.exists()) { + dest.delete(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index 70f83ab3095..69d3ab9ada0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -414,7 +414,7 @@ public class TestFairSchedulerConfiguration { calculator.normalize(customResource(10000L, ""), min, max, increment) .getResourceInformation(A_CUSTOM_RESOURCE)); assertEquals(customResourceInformation(20000L, ""), - calculator.normalize(customResource(10001L, ""), min, max, increment) + calculator.normalize(customResource(19999L, ""), min, max, increment) .getResourceInformation(A_CUSTOM_RESOURCE)); assertEquals(customResourceInformation(10L, "k"), calculator.normalize(customResource(9L, "k"), min, max, increment)