diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java index 3c3c2cce2d1..198b395ab84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java @@ -57,14 +57,15 @@ public final class ResourceTypesTestHelper { Resource resource = RECORD_FACTORY.newRecordInstance(Resource.class); resource.setMemorySize(memory); resource.setVirtualCores(vCores); - - for (Map.Entry customResource : - customResources.entrySet()) { - String resourceName = customResource.getKey(); - ResourceInformation resourceInformation = - createResourceInformation(resourceName, - customResource.getValue()); - resource.setResourceInformation(resourceName, resourceInformation); + if (customResources != null) { + for (Map.Entry customResource : + customResources.entrySet()) { + String resourceName = customResource.getKey(); + ResourceInformation resourceInformation = + createResourceInformation(resourceName, + customResource.getValue()); + resource.setResourceInformation(resourceName, resourceInformation); + } } return resource; } @@ -107,7 +108,12 @@ public final class ResourceTypesTestHelper { public static Map extractCustomResourcesAsStrings( Resource res) { Map resValues = extractCustomResources(res); - return resValues.entrySet().stream() + return convertCustomResources(resValues); + } + + public static Map convertCustomResources( + Map customResources) { + return customResources.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, e -> String.valueOf(e.getValue()))); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java index bfe908d870b..45c5ac80df2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java @@ -82,7 +82,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair @@ -746,8 +745,8 @@ public class TestApplicationMasterService { MockRM rm = new MockRM(yarnConf); rm.start(); - MockNM nm1 = rm.registerNode("199.99.99.1:1234", TestUtils - .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + MockNM nm1 = rm.registerNode("199.99.99.1:1234", ResourceTypesTestHelper + .newResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, null)); RMApp app1 = rm.submitApp(GB, "app", "user", null, "default"); @@ -894,10 +893,10 @@ public class TestApplicationMasterService { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); LeafQueue leafQueue = (LeafQueue) cs.getQueue("default"); - MockNM nm1 = rm.registerNode("199.99.99.1:1234", TestUtils - .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + MockNM nm1 = rm.registerNode("199.99.99.1:1234", ResourceTypesTestHelper + .newResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - ImmutableMap.of("res_1", 4))); + ImmutableMap.of("res_1", "4"))); RMApp app1 = rm.submitApp(GB, "app", "user", null, "default"); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); @@ -909,8 +908,8 @@ public class TestApplicationMasterService { boolean exception = false; try { am1.allocate(Collections.singletonList(ResourceRequest.newBuilder() - .capability(TestUtils.createResource(9 * GB, 1, - ImmutableMap.of("res_1", 1))) + .capability(ResourceTypesTestHelper.newResource(9 * GB, 1, + ImmutableMap.of("res_1", "1"))) .numContainers(1) .resourceName("*") .build()), null); @@ -924,7 +923,8 @@ public class TestApplicationMasterService { // Now request resource, vcores > allowed am1.allocate(Collections.singletonList(ResourceRequest.newBuilder() .capability( - TestUtils.createResource(8 * GB, 18, ImmutableMap.of("res_1", 1))) + ResourceTypesTestHelper.newResource(8 * GB, 18, + ImmutableMap.of("res_1", "1"))) .numContainers(1) .resourceName("*") .build()), null); @@ -937,8 +937,8 @@ public class TestApplicationMasterService { try { // Now request resource, res_1 > allowed am1.allocate(Collections.singletonList(ResourceRequest.newBuilder() - .capability(TestUtils.createResource(8 * GB, 1, - ImmutableMap.of("res_1", 100))) + .capability(ResourceTypesTestHelper.newResource(8 * GB, 1, + ImmutableMap.of("res_1", "100"))) .numContainers(1) .resourceName("*") .build()), null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java index fa2b99354c3..de3673e7891 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; @@ -294,16 +295,16 @@ public class TestCapacitySchedulerWithMultiResourceTypes { MockRM rm = new MockRM(conf); rm.start(); - Map nameToValues = new HashMap<>(); - nameToValues.put(ResourceInformation.GPU_URI, 4); + Map nameToValues = new HashMap<>(); + nameToValues.put(ResourceInformation.GPU_URI, "4"); // Register NM1 with 10GB memory, 4 CPU and 4 GPU MockNM nm1 = rm.registerNode("127.0.0.1:1234", - TestUtils.createResource(10 * GB, 4, nameToValues)); + ResourceTypesTestHelper.newResource(10 * GB, 4, nameToValues)); nameToValues.clear(); // Register NM2 with 10GB memory, 4 CPU and 0 GPU rm.registerNode("127.0.0.1:1235", - TestUtils.createResource(10 * GB, 4, nameToValues)); + ResourceTypesTestHelper.newResource(10 * GB, 4, nameToValues)); RMApp app1 = rm.submitApp(1024, "app-1", "user1", null, "a"); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); @@ -320,9 +321,9 @@ public class TestCapacitySchedulerWithMultiResourceTypes { Assert.assertEquals(4, report_nm1.getAvailableResource() .getResourceInformation(ResourceInformation.GPU_URI).getValue()); - nameToValues.put(ResourceInformation.GPU_URI, 4); + nameToValues.put(ResourceInformation.GPU_URI, "4"); Resource containerGpuResource = - TestUtils.createResource(1 * GB, 1, nameToValues); + ResourceTypesTestHelper.newResource(1 * GB, 1, nameToValues); // Allocate one container which takes all 4 GPU am1.allocate( @@ -345,7 +346,7 @@ public class TestCapacitySchedulerWithMultiResourceTypes { nameToValues.clear(); Resource containerResource = - TestUtils.createResource(1 * GB, 1, nameToValues); + ResourceTypesTestHelper.newResource(1 * GB, 1, nameToValues); // Allocate one more container which doesnt need GPU am1.allocate( Collections.singletonList(ResourceRequest.newInstance( @@ -469,8 +470,8 @@ public class TestCapacitySchedulerWithMultiResourceTypes { for (int i = 0; i < 8; i++) { fiCaApp1.updateResourceRequests(Collections.singletonList( ResourceRequest.newBuilder() - .capability(TestUtils.createResource(1 * GB, 1, - ImmutableMap.of("res_1", 10))) + .capability(ResourceTypesTestHelper.newResource(1 * GB, 1, + ImmutableMap.of("res_1", "10"))) .numContainers(1) .resourceName("*") .build())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index c00b92d3251..418f85983a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -466,23 +466,4 @@ public class TestUtils { cs.submitResourceCommitRequest(clusterResource, csAssignment); } - - /** - * An easy way to create resources other than memory and vcores for tests. - * @param memory memory - * @param vcores vcores - * @param nameToValues resource types other than memory and vcores. - * @return created resource - */ - public static Resource createResource(long memory, int vcores, - Map nameToValues) { - Resource res = Resource.newInstance(memory, vcores); - if (nameToValues != null) { - for (Map.Entry entry : nameToValues.entrySet()) { - res.setResourceInformation(entry.getKey(), ResourceInformation - .newInstance(entry.getKey(), "", entry.getValue())); - } - } - return res; - } }