From cf9cf83a43be12a4325b02c2953c365309352649 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Tue, 10 Mar 2020 16:44:33 +0100 Subject: [PATCH] YARN-9354. Resources should be created with ResourceTypesTestHelper instead of TestUtils. Contributed by Andras Gyori --- .../ResourceTypesTestHelper.java | 24 ++++++++++++------- .../ApplicationMasterServiceTestBase.java | 22 ++++++++++------- ...pacitySchedulerWithMultiResourceTypes.java | 19 ++++++++------- .../scheduler/capacity/TestUtils.java | 19 --------------- 4 files changed, 39 insertions(+), 45 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java index 3c3c2cce2d1..198b395ab84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java @@ -57,14 +57,15 @@ public final class ResourceTypesTestHelper { Resource resource = RECORD_FACTORY.newRecordInstance(Resource.class); resource.setMemorySize(memory); resource.setVirtualCores(vCores); - - for (Map.Entry customResource : - customResources.entrySet()) { - String resourceName = customResource.getKey(); - ResourceInformation resourceInformation = - createResourceInformation(resourceName, - customResource.getValue()); - resource.setResourceInformation(resourceName, resourceInformation); + if (customResources != null) { + for (Map.Entry customResource : + customResources.entrySet()) { + String resourceName = customResource.getKey(); + ResourceInformation resourceInformation = + createResourceInformation(resourceName, + customResource.getValue()); + resource.setResourceInformation(resourceName, resourceInformation); + } } return resource; } @@ -107,7 +108,12 @@ public final class ResourceTypesTestHelper { public static Map extractCustomResourcesAsStrings( Resource res) { Map resValues = extractCustomResources(res); - return resValues.entrySet().stream() + return convertCustomResources(resValues); + } + + public static Map convertCustomResources( + Map customResources) { + return customResources.entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, e -> String.valueOf(e.getValue()))); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java index f13ae4ac06b..cf141181262 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.Event; +import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +47,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -112,8 +112,11 @@ public abstract class ApplicationMasterServiceTestBase { private void requestResources(MockAM am, long memory, int vCores, Map customResources) throws Exception { + Map convertedCustomResources = + ResourceTypesTestHelper.convertCustomResources(customResources); am.allocate(Collections.singletonList(ResourceRequest.newBuilder() - .capability(TestUtils.createResource(memory, vCores, customResources)) + .capability(ResourceTypesTestHelper.newResource( + memory, vCores, convertedCustomResources)) .numContainers(1) .resourceName("*") .build()), null); @@ -551,9 +554,11 @@ public abstract class ApplicationMasterServiceTestBase { MockRM rm = new MockRM(yarnConf); rm.start(); - MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT, TestUtils - .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, null)); + MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT, + ResourceTypesTestHelper.newResource( + DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + null)); MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) @@ -615,10 +620,11 @@ public abstract class ApplicationMasterServiceTestBase { MockRM rm = new MockRM(yarnConf); rm.start(); - MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT, TestUtils - .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT, + ResourceTypesTestHelper.newResource( + DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - ImmutableMap.of(CUSTOM_RES, 4))); + ImmutableMap.of(CUSTOM_RES, "4"))); MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java index 47606925af8..62b43e8e3bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; @@ -298,16 +299,16 @@ public class TestCapacitySchedulerWithMultiResourceTypes { MockRM rm = new MockRM(conf); rm.start(); - Map nameToValues = new HashMap<>(); - nameToValues.put(ResourceInformation.GPU_URI, 4); + Map nameToValues = new HashMap<>(); + nameToValues.put(ResourceInformation.GPU_URI, "4"); // Register NM1 with 10GB memory, 4 CPU and 4 GPU MockNM nm1 = rm.registerNode("127.0.0.1:1234", - TestUtils.createResource(10 * GB, 4, nameToValues)); + ResourceTypesTestHelper.newResource(10 * GB, 4, nameToValues)); nameToValues.clear(); // Register NM2 with 10GB memory, 4 CPU and 0 GPU rm.registerNode("127.0.0.1:1235", - TestUtils.createResource(10 * GB, 4, nameToValues)); + ResourceTypesTestHelper.newResource(10 * GB, 4, nameToValues)); RMApp app1 = MockRMAppSubmitter.submit(rm, MockRMAppSubmissionData.Builder.createWithMemory(1024, rm) @@ -330,9 +331,9 @@ public class TestCapacitySchedulerWithMultiResourceTypes { Assert.assertEquals(4, report_nm1.getAvailableResource() .getResourceInformation(ResourceInformation.GPU_URI).getValue()); - nameToValues.put(ResourceInformation.GPU_URI, 4); + nameToValues.put(ResourceInformation.GPU_URI, "4"); Resource containerGpuResource = - TestUtils.createResource(1 * GB, 1, nameToValues); + ResourceTypesTestHelper.newResource(1 * GB, 1, nameToValues); // Allocate one container which takes all 4 GPU am1.allocate( @@ -355,7 +356,7 @@ public class TestCapacitySchedulerWithMultiResourceTypes { nameToValues.clear(); Resource containerResource = - TestUtils.createResource(1 * GB, 1, nameToValues); + ResourceTypesTestHelper.newResource(1 * GB, 1, nameToValues); // Allocate one more container which doesnt need GPU am1.allocate( Collections.singletonList(ResourceRequest.newInstance( @@ -479,8 +480,8 @@ public class TestCapacitySchedulerWithMultiResourceTypes { for (int i = 0; i < 8; i++) { fiCaApp1.updateResourceRequests(Collections.singletonList( ResourceRequest.newBuilder() - .capability(TestUtils.createResource(1 * GB, 1, - ImmutableMap.of("res_1", 10))) + .capability(ResourceTypesTestHelper.newResource(1 * GB, 1, + ImmutableMap.of("res_1", "10"))) .numContainers(1) .resourceName("*") .build())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 15030a768f9..44b445156dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -467,23 +467,4 @@ public class TestUtils { cs.submitResourceCommitRequest(clusterResource, csAssignment); } - - /** - * An easy way to create resources other than memory and vcores for tests. - * @param memory memory - * @param vcores vcores - * @param nameToValues resource types other than memory and vcores. - * @return created resource - */ - public static Resource createResource(long memory, int vcores, - Map nameToValues) { - Resource res = Resource.newInstance(memory, vcores); - if (nameToValues != null) { - for (Map.Entry entry : nameToValues.entrySet()) { - res.setResourceInformation(entry.getKey(), ResourceInformation - .newInstance(entry.getKey(), "", entry.getValue())); - } - } - return res; - } }