Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9354. Resources should be created with ResourceTypesTestHelper instead of TestUtils. Contributed by Andras Gyori

This commit is contained in:
Szilard Nemeth 2020-03-10 16:44:33 +01:00
parent 61f4cf3055
commit cf9cf83a43
4 changed files with 39 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
View File

@ -57,14 +57,15 @@ public final class ResourceTypesTestHelper {
Resource resource = RECORD_FACTORY.newRecordInstance(Resource.class);
resource.setMemorySize(memory);
resource.setVirtualCores(vCores);
for (Map.Entry<String, String> customResource :
customResources.entrySet()) {
String resourceName = customResource.getKey();
ResourceInformation resourceInformation =
createResourceInformation(resourceName,
customResource.getValue());
resource.setResourceInformation(resourceName, resourceInformation);
if (customResources != null) {
for (Map.Entry<String, String> customResource :
customResources.entrySet()) {
String resourceName = customResource.getKey();
ResourceInformation resourceInformation =
createResourceInformation(resourceName,
customResource.getValue());
resource.setResourceInformation(resourceName, resourceInformation);
}
}
return resource;
}
@ -107,7 +108,12 @@ public final class ResourceTypesTestHelper {
public static Map<String, String> extractCustomResourcesAsStrings(
Resource res) {
Map<String, Long> resValues = extractCustomResources(res);
return resValues.entrySet().stream()
return convertCustomResources(resValues);
}
public static Map<String, String> convertCustomResources(
Map<String, ? extends Number> customResources) {
return customResources.entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey, e -> String.valueOf(e.getValue())));
}

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterServiceTestBase.java
View File

@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -46,7 +47,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@ -112,8 +112,11 @@ public abstract class ApplicationMasterServiceTestBase {
private void requestResources(MockAM am, long memory, int vCores,
Map<String, Integer> customResources) throws Exception {
Map<String, String> convertedCustomResources =
ResourceTypesTestHelper.convertCustomResources(customResources);
am.allocate(Collections.singletonList(ResourceRequest.newBuilder()
.capability(TestUtils.createResource(memory, vCores, customResources))
.capability(ResourceTypesTestHelper.newResource(
memory, vCores, convertedCustomResources))
.numContainers(1)
.resourceName("*")
.build()), null);
@ -551,9 +554,11 @@ public abstract class ApplicationMasterServiceTestBase {
MockRM rm = new MockRM(yarnConf);
rm.start();
MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT, TestUtils
.createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, null));
MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT,
ResourceTypesTestHelper.newResource(
DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
null));
MockRMAppSubmissionData data =
MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
@ -615,10 +620,11 @@ public abstract class ApplicationMasterServiceTestBase {
MockRM rm = new MockRM(yarnConf);
rm.start();
MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT, TestUtils
.createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
MockNM nm1 = rm.registerNode("199.99.99.1:" + DEFAULT_PORT,
ResourceTypesTestHelper.newResource(
DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
ImmutableMap.of(CUSTOM_RES, 4)));
ImmutableMap.of(CUSTOM_RES, "4")));
MockRMAppSubmissionData data =
MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
View File

@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
@ -298,16 +299,16 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
MockRM rm = new MockRM(conf);
rm.start();
Map<String, Integer> nameToValues = new HashMap<>();
nameToValues.put(ResourceInformation.GPU_URI, 4);
Map<String, String> nameToValues = new HashMap<>();
nameToValues.put(ResourceInformation.GPU_URI, "4");
// Register NM1 with 10GB memory, 4 CPU and 4 GPU
MockNM nm1 = rm.registerNode("127.0.0.1:1234",
TestUtils.createResource(10 * GB, 4, nameToValues));
ResourceTypesTestHelper.newResource(10 * GB, 4, nameToValues));
nameToValues.clear();
// Register NM2 with 10GB memory, 4 CPU and 0 GPU
rm.registerNode("127.0.0.1:1235",
TestUtils.createResource(10 * GB, 4, nameToValues));
ResourceTypesTestHelper.newResource(10 * GB, 4, nameToValues));
RMApp app1 = MockRMAppSubmitter.submit(rm,
MockRMAppSubmissionData.Builder.createWithMemory(1024, rm)
@ -330,9 +331,9 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
Assert.assertEquals(4, report_nm1.getAvailableResource()
.getResourceInformation(ResourceInformation.GPU_URI).getValue());
nameToValues.put(ResourceInformation.GPU_URI, 4);
nameToValues.put(ResourceInformation.GPU_URI, "4");
Resource containerGpuResource =
TestUtils.createResource(1 * GB, 1, nameToValues);
ResourceTypesTestHelper.newResource(1 * GB, 1, nameToValues);
// Allocate one container which takes all 4 GPU
am1.allocate(
@ -355,7 +356,7 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
nameToValues.clear();
Resource containerResource =
TestUtils.createResource(1 * GB, 1, nameToValues);
ResourceTypesTestHelper.newResource(1 * GB, 1, nameToValues);
// Allocate one more container which doesnt need GPU
am1.allocate(
Collections.singletonList(ResourceRequest.newInstance(
@ -479,8 +480,8 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
for (int i = 0; i < 8; i++) {
fiCaApp1.updateResourceRequests(Collections.singletonList(
ResourceRequest.newBuilder()
.capability(TestUtils.createResource(1 * GB, 1,
ImmutableMap.of("res_1", 10)))
.capability(ResourceTypesTestHelper.newResource(1 * GB, 1,
ImmutableMap.of("res_1", "10")))
.numContainers(1)
.resourceName("*")
.build()));

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
View File

@ -467,23 +467,4 @@ public class TestUtils {
cs.submitResourceCommitRequest(clusterResource,
csAssignment);
}
/**
* An easy way to create resources other than memory and vcores for tests.
* @param memory memory
* @param vcores vcores
* @param nameToValues resource types other than memory and vcores.
* @return created resource
*/
public static Resource createResource(long memory, int vcores,
Map<String, Integer> nameToValues) {
Resource res = Resource.newInstance(memory, vcores);
if (nameToValues != null) {
for (Map.Entry<String, Integer> entry : nameToValues.entrySet()) {
res.setResourceInformation(entry.getKey(), ResourceInformation
.newInstance(entry.getKey(), "", entry.getValue()));
}
}
return res;
}
}