Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6953. Clean up ResourceUtils.setMinimumAllocationForMandatoryResources() and setMaximumAllocationForMandatoryResources()

Browse Source 
(Contributed by Manikandan R via Daniel Templeton)
This commit is contained in:
Daniel Templeton 2017-11-15 09:55:40 -08:00
parent 69043ba8b5
commit e094eb74b9
1 changed files with 36 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
View File

@ -142,74 +142,44 @@ private static void addMandatoryResources(
} }
} }
private static void setMinimumAllocationForMandatoryResources( private static void setAllocationForMandatoryResources(
Map<String, ResourceInformation> res, Configuration conf) { Map<String, ResourceInformation> res, Configuration conf) {
String[][] resourceTypesKeys = { ResourceInformation mem = res.get(ResourceInformation.MEMORY_MB.getName());
{ResourceInformation.MEMORY_MB.getName(), mem.setMinimumAllocation(getAllocation(conf,
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.RESOURCE_TYPES + "." +
String.valueOf( mem.getName() + MINIMUM_ALLOCATION,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB), YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
ResourceInformation.MEMORY_MB.getName()}, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
{ResourceInformation.VCORES.getName(), mem.setMaximumAllocation(getAllocation(conf,
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, YarnConfiguration.RESOURCE_TYPES + "." +
String.valueOf( mem.getName() + MAXIMUM_ALLOCATION,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES), YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
ResourceInformation.VCORES.getName()}}; YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
for (String[] arr : resourceTypesKeys) {
String resourceTypesKey = ResourceInformation cpu = res.get(ResourceInformation.VCORES.getName());
YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION;
long minimumResourceTypes = conf.getLong(resourceTypesKey, -1); cpu.setMinimumAllocation(getAllocation(conf,
long minimumConf = conf.getLong(arr[1], -1); YarnConfiguration.RESOURCE_TYPES + "." +
long minimum; cpu.getName() + MINIMUM_ALLOCATION,
if (minimumResourceTypes != -1) { YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
minimum = minimumResourceTypes; YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES));
if (minimumConf != -1) { cpu.setMaximumAllocation(getAllocation(conf,
LOG.warn("Using minimum allocation for memory specified in " YarnConfiguration.RESOURCE_TYPES + "." +
+ "resource-types config file with key " cpu.getName() + MAXIMUM_ALLOCATION,
+ minimumResourceTypes + ", ignoring minimum specified using " YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ arr[1]); YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES));
}
} else {
minimum = conf.getLong(arr[1], Long.parseLong(arr[2]));
}
ResourceInformation ri = res.get(arr[3]);
ri.setMinimumAllocation(minimum);
}
} }
private static void setMaximumAllocationForMandatoryResources( private static long getAllocation(Configuration conf,
Map<String, ResourceInformation> res, Configuration conf) { String resourceTypesKey, String schedulerKey, long schedulerDefault) {
String[][] resourceTypesKeys = { long value = conf.getLong(resourceTypesKey, -1L);
{ResourceInformation.MEMORY_MB.getName(), if (value == -1) {
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, LOG.debug("Mandatory Resource '" + resourceTypesKey + "' is not "
String.valueOf( + "configured in resource-types config file. Setting allocation "
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB), + "specified using '" + schedulerKey + "'");
ResourceInformation.MEMORY_MB.getName()}, value = conf.getLong(schedulerKey, schedulerDefault);
{ResourceInformation.VCORES.getName(),
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
String.valueOf(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
ResourceInformation.VCORES.getName()}};
for (String[] arr : resourceTypesKeys) {
String resourceTypesKey =
YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION;
long maximumResourceTypes = conf.getLong(resourceTypesKey, -1);
long maximumConf = conf.getLong(arr[1], -1);
long maximum;
if (maximumResourceTypes != -1) {
maximum = maximumResourceTypes;
if (maximumConf != -1) {
LOG.warn("Using maximum allocation for memory specified in "
+ "resource-types config file with key "
+ maximumResourceTypes + ", ignoring maximum specified using "
+ arr[1]);
}
} else {
maximum = conf.getLong(arr[1], Long.parseLong(arr[2]));
}
ResourceInformation ri = res.get(arr[3]);
ri.setMaximumAllocation(maximum);
} }
return value;
} }
@VisibleForTesting @VisibleForTesting
@ -275,8 +245,7 @@ static void initializeResourcesMap(Configuration conf) {
checkMandatoryResources(resourceInformationMap); checkMandatoryResources(resourceInformationMap);
addMandatoryResources(resourceInformationMap); addMandatoryResources(resourceInformationMap);
setMinimumAllocationForMandatoryResources(resourceInformationMap, conf); setAllocationForMandatoryResources(resourceInformationMap, conf);
setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
initializeResourcesFromResourceInformationMap(resourceInformationMap); initializeResourcesFromResourceInformationMap(resourceInformationMap);
} }
@ -473,8 +442,7 @@ public static Map<String, ResourceInformation> getNodeResourceInformation(
conf); conf);
addMandatoryResources(nodeResources); addMandatoryResources(nodeResources);
checkMandatoryResources(nodeResources); checkMandatoryResources(nodeResources);
setMinimumAllocationForMandatoryResources(nodeResources, conf); setAllocationForMandatoryResources(nodeResources, conf);
setMaximumAllocationForMandatoryResources(nodeResources, conf);
readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
initializedNodeResources = true; initializedNodeResources = true;
} }