Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added a configuration for admins to be able to capped per-container cpu usage based on a multiplier

This commit is contained in:
n.fraison 2019-03-24 22:30:19 +01:00
parent 64f30da428
commit 323707d0e5
3 changed files with 39 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -2190,6 +2190,15 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE =
false;
/**
* In case of strict resource usage provide capability to capped cpu resource
* usage by <code>DEFAULT_NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER</code>
* times the requested one
*/
public static final String NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER =
NM_PREFIX + "linux-container-executor.cgroups.capped-multiplier";
public static final float DEFAULT_NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER =
1.0f;
// Configurations for applicaiton life time monitor feature
public static final String RM_APPLICATION_MONITOR_INTERVAL_MS =

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
View File

@ -94,6 +94,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
private float yarnProcessors;
private int nodeVCores;
private float cappedMultiplier;
public CgroupsLCEResourcesHandler() {
this.controllerPaths = new HashMap<String, String>();
@ -138,13 +139,18 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE,
YarnConfiguration
.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE);
this.cappedMultiplier =
conf
.getFloat(
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER,
YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER);
int len = cgroupPrefix.length();
if (cgroupPrefix.charAt(len - 1) == '/') {
cgroupPrefix = cgroupPrefix.substring(0, len - 1);
}
}
public void init(LinuxContainerExecutor lce) throws IOException {
this.init(lce,
ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf));
@ -334,8 +340,8 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
String.valueOf(cpuShares));
if (strictResourceUsageMode) {
if (nodeVCores != containerVCores) {
float containerCPU =
(containerVCores * yarnProcessors) / (float) nodeVCores;
float containerVCoresCapped = Math.min(containerVCores * cappedMultiplier, nodeVCores);
float containerCPU = (containerVCoresCapped * yarnProcessors) / (float) nodeVCores;
int[] limits = getOverallLimits(containerCPU);
updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US,
String.valueOf(limits[0]));

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
View File

@ -341,8 +341,29 @@ public class TestCgroupsLCEResourcesHandler {
Assert.assertEquals(500 * 1000, readIntFromFile(periodFile));
Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile));
// 1/8 of CPU with multiplier set at 2
FileUtils.deleteQuietly(containerCpuDir);
conf.setBoolean(
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE,
true);
conf.setFloat(
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER,
2.0f);
handler.initConfig();
handler.preExecute(id,
Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 8));
Assert.assertTrue(containerCpuDir.exists());
Assert.assertTrue(containerCpuDir.isDirectory());
periodFile = new File(containerCpuDir, "cpu.cfs_period_us");
quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us");
Assert.assertTrue(periodFile.exists());
Assert.assertTrue(quotaFile.exists());
Assert.assertEquals(1000 * 1000, readIntFromFile(periodFile));
Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile));
// CGroups set to 50% of CPU, container set to 50% of YARN CPU
FileUtils.deleteQuietly(containerCpuDir);
conf.unset(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER);
conf.setBoolean(
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE,
true);