diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b21d76369e5..79b95665feb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2190,6 +2190,15 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE =
false;
+ /**
+ * In case of strict resource usage provide capability to capped cpu resource
+ * usage by DEFAULT_NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER
+ * times the requested one
+ */
+ public static final String NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER =
+ NM_PREFIX + "linux-container-executor.cgroups.capped-multiplier";
+ public static final float DEFAULT_NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER =
+ 1.0f;
// Configurations for applicaiton life time monitor feature
public static final String RM_APPLICATION_MONITOR_INTERVAL_MS =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index 6025260c4bd..07539378c53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -94,6 +94,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
private float yarnProcessors;
private int nodeVCores;
+ private float cappedMultiplier;
public CgroupsLCEResourcesHandler() {
this.controllerPaths = new HashMap();
@@ -138,13 +139,18 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE,
YarnConfiguration
.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE);
+ this.cappedMultiplier =
+ conf
+ .getFloat(
+ YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER,
+ YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER);
int len = cgroupPrefix.length();
if (cgroupPrefix.charAt(len - 1) == '/') {
cgroupPrefix = cgroupPrefix.substring(0, len - 1);
}
}
-
+
public void init(LinuxContainerExecutor lce) throws IOException {
this.init(lce,
ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf));
@@ -334,8 +340,8 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
String.valueOf(cpuShares));
if (strictResourceUsageMode) {
if (nodeVCores != containerVCores) {
- float containerCPU =
- (containerVCores * yarnProcessors) / (float) nodeVCores;
+ float containerVCoresCapped = Math.min(containerVCores * cappedMultiplier, nodeVCores);
+ float containerCPU = (containerVCoresCapped * yarnProcessors) / (float) nodeVCores;
int[] limits = getOverallLimits(containerCPU);
updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US,
String.valueOf(limits[0]));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
index 7d8704f8b42..34dc2b6acd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
@@ -341,8 +341,29 @@ public class TestCgroupsLCEResourcesHandler {
Assert.assertEquals(500 * 1000, readIntFromFile(periodFile));
Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile));
+ // 1/8 of CPU with multiplier set at 2
+ FileUtils.deleteQuietly(containerCpuDir);
+ conf.setBoolean(
+ YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE,
+ true);
+ conf.setFloat(
+ YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER,
+ 2.0f);
+ handler.initConfig();
+ handler.preExecute(id,
+ Resource.newInstance(1024, YarnConfiguration.DEFAULT_NM_VCORES / 8));
+ Assert.assertTrue(containerCpuDir.exists());
+ Assert.assertTrue(containerCpuDir.isDirectory());
+ periodFile = new File(containerCpuDir, "cpu.cfs_period_us");
+ quotaFile = new File(containerCpuDir, "cpu.cfs_quota_us");
+ Assert.assertTrue(periodFile.exists());
+ Assert.assertTrue(quotaFile.exists());
+ Assert.assertEquals(1000 * 1000, readIntFromFile(periodFile));
+ Assert.assertEquals(1000 * 1000, readIntFromFile(quotaFile));
+
// CGroups set to 50% of CPU, container set to 50% of YARN CPU
FileUtils.deleteQuietly(containerCpuDir);
+ conf.unset(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_CAPPED_MULTIPLIER);
conf.setBoolean(
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE,
true);