YARN-4553. Add cgroups support for docker containers. Contributed by Sidharta Seethana.

hadoop-yarn-project/CHANGES.txt

@@ -94,6 +94,9 @@ Release 2.9.0 - UNRELEASED
     YARN-4551. Address the duplication between StatusUpdateWhenHealthy and
     StatusUpdateWhenUnhealthy transitions. (Sunil G via kasha)
 
+    YARN-4553. Add cgroups support for docker containers.
+    (Sidharta Seethana via vvasudev)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java

@@ -27,6 +27,9 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
 
@@ -45,12 +48,19 @@ public void initialize(Configuration conf)
       throws ContainerExecutionException {
     PrivilegedOperationExecutor privilegedOperationExecutor =
         PrivilegedOperationExecutor.getInstance(conf);
+    CGroupsHandler cGroupsHandler;
+    try {
+      cGroupsHandler = ResourceHandlerModule.getCGroupsHandler(conf);
+    } catch (ResourceHandlerException e) {
+      LOG.error("Unable to get cgroups handle.");
+      throw new ContainerExecutionException(e);
+    }
 
     defaultLinuxContainerRuntime = new DefaultLinuxContainerRuntime(
         privilegedOperationExecutor);
     defaultLinuxContainerRuntime.initialize(conf);
     dockerLinuxContainerRuntime = new DockerLinuxContainerRuntime(
-        privilegedOperationExecutor);
+        privilegedOperationExecutor, cGroupsHandler);
     dockerLinuxContainerRuntime.initialize(conf);
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java

@@ -36,8 +36,6 @@
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerClient;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
@@ -76,6 +74,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
   private Configuration conf;
   private DockerClient dockerClient;
   private PrivilegedOperationExecutor privilegedOperationExecutor;
+  private CGroupsHandler cGroupsHandler;
   private AccessControlList privilegedContainersAcl;
 
   public static boolean isDockerContainerRequested(
@@ -90,8 +89,9 @@ public static boolean isDockerContainerRequested(
   }
 
   public DockerLinuxContainerRuntime(PrivilegedOperationExecutor
-      privilegedOperationExecutor) {
+      privilegedOperationExecutor, CGroupsHandler cGroupsHandler) {
     this.privilegedOperationExecutor = privilegedOperationExecutor;
+    this.cGroupsHandler = cGroupsHandler;
   }
 
   @Override
@@ -116,30 +116,23 @@ public void addCGroupParentIfRequired(String resourcesOptions,
     if (resourcesOptions.equals(
         (PrivilegedOperation.CGROUP_ARG_PREFIX + PrivilegedOperation
             .CGROUP_ARG_NO_TASKS))) {
-      if (LOG.isInfoEnabled()) {
-        LOG.info("no resource restrictions specified. not using docker's "
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("no resource restrictions specified. not using docker's "
             + "cgroup options");
       }
     } else {
-      if (LOG.isInfoEnabled()) {
-        LOG.info("using docker's cgroups options");
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("using docker's cgroups options");
       }
 
-      try {
-        CGroupsHandler cGroupsHandler = ResourceHandlerModule
-            .getCGroupsHandler(conf);
-        String cGroupPath = "/" + cGroupsHandler.getRelativePathForCGroup(
-            containerIdStr);
+      String cGroupPath = "/" + cGroupsHandler.getRelativePathForCGroup(
+          containerIdStr);
 
-        if (LOG.isInfoEnabled()) {
-          LOG.info("using cgroup parent: " + cGroupPath);
-        }
-
-        runCommand.setCGroupParent(cGroupPath);
-      } catch (ResourceHandlerException e) {
-        LOG.warn("unable to use cgroups handler. Exception: ", e);
-        throw new ContainerExecutionException(e);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("using cgroup parent: " + cGroupPath);
       }
+
+      runCommand.setCGroupParent(cGroupPath);
     }
   }
 
@@ -256,11 +249,7 @@ public void launchContainer(ContainerRuntimeContext ctx)
 
     String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS);
 
-    /** Disabling docker's cgroup parent support for the time being. Docker
-     * needs to use a more recent libcontainer that supports net_cls. In
-     * addition we also need to revisit current cgroup creation in YARN.
-     */
-    //addCGroupParentIfRequired(resourcesOpts, containerIdStr, runCommand);
+    addCGroupParentIfRequired(resourcesOpts, containerIdStr, runCommand);
 
    Path nmPrivateContainerScriptPath = ctx.getExecutionAttribute(
         NM_PRIVATE_CONTAINER_SCRIPT_PATH);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java

@@ -31,6 +31,8 @@
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
@@ -62,6 +64,7 @@ public class TestDockerContainerRuntime {
       .getLog(TestDockerContainerRuntime.class);
   private Configuration conf;
   PrivilegedOperationExecutor mockExecutor;
+  CGroupsHandler mockCGroupsHandler;
   String containerId;
   Container container;
   ContainerId cId;
@@ -94,6 +97,7 @@ public void setup() {
 
     mockExecutor = Mockito
         .mock(PrivilegedOperationExecutor.class);
+    mockCGroupsHandler = Mockito.mock(CGroupsHandler.class);
     containerId = "container_id";
     container = mock(Container.class);
     cId = mock(ContainerId.class);
@@ -118,7 +122,7 @@ public void setup() {
     pidFilePath = new Path("/test_pid_file_path");
     localDirs = new ArrayList<>();
     logDirs = new ArrayList<>();
-    resourcesOptions = "cgroups:none";
+    resourcesOptions = "cgroups=none";
 
     localDirs.add("/test_local_dir");
     logDirs.add("/test_log_dir");
@@ -204,7 +208,7 @@ public void testDockerContainerLaunch()
       throws ContainerExecutionException, PrivilegedOperationException,
       IOException {
     DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
-        mockExecutor);
+        mockExecutor, mockCGroupsHandler);
     runtime.initialize(conf);
 
     String[] testCapabilities = {"NET_BIND_SERVICE", "SYS_CHROOT"};
@@ -256,7 +260,7 @@ public void testLaunchPrivilegedContainersInvalidEnvVar()
       throws ContainerExecutionException, PrivilegedOperationException,
       IOException{
     DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
-        mockExecutor);
+        mockExecutor, mockCGroupsHandler);
     runtime.initialize(conf);
 
     env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
@@ -284,7 +288,7 @@ public void testLaunchPrivilegedContainersWithDisabledSetting()
       throws ContainerExecutionException, PrivilegedOperationException,
       IOException{
     DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
-        mockExecutor);
+        mockExecutor, mockCGroupsHandler);
     runtime.initialize(conf);
 
     env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
@@ -307,7 +311,7 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL()
         true);
 
     DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
-        mockExecutor);
+        mockExecutor, mockCGroupsHandler);
     runtime.initialize(conf);
 
     env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
@@ -338,7 +342,7 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL()
         whitelistedUser);
 
     DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
-        mockExecutor);
+        mockExecutor, mockCGroupsHandler);
     runtime.initialize(conf);
 
     env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
@@ -365,7 +369,7 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL()
         submittingUser);
 
     DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
-        mockExecutor);
+        mockExecutor, mockCGroupsHandler);
     runtime.initialize(conf);
 
     env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
@@ -388,4 +392,34 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL()
         + ": " + command, command.contains("--privileged"));
   }
 
+  @Test
+  public void testCGroupParent() throws ContainerExecutionException {
+    String hierarchy = "hadoop-yarn-test";
+    conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
+        hierarchy);
+
+    DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime
+        (mockExecutor, mockCGroupsHandler);
+    runtime.initialize(conf);
+
+    String resourceOptionsNone = "cgroups=none";
+    DockerRunCommand command = Mockito.mock(DockerRunCommand.class);
+
+    Mockito.when(mockCGroupsHandler.getRelativePathForCGroup(containerId))
+        .thenReturn(hierarchy + "/" + containerIdStr);
+    runtime.addCGroupParentIfRequired(resourceOptionsNone, containerIdStr,
+        command);
+
+    //no --cgroup-parent should be added here
+    Mockito.verifyZeroInteractions(command);
+
+    String resourceOptionsCpu = "/sys/fs/cgroup/cpu/" + hierarchy +
+        containerIdStr;
+    runtime.addCGroupParentIfRequired(resourceOptionsCpu, containerIdStr,
+        command);
+
+    //--cgroup-parent should be added for the containerId in question
+    String expectedPath = "/" + hierarchy + "/" + containerIdStr;
+    Mockito.verify(command).setCGroupParent(expectedPath);
+  }
 }