YARN-4258. Add support for controlling capabilities for docker containers. Contributed by Sidharta Seethana.

(cherry picked from commit 63020c54c1)

hadoop-yarn-project/CHANGES.txt

@@ -452,6 +452,9 @@ Release 2.8.0 - UNRELEASED
     YARN-4252. Log container-executor invocation details when exit code is non-zero.
     (Sidharta Seethana via vvasudev)
 
+    YARN-4258. Add support for controlling capabilities for docker containers.
+    (Sidharta Seethana via vvasudev)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -1120,6 +1120,36 @@ public class YarnConfiguration extends Configuration {
   public static final String NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME =
           "/usr/bin/docker";
 
+  /** Prefix for runtime configuration constants. */
+  public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX +
+      "runtime.linux.";
+  public static final String DOCKER_CONTAINER_RUNTIME_PREFIX =
+      LINUX_CONTAINER_RUNTIME_PREFIX + "docker.";
+
+  /** Capabilities allowed (and added by default) for docker containers. **/
+  public static final String NM_DOCKER_CONTAINER_CAPABILITIES =
+      DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities";
+
+  /** These are the default capabilities added by docker. We'll use the same
+   * set here. While these may not be case-sensitive from a docker
+   * perspective, it is best to keep these uppercase.
+   */
+  public static final String[] DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES = {
+      "CHOWN",
+      "DAC_OVERRIDE",
+      "FSETID",
+      "FOWNER",
+      "MKNOD",
+      "NET_RAW",
+      "SETGID",
+      "SETUID",
+      "SETFCAP",
+      "SETPCAP",
+      "NET_BIND_SERVICE",
+      "SYS_CHROOT",
+      "KILL",
+      "AUDIT_WRITE" };
+
   /** The path to the Linux container executor.*/
   public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH =
     NM_PREFIX + "linux-container-executor.path";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -1419,6 +1419,15 @@
     <value>false</value>
   </property>
 
+  <property>
+    <description>This configuration setting determines the capabilities
+      assigned to docker containers when they are launched. While these may not
+      be case-sensitive from a docker perspective, it is best to keep these
+      uppercase.</description>
+    <name>yarn.nodemanager.runtime.linux.docker.capabilities</name>
+    <value>CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE</value>
+  </property>
+
   <property>
     <description>This flag determines whether memory limit will be set for the Windows Job
     Object of the containers launched by the default container executor.</description>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java

@@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
@@ -43,8 +44,11 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.Contai
 
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
 
@@ -154,12 +158,17 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
     List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
     @SuppressWarnings("unchecked")
     List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
+    Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
+        YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+        YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
+
     @SuppressWarnings("unchecked")
     DockerRunCommand runCommand = new DockerRunCommand(containerIdStr,
         runAsUser, imageName)
         .detachOnRun()
         .setContainerWorkDir(containerWorkDir.toString())
         .setNetworkType("host")
+        .setCapabilities(capabilities)
         .addMountLocation("/etc/passwd", "/etc/password:ro");
     List<String> allDirs = new ArrayList<>(localDirs);
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java

@@ -24,6 +24,7 @@ import org.apache.hadoop.util.StringUtils;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 public class DockerRunCommand extends DockerCommand {
   private static final String RUN_COMMAND = "run";
@@ -68,6 +69,17 @@ public class DockerRunCommand extends DockerCommand {
     return this;
   }
 
+  public DockerRunCommand setCapabilities(Set<String> capabilties) {
+    //first, drop all capabilities
+    super.addCommandArguments("--cap-drop=ALL");
+
+    //now, add the capabilities supplied
+    for (String capability : capabilties) {
+      super.addCommandArguments("--cap-add=" + capability);
+    }
+
+    return this;
+  }
   public DockerRunCommand addDevice(String sourceDevice, String
       destinationDevice) {
     super.addCommandArguments("--device=" + sourceDevice + ":" +

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java

@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
@@ -44,9 +45,12 @@ import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
 import static org.mockito.Matchers.eq;
@@ -154,6 +158,10 @@ public class TestDockerContainerRuntime {
         .setExecutionAttribute(LOG_DIRS, logDirs)
         .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
 
+    String[] testCapabilities = {"NET_BIND_SERVICE", "SYS_CHROOT"};
+
+    conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
+        testCapabilities);
     runtime.launchContainer(builder.build());
 
     ArgumentCaptor<PrivilegedOperation> opCaptor = ArgumentCaptor.forClass(
@@ -195,11 +203,23 @@ public class TestDockerContainerRuntime {
 
     String dockerCommandFile = args.get(11);
 
+    /* Ordering of capabilities depends on HashSet ordering. */
+
+    Set<String> capabilitySet = new HashSet<>(Arrays.asList(testCapabilities));
+    StringBuilder expectedCapabilitiesString = new StringBuilder(
+        "--cap-drop=ALL ");
+    for(String capability : capabilitySet) {
+      expectedCapabilitiesString.append("--cap-add=").append(capability)
+          .append(" ");
+    }
+    
     //This is the expected docker invocation for this case
     StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ")
         .append("--user=%2$s -d ")
         .append("--workdir=%3$s ")
-        .append("--net=host -v /etc/passwd:/etc/password:ro ")
+        .append("--net=host ")
+        .append(expectedCapabilitiesString)
+        .append("-v /etc/passwd:/etc/password:ro ")
         .append("-v %4$s:%4$s ")
         .append("-v %5$s:%5$s ")
         .append("-v %6$s:%6$s ")