From d42a336cfab106d052aa30d80d9d30904123cb55 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Wed, 22 Nov 2017 13:05:34 -0500 Subject: [PATCH] YARN-5534. Allow user provided Docker volume mount list. (Contributed by Shane Kumpf) --- .../runtime/DockerLinuxContainerRuntime.java | 42 +++++++ .../runtime/docker/DockerRunCommand.java | 12 ++ .../runtime/TestDockerContainerRuntime.java | 109 ++++++++++++++++++ .../src/site/markdown/DockerContainers.md | 48 ++++++++ 4 files changed, 211 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 75a28e648b3..e61dc23376b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -65,6 +65,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*; @@ -134,6 +135,16 @@ * source is an absolute path that is not a symlink and that points to a * localized resource. * + *
  • + * {@code YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS} allows users to specify + + additional volume mounts for the Docker container. The value of the + * environment variable should be a comma-separated list of mounts. + * All such mounts must be given as {@code source:dest:mode}, and the mode + * must be "ro" (read-only) or "rw" (read-write) to specify the type of + * access being requested. The requested mounts will be validated by + * container-executor based on the values set in container-executor.cfg for + * {@code docker.allowed.ro-mounts} and {@code docker.allowed.rw-mounts}. + *
    • * */ @InterfaceAudience.Private @@ -151,6 +162,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { "^[a-zA-Z0-9][a-zA-Z0-9_.-]+$"; private static final Pattern hostnamePattern = Pattern.compile( HOSTNAME_PATTERN); + private static final Pattern USER_MOUNT_PATTERN = Pattern.compile( + "(?<=^|,)([^:\\x00]+):([^:\\x00]+):([a-z]+)"); @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_IMAGE = @@ -176,6 +189,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS = "YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_MOUNTS = + "YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS"; private Configuration conf; private Context nmContext; @@ -675,6 +691,32 @@ public void launchContainer(ContainerRuntimeContext ctx) } } + if (environment.containsKey(ENV_DOCKER_CONTAINER_MOUNTS)) { + Matcher parsedMounts = USER_MOUNT_PATTERN.matcher( + environment.get(ENV_DOCKER_CONTAINER_MOUNTS)); + if (!parsedMounts.find()) { + throw new ContainerExecutionException( + "Unable to parse user supplied mount list: " + + environment.get(ENV_DOCKER_CONTAINER_MOUNTS)); + } + parsedMounts.reset(); + while (parsedMounts.find()) { + String src = parsedMounts.group(1); + String dst = parsedMounts.group(2); + String mode = parsedMounts.group(3); + if (!mode.equals("ro") && !mode.equals("rw")) { + throw new ContainerExecutionException( + "Invalid mount mode requested for mount: " + + parsedMounts.group()); + } + if (mode.equals("ro")) { + runCommand.addReadOnlyMountLocation(src, dst); + } else { + runCommand.addReadWriteMountLocation(src, dst); + } + } + } + if (allowPrivilegedContainerExecution(container)) { runCommand.setPrivileged(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index 8734ba6b8de..aa7d4d57144 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -66,6 +66,12 @@ public DockerRunCommand addMountLocation(String sourcePath, String return this; } + public DockerRunCommand addReadWriteMountLocation(String sourcePath, String + destinationPath) { + super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath); + return this; + } + public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String destinationPath, boolean createSource) { boolean sourceExists = new File(sourcePath).exists(); @@ -76,6 +82,12 @@ public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String return this; } + public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String + destinationPath) { + super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath); + return this; + } + public DockerRunCommand setVolumeDriver(String volumeDriver) { super.addCommandArguments("volume-driver", volumeDriver); return this; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 76aca04570b..6135493be63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -1035,6 +1035,115 @@ public void testMountMultiple() } + @Test + public void testUserMounts() + throws ContainerExecutionException, PrivilegedOperationException, + IOException{ + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, null); + + env.put( + DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS, + "/tmp/foo:/tmp/foo:ro,/tmp/bar:/tmp/bar:rw"); + + runtime.launchContainer(builder.build()); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines( + Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + Assert.assertEquals(14, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(1)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2)); + Assert.assertEquals(" detach=true", dockerCommands.get(3)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(4)); + Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5)); + Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(7)); + Assert.assertEquals(" name=container_id", dockerCommands.get(8)); + Assert.assertEquals(" net=host", dockerCommands.get(9)); + Assert.assertEquals(" ro-mounts=/tmp/foo:/tmp/foo", + dockerCommands.get(10)); + Assert.assertEquals( + " rw-mounts=/test_container_local_dir:/test_container_local_dir," + + "/test_filecache_dir:/test_filecache_dir," + + "/test_container_work_dir:/test_container_work_dir," + + "/test_container_log_dir:/test_container_log_dir," + + "/test_user_local_dir:/test_user_local_dir," + + "/tmp/bar:/tmp/bar", + dockerCommands.get(11)); + Assert.assertEquals(" user=run_as_user", dockerCommands.get(12)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(13)); + } + + @Test + public void testUserMountInvalid() + throws ContainerExecutionException, PrivilegedOperationException, + IOException{ + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, null); + + env.put( + DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS, + "source:target"); + + try { + runtime.launchContainer(builder.build()); + Assert.fail("Expected a launch container failure due to invalid mount."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + + @Test + public void testUserMountModeInvalid() + throws ContainerExecutionException, PrivilegedOperationException, + IOException{ + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, null); + + env.put( + DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS, + "source:target:other"); + + try { + runtime.launchContainer(builder.build()); + Assert.fail("Expected a launch container failure due to invalid mode."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + + @Test + public void testUserMountModeNulInvalid() + throws ContainerExecutionException, PrivilegedOperationException, + IOException{ + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, null); + + env.put( + DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS, + "s\0ource:target:ro"); + + try { + runtime.launchContainer(builder.build()); + Assert.fail("Expected a launch container failure due to NUL in mount."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + @Test public void testContainerLivelinessCheck() throws ContainerExecutionException, PrivilegedOperationException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index dbbce7f10dc..1a50c92b965 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -290,6 +290,7 @@ environment variables in the application's environment: | `YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK` | Sets the network type to be used by the Docker container. It must be a valid value as determined by the yarn.nodemanager.runtime.linux.docker.allowed-container-networks property. | | `YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER` | Controls whether the Docker container is a privileged container. In order to use privileged containers, the yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed property must be set to true, and the application owner must appear in the value of the yarn.nodemanager.runtime.linux.docker.privileged-containers.acl property. If this environment variable is set to true, a privileged Docker container will be used if allowed. No other value is allowed, so the environment variable should be left unset rather than setting it to false. | | `YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest", where the source is an absolute path that is not a symlink and that points to a localized resource. Note that as of YARN-5298, localized directories are automatically mounted into the container as volumes. | +| `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest:mode" and the mode must be "ro" (read-only) or "rw" (read-write) to specify the type of access being requested. The requested mounts will be validated by container-executor based on the values set in container-executor.cfg for docker.allowed.ro-mounts and docker.allowed.rw-mounts. | The first two are required. The remainder can be set as needed. While controlling the container type through environment variables is somewhat less @@ -302,6 +303,53 @@ the application will behave exactly as any other YARN application. Logs will be aggregated and stored in the relevant history server. The application life cycle will be the same as for a non-Docker application. +Using Docker Bind Mounted Volumes +--------------------------------- + +**WARNING** Care should be taken when enabling this feature. Enabling access to +directories such as, but not limited to, /, /etc, /run, or /home is not +advisable and can result in containers negatively impacting the host or leaking +sensitive information. **WARNING** + +Files and directories from the host are commonly needed within the Docker +containers, which Docker provides through +[volumes](https://docs.docker.com/engine/tutorials/dockervolumes/). +Examples include localized resources, Apache Hadoop binaries, and sockets. To +facilitate this need, YARN-6623 added the ability for administrators to set a +whitelist of host directories that are allowed to be bind mounted as volumes +into containers. YARN-5534 added the ability for users to supply a list of +mounts that will be mounted into the containers, if allowed by the +administrative whitelist. + +In order to make use of this feature, the following must be configured. + +* The administrator must define the volume whitelist in container-executor.cfg by setting `docker.allowed.ro-mounts` and `docker.allowed.rw-mounts` to the list of parent directories that are allowed to be mounted. +* The application submitter requests the required volumes at application submission time using the `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` environment variable. + +The administrator supplied whitelist is defined as a comma separated list of +directories that are allowed to be mounted into containers. The source directory +supplied by the user must either match or be a child of the specified +directory. + +The user supplied mount list is defined as a comma separated list in the form +*source*:*destination*:*mode*. The source is the file or directory on the host. +The destination is the path within the contatiner where the source will be bind +mounted. The mode defines the mode the user expects for the mount, which can be +ro (read-only) or rw (read-write). + +The following example outlines how to use this feature to mount the commonly +needed /sys/fs/cgroup directory into the container running on YARN. + +The administrator sets docker.allowed.ro-mounts in container-executor.cfg to +"/sys/fs/cgroup". Applications can now request that "/sys/fs/cgroup" be mounted +from the host into the container in read-only mode. + +At application submission time, the YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS +environment variable can then be set to request this mount. In this example, +the environment variable would be set to "/sys/fs/cgroup:/sys/fs/cgroup:ro". +The destination path is not restricted, "/sys/fs/cgroup:/cgroup:ro" would also +be valid given the example admin whitelist. + Connecting to a Secure Docker Repository ----------------------------------------