YARN-5534. Allow user provided Docker volume mount list. (Contributed by Shane Kumpf)
This commit is contained in:
Eric Yang
parent
de8b6ca5ef
commit
d42a336cfa
|
|
@ -65,6 +65,7 @@
|
|||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
|
||||
|
|
@ -134,6 +135,16 @@
|
|||
* source is an absolute path that is not a symlink and that points to a
|
||||
* localized resource.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@code YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS} allows users to specify
|
||||
+ additional volume mounts for the Docker container. The value of the
|
||||
* environment variable should be a comma-separated list of mounts.
|
||||
* All such mounts must be given as {@code source:dest:mode}, and the mode
|
||||
* must be "ro" (read-only) or "rw" (read-write) to specify the type of
|
||||
* access being requested. The requested mounts will be validated by
|
||||
* container-executor based on the values set in container-executor.cfg for
|
||||
* {@code docker.allowed.ro-mounts} and {@code docker.allowed.rw-mounts}.
|
||||
* </li>
|
||||
* </ul>
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
|
|
@ -151,6 +162,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
"^[a-zA-Z0-9][a-zA-Z0-9_.-]+$";
|
||||
private static final Pattern hostnamePattern = Pattern.compile(
|
||||
HOSTNAME_PATTERN);
|
||||
private static final Pattern USER_MOUNT_PATTERN = Pattern.compile(
|
||||
"(?<=^|,)([^:\\x00]+):([^:\\x00]+):([a-z]+)");
|
||||
|
||||
@InterfaceAudience.Private
|
||||
public static final String ENV_DOCKER_CONTAINER_IMAGE =
|
||||
|
|
@ -176,6 +189,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
@InterfaceAudience.Private
|
||||
public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS =
|
||||
"YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS";
|
||||
@InterfaceAudience.Private
|
||||
public static final String ENV_DOCKER_CONTAINER_MOUNTS =
|
||||
"YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS";
|
||||
|
||||
private Configuration conf;
|
||||
private Context nmContext;
|
||||
|
|
@ -675,6 +691,32 @@ public void launchContainer(ContainerRuntimeContext ctx)
|
|||
}
|
||||
}
|
||||
|
||||
if (environment.containsKey(ENV_DOCKER_CONTAINER_MOUNTS)) {
|
||||
Matcher parsedMounts = USER_MOUNT_PATTERN.matcher(
|
||||
environment.get(ENV_DOCKER_CONTAINER_MOUNTS));
|
||||
if (!parsedMounts.find()) {
|
||||
throw new ContainerExecutionException(
|
||||
"Unable to parse user supplied mount list: "
|
||||
+ environment.get(ENV_DOCKER_CONTAINER_MOUNTS));
|
||||
}
|
||||
parsedMounts.reset();
|
||||
while (parsedMounts.find()) {
|
||||
String src = parsedMounts.group(1);
|
||||
String dst = parsedMounts.group(2);
|
||||
String mode = parsedMounts.group(3);
|
||||
if (!mode.equals("ro") && !mode.equals("rw")) {
|
||||
throw new ContainerExecutionException(
|
||||
"Invalid mount mode requested for mount: "
|
||||
+ parsedMounts.group());
|
||||
}
|
||||
if (mode.equals("ro")) {
|
||||
runCommand.addReadOnlyMountLocation(src, dst);
|
||||
} else {
|
||||
runCommand.addReadWriteMountLocation(src, dst);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (allowPrivilegedContainerExecution(container)) {
|
||||
runCommand.setPrivileged();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -66,6 +66,12 @@ public DockerRunCommand addMountLocation(String sourcePath, String
|
|||
return this;
|
||||
}
|
||||
|
||||
public DockerRunCommand addReadWriteMountLocation(String sourcePath, String
|
||||
destinationPath) {
|
||||
super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath);
|
||||
return this;
|
||||
}
|
||||
|
||||
public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
|
||||
destinationPath, boolean createSource) {
|
||||
boolean sourceExists = new File(sourcePath).exists();
|
||||
|
|
@ -76,6 +82,12 @@ public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
|
|||
return this;
|
||||
}
|
||||
|
||||
public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
|
||||
destinationPath) {
|
||||
super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath);
|
||||
return this;
|
||||
}
|
||||
|
||||
public DockerRunCommand setVolumeDriver(String volumeDriver) {
|
||||
super.addCommandArguments("volume-driver", volumeDriver);
|
||||
return this;
|
||||
|
|
|
|||
|
|
@ -1035,6 +1035,115 @@ public void testMountMultiple()
|
|||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUserMounts()
|
||||
throws ContainerExecutionException, PrivilegedOperationException,
|
||||
IOException{
|
||||
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
|
||||
mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf, null);
|
||||
|
||||
env.put(
|
||||
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
|
||||
"/tmp/foo:/tmp/foo:ro,/tmp/bar:/tmp/bar:rw");
|
||||
|
||||
runtime.launchContainer(builder.build());
|
||||
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
|
||||
List<String> args = op.getArguments();
|
||||
String dockerCommandFile = args.get(11);
|
||||
|
||||
List<String> dockerCommands = Files.readAllLines(
|
||||
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
Assert.assertEquals(14, dockerCommands.size());
|
||||
Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
|
||||
Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
|
||||
dockerCommands.get(1));
|
||||
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
|
||||
Assert.assertEquals(" detach=true", dockerCommands.get(3));
|
||||
Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
|
||||
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
|
||||
Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
|
||||
Assert.assertEquals(
|
||||
" launch-command=bash,/test_container_work_dir/launch_container.sh",
|
||||
dockerCommands.get(7));
|
||||
Assert.assertEquals(" name=container_id", dockerCommands.get(8));
|
||||
Assert.assertEquals(" net=host", dockerCommands.get(9));
|
||||
Assert.assertEquals(" ro-mounts=/tmp/foo:/tmp/foo",
|
||||
dockerCommands.get(10));
|
||||
Assert.assertEquals(
|
||||
" rw-mounts=/test_container_local_dir:/test_container_local_dir,"
|
||||
+ "/test_filecache_dir:/test_filecache_dir,"
|
||||
+ "/test_container_work_dir:/test_container_work_dir,"
|
||||
+ "/test_container_log_dir:/test_container_log_dir,"
|
||||
+ "/test_user_local_dir:/test_user_local_dir,"
|
||||
+ "/tmp/bar:/tmp/bar",
|
||||
dockerCommands.get(11));
|
||||
Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
|
||||
Assert.assertEquals(" workdir=/test_container_work_dir",
|
||||
dockerCommands.get(13));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUserMountInvalid()
|
||||
throws ContainerExecutionException, PrivilegedOperationException,
|
||||
IOException{
|
||||
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
|
||||
mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf, null);
|
||||
|
||||
env.put(
|
||||
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
|
||||
"source:target");
|
||||
|
||||
try {
|
||||
runtime.launchContainer(builder.build());
|
||||
Assert.fail("Expected a launch container failure due to invalid mount.");
|
||||
} catch (ContainerExecutionException e) {
|
||||
LOG.info("Caught expected exception : " + e);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUserMountModeInvalid()
|
||||
throws ContainerExecutionException, PrivilegedOperationException,
|
||||
IOException{
|
||||
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
|
||||
mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf, null);
|
||||
|
||||
env.put(
|
||||
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
|
||||
"source:target:other");
|
||||
|
||||
try {
|
||||
runtime.launchContainer(builder.build());
|
||||
Assert.fail("Expected a launch container failure due to invalid mode.");
|
||||
} catch (ContainerExecutionException e) {
|
||||
LOG.info("Caught expected exception : " + e);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUserMountModeNulInvalid()
|
||||
throws ContainerExecutionException, PrivilegedOperationException,
|
||||
IOException{
|
||||
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
|
||||
mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf, null);
|
||||
|
||||
env.put(
|
||||
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
|
||||
"s\0ource:target:ro");
|
||||
|
||||
try {
|
||||
runtime.launchContainer(builder.build());
|
||||
Assert.fail("Expected a launch container failure due to NUL in mount.");
|
||||
} catch (ContainerExecutionException e) {
|
||||
LOG.info("Caught expected exception : " + e);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testContainerLivelinessCheck()
|
||||
throws ContainerExecutionException, PrivilegedOperationException {
|
||||
|
|
|
|||
|
|
@ -290,6 +290,7 @@ environment variables in the application's environment:
|
|||
| `YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK` | Sets the network type to be used by the Docker container. It must be a valid value as determined by the yarn.nodemanager.runtime.linux.docker.allowed-container-networks property. |
|
||||
| `YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER` | Controls whether the Docker container is a privileged container. In order to use privileged containers, the yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed property must be set to true, and the application owner must appear in the value of the yarn.nodemanager.runtime.linux.docker.privileged-containers.acl property. If this environment variable is set to true, a privileged Docker container will be used if allowed. No other value is allowed, so the environment variable should be left unset rather than setting it to false. |
|
||||
| `YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest", where the source is an absolute path that is not a symlink and that points to a localized resource. Note that as of YARN-5298, localized directories are automatically mounted into the container as volumes. |
|
||||
| `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest:mode" and the mode must be "ro" (read-only) or "rw" (read-write) to specify the type of access being requested. The requested mounts will be validated by container-executor based on the values set in container-executor.cfg for docker.allowed.ro-mounts and docker.allowed.rw-mounts. |
|
||||
|
||||
The first two are required. The remainder can be set as needed. While
|
||||
controlling the container type through environment variables is somewhat less
|
||||
|
|
@ -302,6 +303,53 @@ the application will behave exactly as any other YARN application. Logs will be
|
|||
aggregated and stored in the relevant history server. The application life cycle
|
||||
will be the same as for a non-Docker application.
|
||||
|
||||
Using Docker Bind Mounted Volumes
|
||||
---------------------------------
|
||||
|
||||
**WARNING** Care should be taken when enabling this feature. Enabling access to
|
||||
directories such as, but not limited to, /, /etc, /run, or /home is not
|
||||
advisable and can result in containers negatively impacting the host or leaking
|
||||
sensitive information. **WARNING**
|
||||
|
||||
Files and directories from the host are commonly needed within the Docker
|
||||
containers, which Docker provides through
|
||||
[volumes](https://docs.docker.com/engine/tutorials/dockervolumes/).
|
||||
Examples include localized resources, Apache Hadoop binaries, and sockets. To
|
||||
facilitate this need, YARN-6623 added the ability for administrators to set a
|
||||
whitelist of host directories that are allowed to be bind mounted as volumes
|
||||
into containers. YARN-5534 added the ability for users to supply a list of
|
||||
mounts that will be mounted into the containers, if allowed by the
|
||||
administrative whitelist.
|
||||
|
||||
In order to make use of this feature, the following must be configured.
|
||||
|
||||
* The administrator must define the volume whitelist in container-executor.cfg by setting `docker.allowed.ro-mounts` and `docker.allowed.rw-mounts` to the list of parent directories that are allowed to be mounted.
|
||||
* The application submitter requests the required volumes at application submission time using the `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` environment variable.
|
||||
|
||||
The administrator supplied whitelist is defined as a comma separated list of
|
||||
directories that are allowed to be mounted into containers. The source directory
|
||||
supplied by the user must either match or be a child of the specified
|
||||
directory.
|
||||
|
||||
The user supplied mount list is defined as a comma separated list in the form
|
||||
*source*:*destination*:*mode*. The source is the file or directory on the host.
|
||||
The destination is the path within the contatiner where the source will be bind
|
||||
mounted. The mode defines the mode the user expects for the mount, which can be
|
||||
ro (read-only) or rw (read-write).
|
||||
|
||||
The following example outlines how to use this feature to mount the commonly
|
||||
needed /sys/fs/cgroup directory into the container running on YARN.
|
||||
|
||||
The administrator sets docker.allowed.ro-mounts in container-executor.cfg to
|
||||
"/sys/fs/cgroup". Applications can now request that "/sys/fs/cgroup" be mounted
|
||||
from the host into the container in read-only mode.
|
||||
|
||||
At application submission time, the YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS
|
||||
environment variable can then be set to request this mount. In this example,
|
||||
the environment variable would be set to "/sys/fs/cgroup:/sys/fs/cgroup:ro".
|
||||
The destination path is not restricted, "/sys/fs/cgroup:/cgroup:ro" would also
|
||||
be valid given the example admin whitelist.
|
||||
|
||||
Connecting to a Secure Docker Repository
|
||||
----------------------------------------
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue