diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index 137421a3e45..06a32be9d5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime. import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DelegatingLinuxContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerCommandExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRmCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; @@ -94,14 +95,14 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r * appropriate {@link LinuxContainerRuntime} instance. This class uses a * {@link DelegatingLinuxContainerRuntime} instance, which will delegate calls * to either a {@link DefaultLinuxContainerRuntime} instance or a - * {@link DockerLinuxContainerRuntime} instance, depending on the job's + * {@link OCIContainerRuntime} instance, depending on the job's * configuration.

* * @see LinuxContainerRuntime * @see DelegatingLinuxContainerRuntime * @see DefaultLinuxContainerRuntime * @see DockerLinuxContainerRuntime - * @see DockerLinuxContainerRuntime#isDockerContainerRequested + * @see OCIContainerRuntime#isOCICompliantContainerRequested */ public class LinuxContainerExecutor extends ContainerExecutor { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java index faf926aab64..e92560e3f87 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java @@ -35,7 +35,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DockerContainerDeletionTask; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext; import org.slf4j.Logger; @@ -147,7 +147,7 @@ public class ContainerCleanup implements Runnable { } // rm container in docker - if (DockerLinuxContainerRuntime.isDockerContainerRequested(conf, + if (OCIContainerRuntime.isOCICompliantContainerRequested(conf, container.getLaunchContext().getEnvironment())) { rmDockerContainerDelayed(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java index 9474b0f8471..aa52dd32da6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java @@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDiscoverer; @@ -105,7 +105,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler { // Create device cgroups for the container cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES, containerIdStr); - if (!DockerLinuxContainerRuntime.isDockerContainerRequested( + if (!OCIContainerRuntime.isOCICompliantContainerRequested( nmContext.getConf(), container.getLaunchContext().getEnvironment())) { // Write to devices cgroup only for non-docker container. The reason is diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 85ddca90dad..50721debe5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -23,9 +23,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; -import org.apache.hadoop.yarn.api.CsiAdaptorProtocol; -import org.apache.hadoop.yarn.api.impl.pb.client.CsiAdaptorProtocolPBClientImpl; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.nodemanager.Context; @@ -41,7 +40,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.volume.csi.ContainerVolumePublisher; import org.apache.hadoop.yarn.util.DockerClientConfigHandler; -import org.apache.hadoop.yarn.util.csi.CsiConfigUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -50,9 +48,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; @@ -67,7 +62,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime. import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerInspectCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext; @@ -75,19 +69,14 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext; import java.io.File; import java.io.IOException; import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -95,7 +84,7 @@ import java.util.regex.Pattern; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*; /** - *

This class is a {@link ContainerRuntime} implementation that uses the + *

This class is an extension of {@link OCIContainerRuntime} that uses the * native {@code container-executor} binary via a * {@link PrivilegedOperationExecutor} instance to launch processes inside * Docker containers.

@@ -169,7 +158,7 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r * *
  • * {@code YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS} allows users to specify - + additional volume mounts for the Docker container. The value of the + * additional volume mounts for the Docker container. The value of the * environment variable should be a comma-separated list of mounts. * All such mounts must be given as {@code source:dest[:mode]} and the mode * must be "ro" (read-only) or "rw" (read-write) to specify the type of @@ -200,57 +189,35 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r *
  • * {@code YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE} allows export yarn * service json to docker container. This feature is disabled by default. - * when this feature is set, app.json will be available in + * When this feature is set, app.json will be available in * /hadoop/yarn/sysfs/app.json. *
    • * */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { +public class DockerLinuxContainerRuntime extends OCIContainerRuntime { private static final Logger LOG = - LoggerFactory.getLogger(DockerLinuxContainerRuntime.class); + LoggerFactory.getLogger(DockerLinuxContainerRuntime.class); // This validates that the image is a proper docker image public static final String DOCKER_IMAGE_PATTERN = "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$"; private static final Pattern dockerImagePattern = Pattern.compile(DOCKER_IMAGE_PATTERN); - public static final String HOSTNAME_PATTERN = - "^[a-zA-Z0-9][a-zA-Z0-9_.-]+$"; - private static final Pattern hostnamePattern = Pattern.compile( - HOSTNAME_PATTERN); - private static final Pattern USER_MOUNT_PATTERN = Pattern.compile( - "(?<=^|,)([^:\\x00]+):([^:\\x00]+)" + - "(:(r[ow]|(r[ow][+])?(r?shared|r?slave|r?private)))?(?:,|$)"); - private static final Pattern TMPFS_MOUNT_PATTERN = Pattern.compile( - "^/[^:\\x00]+$"); - public static final String PORTS_MAPPING_PATTERN = - "^:[0-9]+|^[0-9]+:[0-9]+|^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]" + - "|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" + - ":[0-9]+:[0-9]+$"; - private static final int HOST_NAME_LENGTH = 64; + private static final String DEFAULT_PROCFS = "/proc"; @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_IMAGE = "YARN_CONTAINER_RUNTIME_DOCKER_IMAGE"; @InterfaceAudience.Private - public static final String ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE = - "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE"; - @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_NETWORK = "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK"; @InterfaceAudience.Private - public static final String ENV_DOCKER_CONTAINER_PID_NAMESPACE = - "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_PID_NAMESPACE"; - @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_HOSTNAME = "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME"; @InterfaceAudience.Private - public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER = - "YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER"; - @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_MOUNTS = "YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS"; @InterfaceAudience.Private @@ -261,26 +228,32 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { "YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL"; @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_PORTS_MAPPING = - "YARN_CONTAINER_RUNTIME_DOCKER_PORTS_MAPPING"; + "YARN_CONTAINER_RUNTIME_DOCKER_PORTS_MAPPING"; @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_YARN_SYSFS = "YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE"; @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_DOCKER_RUNTIME = "YARN_CONTAINER_RUNTIME_DOCKER_RUNTIME"; - public static final String YARN_SYSFS_PATH = - "/hadoop/yarn/sysfs"; + + @InterfaceAudience.Private + private static final String RUNTIME_TYPE = "DOCKER"; + @InterfaceAudience.Private + private final static String ENV_OCI_CONTAINER_PID_NAMESPACE = + formatOciEnvKey(RUNTIME_TYPE, CONTAINER_PID_NAMESPACE_SUFFIX); + @InterfaceAudience.Private + private final static String ENV_OCI_CONTAINER_RUN_PRIVILEGED_CONTAINER = + formatOciEnvKey(RUNTIME_TYPE, RUN_PRIVILEGED_CONTAINER_SUFFIX); + private Configuration conf; private Context nmContext; private DockerClient dockerClient; - private Map csiClients = new HashMap<>(); private PrivilegedOperationExecutor privilegedOperationExecutor; private String defaultImageName; private Boolean defaultImageUpdate; private Set allowedNetworks = new HashSet<>(); private Set allowedRuntimes = new HashSet<>(); private String defaultNetwork; - private String defaultRuntime; private CGroupsHandler cGroupsHandler; private AccessControlList privilegedContainersAcl; private boolean enableUserReMapping; @@ -338,6 +311,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { public DockerLinuxContainerRuntime( PrivilegedOperationExecutor privilegedOperationExecutor, CGroupsHandler cGroupsHandler) { + super(privilegedOperationExecutor, cGroupsHandler); + this.privilegedOperationExecutor = privilegedOperationExecutor; if (cGroupsHandler == null) { @@ -350,8 +325,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { @Override public void initialize(Configuration conf, Context nmContext) throws ContainerExecutionException { + super.initialize(conf, nmContext); this.nmContext = nmContext; this.conf = conf; + dockerClient = new DockerClient(); allowedNetworks.clear(); allowedRuntimes.clear(); @@ -423,10 +400,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { YarnConfiguration.NM_DOCKER_DEFAULT_TMPFS_MOUNTS))); } - public Map getCsiClients() { - return csiClients; - } - @Override public boolean isRuntimeRequested(Map env) { return isDockerContainerRequested(conf, env); @@ -479,12 +452,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { + dockerVolumeCommand, e); throw new ContainerExecutionException(e); } - - } - - @Override - public void prepareContainer(ContainerRuntimeContext ctx) - throws ContainerExecutionException { } private void checkDockerVolumeCreated( @@ -527,87 +494,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { throw new ContainerExecutionException(message); } - private void validateContainerNetworkType(String network) - throws ContainerExecutionException { - if (allowedNetworks.contains(network)) { - return; - } - - String msg = "Disallowed network: '" + network - + "' specified. Allowed networks: are " + allowedNetworks - .toString(); - throw new ContainerExecutionException(msg); - } - - private void validateContainerRuntimeType(String runtime) - throws ContainerExecutionException { - if (runtime == null || runtime.isEmpty() - || allowedRuntimes.contains(runtime)) { - return; - } - - String msg = "Disallowed runtime: '" + runtime - + "' specified. Allowed networks: are " + allowedRuntimes - .toString(); - throw new ContainerExecutionException(msg); - } - - /** - * Return whether the YARN container is allowed to run using the host's PID - * namespace for the Docker container. For this to be allowed, the submitting - * user must request the feature and the feature must be enabled on the - * cluster. - * - * @param container the target YARN container - * @return whether host pid namespace is requested and allowed - * @throws ContainerExecutionException if host pid namespace is requested - * but is not allowed - */ - private boolean allowHostPidNamespace(Container container) - throws ContainerExecutionException { - Map environment = container.getLaunchContext() - .getEnvironment(); - String pidNamespace = environment.get(ENV_DOCKER_CONTAINER_PID_NAMESPACE); - - if (pidNamespace == null) { - return false; - } - - if (!pidNamespace.equalsIgnoreCase("host")) { - LOG.warn("NOT requesting PID namespace. Value of " + - ENV_DOCKER_CONTAINER_PID_NAMESPACE + "is invalid: " + pidNamespace); - return false; - } - - boolean hostPidNamespaceEnabled = conf.getBoolean( - YarnConfiguration.NM_DOCKER_ALLOW_HOST_PID_NAMESPACE, - YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_HOST_PID_NAMESPACE); - - if (!hostPidNamespaceEnabled) { - String message = "Host pid namespace being requested but this is not " - + "enabled on this cluster"; - LOG.warn(message); - throw new ContainerExecutionException(message); - } - - return true; - } - - public static void validateHostname(String hostname) throws - ContainerExecutionException { - if (hostname != null && !hostname.isEmpty()) { - if (!hostnamePattern.matcher(hostname).matches()) { - throw new ContainerExecutionException("Hostname '" + hostname - + "' doesn't match docker hostname pattern"); - } - if (hostname.length() > HOST_NAME_LENGTH) { - throw new ContainerExecutionException( - "Hostname can not be greater than " + HOST_NAME_LENGTH - + " characters: " + hostname); - } - } - } - /** Set a DNS friendly hostname. * Only add hostname if network is not host or if hostname is * specified via YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME @@ -671,129 +557,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } } - /** - * Return whether the YARN container is allowed to run in a privileged - * Docker container. For a privileged container to be allowed all of the - * following three conditions must be satisfied: - * - *
      - *
    1. Submitting user must request for a privileged container
      2. - *
    2. Privileged containers must be enabled on the cluster
      3. - *
    3. Submitting user must be white-listed to run a privileged - * container
      4. - *
    - * - * @param container the target YARN container - * @return whether privileged container execution is allowed - * @throws ContainerExecutionException if privileged container execution - * is requested but is not allowed - */ - private boolean allowPrivilegedContainerExecution(Container container) - throws ContainerExecutionException { - - if(!isContainerRequestedAsPrivileged(container)) { - return false; - } - - LOG.info("Privileged container requested for : " + container - .getContainerId().toString()); - - //Ok, so we have been asked to run a privileged container. Security - // checks need to be run. Each violation is an error. - - //check if privileged containers are enabled. - boolean privilegedContainersEnabledOnCluster = conf.getBoolean( - YarnConfiguration.NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS, - YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS); - - if (!privilegedContainersEnabledOnCluster) { - String message = "Privileged container being requested but privileged " - + "containers are not enabled on this cluster"; - LOG.warn(message); - throw new ContainerExecutionException(message); - } - - //check if submitting user is in the whitelist. - String submittingUser = container.getUser(); - UserGroupInformation submitterUgi = UserGroupInformation - .createRemoteUser(submittingUser); - - if (!privilegedContainersAcl.isUserAllowed(submitterUgi)) { - String message = "Cannot launch privileged container. Submitting user (" - + submittingUser + ") fails ACL check."; - LOG.warn(message); - throw new ContainerExecutionException(message); - } - - LOG.info("All checks pass. Launching privileged container for : " - + container.getContainerId().toString()); - - return true; - } - - /** - * This function only returns whether a privileged container was requested, - * not whether the container was or will be launched as privileged. - * @param container - * @return - */ - private boolean isContainerRequestedAsPrivileged( - Container container) { - String runPrivilegedContainerEnvVar = container.getLaunchContext() - .getEnvironment().get(ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER); - return Boolean.parseBoolean(runPrivilegedContainerEnvVar); - } - - @VisibleForTesting - private String mountReadOnlyPath(String mount, - Map> localizedResources) - throws ContainerExecutionException { - for (Entry> resource : localizedResources.entrySet()) { - if (resource.getValue().contains(mount)) { - java.nio.file.Path path = Paths.get(resource.getKey().toString()); - if (!path.isAbsolute()) { - throw new ContainerExecutionException("Mount must be absolute: " + - mount); - } - if (Files.isSymbolicLink(path)) { - throw new ContainerExecutionException("Mount cannot be a symlink: " + - mount); - } - return path.toString(); - } - } - throw new ContainerExecutionException("Mount must be a localized " + - "resource: " + mount); - } - - private String getUserIdInfo(String userName) - throws ContainerExecutionException { - String id = ""; - Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor( - new String[]{"id", "-u", userName}); - try { - shexec.execute(); - id = shexec.getOutput().replaceAll("[^0-9]", ""); - } catch (Exception e) { - throw new ContainerExecutionException(e); - } - return id; - } - - private String[] getGroupIdInfo(String userName) - throws ContainerExecutionException { - String[] id = null; - Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor( - new String[]{"id", "-G", userName}); - try { - shexec.execute(); - id = shexec.getOutput().replace("\n", "").split(" "); - } catch (Exception e) { - throw new ContainerExecutionException(e); - } - return id; - } - /** * Check if system is default to disable docker override or * user requested a Docker container with ENTRY_POINT support. @@ -1140,7 +903,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } } - /** * Signal the docker container. * @@ -1217,7 +979,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } /** - * Perform docker exec command into running container + * Perform docker exec command into running container. * * @param ctx container exec context * @return IOStreams of docker exec @@ -1262,7 +1024,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { return output; } - // ipAndHost[0] contains comma separated list of IPs // ipAndHost[1] contains the hostname. @Override @@ -1287,7 +1048,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { String network; try { network = container.getLaunchContext().getEnvironment() - .get("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK"); + .get(ENV_DOCKER_CONTAINER_NETWORK); if (network == null || network.isEmpty()) { network = defaultNetwork; } @@ -1322,8 +1083,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } @Override - public String getExposedPorts(Container container) - throws ContainerExecutionException { + public String getExposedPorts(Container container) { ContainerId containerId = container.getContainerId(); String containerIdStr = containerId.toString(); DockerInspectCommand inspectCommand = @@ -1481,7 +1241,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } if (DockerCommandExecutor.isStoppable(containerStatus)) { - DockerKillCommand dockerStopCommand = new DockerKillCommand( containerId.toString()).setSignal(stopSignal); DockerCommandExecutor.executeDockerCommand(dockerStopCommand, @@ -1500,6 +1259,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { PrivilegedOperation privOp = new PrivilegedOperation( PrivilegedOperation.OperationType.RUN_DOCKER_CMD); privOp.appendArgs(commandFile); + String output = privilegedOperationExecutor.executePrivilegedOperation(null, privOp, null, null, true, false); LOG.info("{} : docker inspect output {} ", containerId, output); @@ -1614,33 +1374,36 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } } - /** - * Initiate CSI clients to talk to the CSI adaptors on this node and - * cache the clients for easier fetch. - * @param config configuration - * @throws ContainerExecutionException - */ - private void initiateCsiClients(Configuration config) - throws ContainerExecutionException { - String[] driverNames = CsiConfigUtils.getCsiDriverNames(config); - if (driverNames != null && driverNames.length > 0) { - for (String driverName : driverNames) { - try { - // find out the adaptors service address - InetSocketAddress adaptorServiceAddress = - CsiConfigUtils.getCsiAdaptorAddressForDriver(driverName, config); - LOG.info("Initializing a csi-adaptor-client for csi-adaptor {}," - + " csi-driver {}", adaptorServiceAddress.toString(), driverName); - CsiAdaptorProtocolPBClientImpl client = - new CsiAdaptorProtocolPBClientImpl(1L, adaptorServiceAddress, - config); - csiClients.put(driverName, client); - } catch (IOException e1) { - throw new ContainerExecutionException(e1.getMessage()); - } catch (YarnException e2) { - throw new ContainerExecutionException(e2.getMessage()); - } - } - } + boolean getHostPidNamespaceEnabled() { + return conf.getBoolean( + YarnConfiguration.NM_DOCKER_ALLOW_HOST_PID_NAMESPACE, + YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_HOST_PID_NAMESPACE); } + + boolean getPrivilegedContainersEnabledOnCluster() { + return conf.getBoolean( + YarnConfiguration.NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS, + YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS); + } + + Set getAllowedNetworks() { + return allowedNetworks; + } + + Set getAllowedRuntimes() { + return allowedRuntimes; + } + + AccessControlList getPrivilegedContainersAcl() { + return privilegedContainersAcl; + } + + String getEnvOciContainerPidNamespace() { + return ENV_OCI_CONTAINER_PID_NAMESPACE; + } + + String getEnvOciContainerRunPrivilegedContainer() { + return ENV_OCI_CONTAINER_RUN_PRIVILEGED_CONTAINER; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/OCIContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/OCIContainerRuntime.java new file mode 100644 index 00000000000..45105f7ad17 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/OCIContainerRuntime.java @@ -0,0 +1,374 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.CsiAdaptorProtocol; +import org.apache.hadoop.yarn.api.impl.pb.client.CsiAdaptorProtocolPBClientImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; +import org.apache.hadoop.yarn.util.csi.CsiConfigUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime.isDockerContainerRequested; + +/** + *

    This class is a {@link ContainerRuntime} implementation that uses the + * native {@code container-executor} binary via a + * {@link PrivilegedOperationExecutor} instance to launch processes inside + * OCI-compliant containers.

    + * + */ + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class OCIContainerRuntime implements LinuxContainerRuntime { + private static final Logger LOG = + LoggerFactory.getLogger(OCIContainerRuntime.class); + + private static final Pattern HOSTNAME_PATTERN = Pattern.compile( + "^[a-zA-Z0-9][a-zA-Z0-9_.-]+$"); + static final Pattern USER_MOUNT_PATTERN = Pattern.compile( + "(?<=^|,)([^:\\x00]+):([^:\\x00]+)" + + "(:(r[ow]|(r[ow][+])?(r?shared|r?slave|r?private)))?(?:,|$)"); + static final Pattern TMPFS_MOUNT_PATTERN = Pattern.compile( + "^/[^:\\x00]+$"); + static final String PORTS_MAPPING_PATTERN = + "^:[0-9]+|^[0-9]+:[0-9]+|^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]" + + "|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])" + + ":[0-9]+:[0-9]+$"; + private static final int HOST_NAME_LENGTH = 64; + + @InterfaceAudience.Private + public static final String RUNTIME_PREFIX = "YARN_CONTAINER_RUNTIME_%s_%s"; + @InterfaceAudience.Private + public static final String CONTAINER_PID_NAMESPACE_SUFFIX = + "CONTAINER_PID_NAMESPACE"; + @InterfaceAudience.Private + public static final String RUN_PRIVILEGED_CONTAINER_SUFFIX = + "RUN_PRIVILEGED_CONTAINER"; + + private Map csiClients = new HashMap<>(); + + abstract Set getAllowedNetworks(); + abstract Set getAllowedRuntimes(); + abstract boolean getHostPidNamespaceEnabled(); + abstract boolean getPrivilegedContainersEnabledOnCluster(); + abstract AccessControlList getPrivilegedContainersAcl(); + abstract String getEnvOciContainerPidNamespace(); + abstract String getEnvOciContainerRunPrivilegedContainer(); + + public OCIContainerRuntime(PrivilegedOperationExecutor + privilegedOperationExecutor) { + this(privilegedOperationExecutor, ResourceHandlerModule + .getCGroupsHandler()); + } + + public OCIContainerRuntime(PrivilegedOperationExecutor + privilegedOperationExecutor, CGroupsHandler cGroupsHandler) { + } + + public void initialize(Configuration conf, Context nmContext) + throws ContainerExecutionException { + + } + + public static boolean isOCICompliantContainerRequested( + Configuration daemonConf, Map env) { + return isDockerContainerRequested(daemonConf, env); + } + + @VisibleForTesting + protected String mountReadOnlyPath(String mount, + Map> localizedResources) + throws ContainerExecutionException { + for (Map.Entry> resource : + localizedResources.entrySet()) { + if (resource.getValue().contains(mount)) { + java.nio.file.Path path = Paths.get(resource.getKey().toString()); + if (!path.isAbsolute()) { + throw new ContainerExecutionException("Mount must be absolute: " + + mount); + } + if (Files.isSymbolicLink(path)) { + throw new ContainerExecutionException("Mount cannot be a symlink: " + + mount); + } + return path.toString(); + } + } + throw new ContainerExecutionException("Mount must be a localized " + + "resource: " + mount); + } + + @Override + public void prepareContainer(ContainerRuntimeContext ctx) + throws ContainerExecutionException { + } + + protected String getUserIdInfo(String userName) + throws ContainerExecutionException { + String id; + Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor( + new String[]{"id", "-u", userName}); + try { + shexec.execute(); + id = shexec.getOutput().replaceAll("[^0-9]", ""); + } catch (Exception e) { + throw new ContainerExecutionException(e); + } + return id; + } + + protected String[] getGroupIdInfo(String userName) + throws ContainerExecutionException { + String[] id; + Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor( + new String[]{"id", "-G", userName}); + try { + shexec.execute(); + id = shexec.getOutput().replace("\n", "").split(" "); + } catch (Exception e) { + throw new ContainerExecutionException(e); + } + return id; + } + + protected void validateContainerNetworkType(String network) + throws ContainerExecutionException { + Set allowedNetworks = getAllowedNetworks(); + if (allowedNetworks.contains(network)) { + return; + } + + String msg = "Disallowed network: '" + network + + "' specified. Allowed networks: are " + allowedNetworks + .toString(); + throw new ContainerExecutionException(msg); + } + + protected void validateContainerRuntimeType(String runtime) + throws ContainerExecutionException { + Set allowedRuntimes = getAllowedRuntimes(); + if (runtime == null || runtime.isEmpty() + || allowedRuntimes.contains(runtime)) { + return; + } + + String msg = "Disallowed runtime: '" + runtime + + "' specified. Allowed runtimes: are " + allowedRuntimes + .toString(); + throw new ContainerExecutionException(msg); + } + + /** + * Return whether the YARN container is allowed to run using the host's PID + * namespace for the OCI-compliant container. For this to be allowed, the + * submitting user must request the feature and the feature must be enabled + * on the cluster. + * + * @param container the target YARN container + * @return whether host pid namespace is requested and allowed + * @throws ContainerExecutionException if host pid namespace is requested + * but is not allowed + */ + protected boolean allowHostPidNamespace(Container container) + throws ContainerExecutionException { + Map environment = container.getLaunchContext() + .getEnvironment(); + String envOciContainerPidNamespace = getEnvOciContainerPidNamespace(); + + String pidNamespace = environment.get(envOciContainerPidNamespace); + + if (pidNamespace == null) { + return false; + } + + if (!pidNamespace.equalsIgnoreCase("host")) { + LOG.warn("NOT requesting PID namespace. Value of " + + envOciContainerPidNamespace + + "is invalid: " + pidNamespace); + return false; + } + + boolean hostPidNamespaceEnabled = getHostPidNamespaceEnabled(); + + if (!hostPidNamespaceEnabled) { + String message = "Host pid namespace being requested but this is not " + + "enabled on this cluster"; + LOG.warn(message); + throw new ContainerExecutionException(message); + } + + return true; + } + + + protected static void validateHostname(String hostname) throws + ContainerExecutionException { + if (hostname != null && !hostname.isEmpty()) { + if (!HOSTNAME_PATTERN.matcher(hostname).matches()) { + throw new ContainerExecutionException("Hostname '" + hostname + + "' doesn't match OCI-compliant hostname pattern"); + } + if (hostname.length() > HOST_NAME_LENGTH) { + throw new ContainerExecutionException( + "Hostname can not be greater than " + HOST_NAME_LENGTH + + " characters: " + hostname); + } + } + } + + /** + * Return whether the YARN container is allowed to run in a privileged + * OCI-compliant container. For a privileged container to be allowed all of + * the following three conditions must be satisfied: + * + *
      + *
    1. Submitting user must request for a privileged container
      2. + *
    2. Privileged containers must be enabled on the cluster
      3. + *
    3. Submitting user must be white-listed to run a privileged + * container
      4. + *
    + * + * @param container the target YARN container + * @return whether privileged container execution is allowed + * @throws ContainerExecutionException if privileged container execution + * is requested but is not allowed + */ + protected boolean allowPrivilegedContainerExecution(Container container) + throws ContainerExecutionException { + + if(!isContainerRequestedAsPrivileged(container)) { + return false; + } + + LOG.info("Privileged container requested for : " + container + .getContainerId().toString()); + + //Ok, so we have been asked to run a privileged container. Security + // checks need to be run. Each violation is an error. + + //check if privileged containers are enabled. + boolean privilegedContainersEnabledOnCluster = + getPrivilegedContainersEnabledOnCluster(); + + if (!privilegedContainersEnabledOnCluster) { + String message = "Privileged container being requested but privileged " + + "containers are not enabled on this cluster"; + LOG.warn(message); + throw new ContainerExecutionException(message); + } + + //check if submitting user is in the whitelist. + String submittingUser = container.getUser(); + UserGroupInformation submitterUgi = UserGroupInformation + .createRemoteUser(submittingUser); + + if (!getPrivilegedContainersAcl().isUserAllowed(submitterUgi)) { + String message = "Cannot launch privileged container. Submitting user (" + + submittingUser + ") fails ACL check."; + LOG.warn(message); + throw new ContainerExecutionException(message); + } + + LOG.info("All checks pass. Launching privileged container for : " + + container.getContainerId().toString()); + + return true; + } + + /** + * This function only returns whether a privileged container was requested, + * not whether the container was or will be launched as privileged. + * @param container + * @return true if container is requested as privileged + */ + protected boolean isContainerRequestedAsPrivileged( + Container container) { + String envOciContainerRunPrivilegedContainer = + getEnvOciContainerRunPrivilegedContainer(); + String runPrivilegedContainerEnvVar = container.getLaunchContext() + .getEnvironment().get(envOciContainerRunPrivilegedContainer); + return Boolean.parseBoolean(runPrivilegedContainerEnvVar); + } + + public Map getCsiClients() { + return csiClients; + } + + /** + * Initiate CSI clients to talk to the CSI adaptors on this node and + * cache the clients for easier fetch. + * @param config configuration + * @throws ContainerExecutionException + */ + protected void initiateCsiClients(Configuration config) + throws ContainerExecutionException { + String[] driverNames = CsiConfigUtils.getCsiDriverNames(config); + if (driverNames != null && driverNames.length > 0) { + for (String driverName : driverNames) { + try { + // find out the adaptors service address + InetSocketAddress adaptorServiceAddress = + CsiConfigUtils.getCsiAdaptorAddressForDriver(driverName, config); + LOG.info("Initializing a csi-adaptor-client for csi-adaptor {}," + + " csi-driver {}", adaptorServiceAddress.toString(), driverName); + CsiAdaptorProtocolPBClientImpl client = + new CsiAdaptorProtocolPBClientImpl(1L, adaptorServiceAddress, + config); + csiClients.put(driverName, client); + } catch (IOException | YarnException e1) { + throw new ContainerExecutionException(e1.getMessage()); + } + } + } + } + + public static String formatOciEnvKey(String runtimeTypeUpper, + String envKeySuffix) { + return String.format(RUNTIME_PREFIX, runtimeTypeUpper, envKeySuffix); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java index 97ff94f7880..03a22f2d743 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceResourceHandlerImpl.java @@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime; import java.io.IOException; import java.util.ArrayList; @@ -153,7 +153,7 @@ public class DeviceResourceHandlerImpl implements ResourceHandler { cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES, containerIdStr); // non-Docker, use cgroups to do isolation - if (!DockerLinuxContainerRuntime.isDockerContainerRequested( + if (!OCIContainerRuntime.isOCICompliantContainerRequested( nmContext.getConf(), container.getLaunchContext().getEnvironment())) { tryIsolateDevices(allocation, containerIdStr); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 5d4d22ebd28..8ab9df6321b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -91,7 +91,6 @@ import java.util.concurrent.ConcurrentMap; import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_DOCKER_DEFAULT_RO_MOUNTS; import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_DOCKER_DEFAULT_RW_MOUNTS; import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_DOCKER_DEFAULT_TMPFS_MOUNTS; -import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.APPID; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.APPLICATION_LOCAL_DIRS; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR; @@ -113,6 +112,9 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.SIGNAL; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER_FILECACHE_DIRS; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime.CONTAINER_PID_NAMESPACE_SUFFIX; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime.RUN_PRIVILEGED_CONTAINER_SUFFIX; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.OCIContainerRuntime.formatOciEnvKey; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; @@ -163,6 +165,13 @@ public class TestDockerContainerRuntime { private final String whitelistedUser = "yoda"; private String[] testCapabilities; private final String signalPid = "1234"; + private String runtimeTypeUpper = "DOCKER"; + + private static final String RUNTIME_TYPE = "DOCKER"; + private final static String ENV_OCI_CONTAINER_PID_NAMESPACE = + formatOciEnvKey(RUNTIME_TYPE, CONTAINER_PID_NAMESPACE_SUFFIX); + private final static String ENV_OCI_CONTAINER_RUN_PRIVILEGED_CONTAINER = + formatOciEnvKey(RUNTIME_TYPE, RUN_PRIVILEGED_CONTAINER_SUFFIX); @Rule public TemporaryFolder tempDir = new TemporaryFolder(); @@ -989,8 +998,7 @@ public class TestDockerContainerRuntime { mockExecutor, mockCGroupsHandler); runtime.initialize(conf, nmContext); - env.put(DockerLinuxContainerRuntime - .ENV_DOCKER_CONTAINER_PID_NAMESPACE, "invalid-value"); + env.put(ENV_OCI_CONTAINER_PID_NAMESPACE, "invalid-value"); runtime.launchContainer(builder.build()); List dockerCommands = readDockerCommands(); @@ -1011,8 +1019,7 @@ public class TestDockerContainerRuntime { mockExecutor, mockCGroupsHandler); runtime.initialize(conf, nmContext); - env.put(DockerLinuxContainerRuntime - .ENV_DOCKER_CONTAINER_PID_NAMESPACE, "host"); + env.put(ENV_OCI_CONTAINER_PID_NAMESPACE, "host"); try { runtime.launchContainer(builder.build()); @@ -1034,8 +1041,7 @@ public class TestDockerContainerRuntime { mockExecutor, mockCGroupsHandler); runtime.initialize(conf, nmContext); - env.put(DockerLinuxContainerRuntime - .ENV_DOCKER_CONTAINER_PID_NAMESPACE, "host"); + env.put(ENV_OCI_CONTAINER_PID_NAMESPACE, "host"); runtime.launchContainer(builder.build()); List dockerCommands = readDockerCommands(); @@ -1081,8 +1087,7 @@ public class TestDockerContainerRuntime { mockExecutor, mockCGroupsHandler); runtime.initialize(conf, nmContext); - env.put(DockerLinuxContainerRuntime - .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "invalid-value"); + env.put(ENV_OCI_CONTAINER_RUN_PRIVILEGED_CONTAINER, "invalid-value"); runtime.launchContainer(builder.build()); List dockerCommands = readDockerCommands(); @@ -1105,8 +1110,7 @@ public class TestDockerContainerRuntime { mockExecutor, mockCGroupsHandler); runtime.initialize(conf, nmContext); - env.put(DockerLinuxContainerRuntime - .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); + env.put(ENV_OCI_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); try { runtime.launchContainer(builder.build()); @@ -1128,8 +1132,7 @@ public class TestDockerContainerRuntime { mockExecutor, mockCGroupsHandler); runtime.initialize(conf, nmContext); - env.put(DockerLinuxContainerRuntime - .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); + env.put(ENV_OCI_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); //By default // yarn.nodemanager.runtime.linux.docker.privileged-containers.acl // is empty. So we expect this launch to fail. @@ -1157,8 +1160,7 @@ public class TestDockerContainerRuntime { mockExecutor, mockCGroupsHandler); runtime.initialize(conf, nmContext); - env.put(DockerLinuxContainerRuntime - .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); + env.put(ENV_OCI_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); try { runtime.launchContainer(builder.build()); @@ -1184,8 +1186,7 @@ public class TestDockerContainerRuntime { mockExecutor, mockCGroupsHandler); runtime.initialize(conf, nmContext); - env.put(DockerLinuxContainerRuntime - .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); + env.put(ENV_OCI_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); runtime.launchContainer(builder.build()); List dockerCommands = readDockerCommands(); @@ -1853,7 +1854,7 @@ public class TestDockerContainerRuntime { conf.set(YarnConfiguration.NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS, "true"); conf.set(YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL, submittingUser); - env.put(ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); + env.put(ENV_OCI_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); when(mockExecutor .executePrivilegedOperation(any(), any(PrivilegedOperation.class), any(), any(), anyBoolean(), anyBoolean())).thenReturn( @@ -1870,7 +1871,7 @@ public class TestDockerContainerRuntime { conf.set(YarnConfiguration.NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS, "true"); conf.set(YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL, submittingUser); - env.put(ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); + env.put(ENV_OCI_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); when(mockExecutor .executePrivilegedOperation(any(), any(PrivilegedOperation.class), any(), any(), anyBoolean(), anyBoolean())).thenReturn( @@ -1892,7 +1893,7 @@ public class TestDockerContainerRuntime { conf.set(YarnConfiguration.NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS, "true"); conf.set(YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL, submittingUser); - env.put(ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); + env.put(ENV_OCI_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); when(mockExecutor .executePrivilegedOperation(any(), any(PrivilegedOperation.class), any(), any(), anyBoolean(), anyBoolean())).thenReturn(