YARN-6456. Added config to set default container runtimes.
Contributed by Craig Condit
This commit is contained in:
parent
184544eff8
commit
b237a0dd44
|
@ -1882,9 +1882,17 @@ public class YarnConfiguration extends Configuration {
|
|||
public static final String[] DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES
|
||||
= {"default"};
|
||||
|
||||
/** Default runtime to be used. */
|
||||
public static final String LINUX_CONTAINER_RUNTIME_TYPE =
|
||||
LINUX_CONTAINER_RUNTIME_PREFIX + "type";
|
||||
|
||||
public static final String DOCKER_CONTAINER_RUNTIME_PREFIX =
|
||||
LINUX_CONTAINER_RUNTIME_PREFIX + "docker.";
|
||||
|
||||
/** Default docker image to be used. */
|
||||
public static final String NM_DOCKER_IMAGE_NAME =
|
||||
DOCKER_CONTAINER_RUNTIME_PREFIX + "image-name";
|
||||
|
||||
/** Capabilities allowed (and added by default) for docker containers. **/
|
||||
public static final String NM_DOCKER_CONTAINER_CAPABILITIES =
|
||||
DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities";
|
||||
|
|
|
@ -1725,6 +1725,12 @@
|
|||
<value>default</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>Default container runtime to use.</description>
|
||||
<name>yarn.nodemanager.runtime.linux.type</name>
|
||||
<value></value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>This configuration setting determines the capabilities
|
||||
assigned to docker containers when they are launched. While these may not
|
||||
|
@ -1735,6 +1741,13 @@
|
|||
<value>CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>Default docker image to be used when the docker runtime is
|
||||
selected.</description>
|
||||
<name>yarn.nodemanager.runtime.linux.docker.image-name</name>
|
||||
<value></value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>This configuration setting determines if privileged docker
|
||||
containers are allowed on this cluster. Privileged containers are granted
|
||||
|
|
|
@ -1544,6 +1544,7 @@ public class ContainerImpl implements Container {
|
|||
// TODO: Add containerWorkDir to the deletion service.
|
||||
|
||||
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
|
||||
container.daemonConf,
|
||||
container.getLaunchContext().getEnvironment())) {
|
||||
removeDockerContainer(container);
|
||||
}
|
||||
|
@ -1584,6 +1585,7 @@ public class ContainerImpl implements Container {
|
|||
// TODO: Add containerOuputDir to the deletion service.
|
||||
|
||||
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
|
||||
container.daemonConf,
|
||||
container.getLaunchContext().getEnvironment())) {
|
||||
removeDockerContainer(container);
|
||||
}
|
||||
|
@ -1858,6 +1860,7 @@ public class ContainerImpl implements Container {
|
|||
}
|
||||
|
||||
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
|
||||
container.daemonConf,
|
||||
container.getLaunchContext().getEnvironment())) {
|
||||
removeDockerContainer(container);
|
||||
}
|
||||
|
|
|
@ -799,6 +799,7 @@ public class ContainerLaunch implements Callable<Integer> {
|
|||
}
|
||||
// The Docker container may not have fully started, reap the container.
|
||||
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
|
||||
conf,
|
||||
container.getLaunchContext().getEnvironment())) {
|
||||
reapDockerContainerNoPid(user);
|
||||
}
|
||||
|
|
|
@ -48,6 +48,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
|
|||
public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus";
|
||||
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
|
||||
|
||||
private Context nmContext;
|
||||
private GpuResourceAllocator gpuAllocator;
|
||||
private CGroupsHandler cGroupsHandler;
|
||||
private PrivilegedOperationExecutor privilegedOperationExecutor;
|
||||
|
@ -55,6 +56,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
|
|||
public GpuResourceHandlerImpl(Context nmContext,
|
||||
CGroupsHandler cGroupsHandler,
|
||||
PrivilegedOperationExecutor privilegedOperationExecutor) {
|
||||
this.nmContext = nmContext;
|
||||
this.cGroupsHandler = cGroupsHandler;
|
||||
this.privilegedOperationExecutor = privilegedOperationExecutor;
|
||||
gpuAllocator = new GpuResourceAllocator(nmContext);
|
||||
|
@ -102,6 +104,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
|
|||
cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES,
|
||||
containerIdStr);
|
||||
if (!DockerLinuxContainerRuntime.isDockerContainerRequested(
|
||||
nmContext.getConf(),
|
||||
container.getLaunchContext().getEnvironment())) {
|
||||
// Write to devices cgroup only for non-docker container. The reason is
|
||||
// docker engine runtime runc do the devices cgroups initialize in the
|
||||
|
|
|
@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
|||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.Context;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
|
||||
|
@ -71,7 +72,10 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
@Override
|
||||
public boolean isRuntimeRequested(Map<String, String> env) {
|
||||
String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
|
||||
return type == null || type.equals("default");
|
||||
if (type == null) {
|
||||
type = conf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
|
||||
}
|
||||
return type == null || type.isEmpty() || type.equals("default");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -235,6 +235,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
private Context nmContext;
|
||||
private DockerClient dockerClient;
|
||||
private PrivilegedOperationExecutor privilegedOperationExecutor;
|
||||
private String defaultImageName;
|
||||
private Set<String> allowedNetworks = new HashSet<>();
|
||||
private String defaultNetwork;
|
||||
private CGroupsHandler cGroupsHandler;
|
||||
|
@ -254,17 +255,17 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
* called {@code YARN_CONTAINER_RUNTIME_TYPE} whose value is {@code docker},
|
||||
* this method will return true. Otherwise it will return false.
|
||||
*
|
||||
* @param daemonConf the NodeManager daemon configuration
|
||||
* @param env the environment variable settings for the operation
|
||||
* @return whether a Docker container is requested
|
||||
*/
|
||||
public static boolean isDockerContainerRequested(
|
||||
public static boolean isDockerContainerRequested(Configuration daemonConf,
|
||||
Map<String, String> env) {
|
||||
if (env == null) {
|
||||
return false;
|
||||
String type = (env == null)
|
||||
? null : env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
|
||||
if (type == null) {
|
||||
type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
|
||||
}
|
||||
|
||||
String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
|
||||
|
||||
return type != null && type.equals("docker");
|
||||
}
|
||||
|
||||
|
@ -312,6 +313,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
defaultROMounts.clear();
|
||||
defaultRWMounts.clear();
|
||||
defaultTmpfsMounts.clear();
|
||||
defaultImageName = conf.getTrimmed(
|
||||
YarnConfiguration.NM_DOCKER_IMAGE_NAME, "");
|
||||
allowedNetworks.addAll(Arrays.asList(
|
||||
conf.getTrimmedStrings(
|
||||
YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
|
||||
|
@ -325,8 +328,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
+ " is not in the set of allowed networks: " + allowedNetworks;
|
||||
|
||||
if (LOG.isWarnEnabled()) {
|
||||
LOG.warn(message + ". Please check "
|
||||
+ "configuration");
|
||||
LOG.warn(message + ". Please check configuration");
|
||||
}
|
||||
|
||||
throw new ContainerExecutionException(message);
|
||||
|
@ -369,7 +371,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
|
||||
@Override
|
||||
public boolean isRuntimeRequested(Map<String, String> env) {
|
||||
return isDockerContainerRequested(env);
|
||||
return isDockerContainerRequested(conf, env);
|
||||
}
|
||||
|
||||
private Set<String> getDockerCapabilitiesFromConf() throws
|
||||
|
@ -789,6 +791,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
|
||||
boolean useEntryPoint = checkUseEntryPoint(environment);
|
||||
|
||||
if (imageName == null || imageName.isEmpty()) {
|
||||
imageName = defaultImageName;
|
||||
}
|
||||
if(network == null || network.isEmpty()) {
|
||||
network = defaultNetwork;
|
||||
}
|
||||
|
|
|
@ -80,8 +80,11 @@ public class TestGpuResourceHandler {
|
|||
mockPrivilegedExecutor = mock(PrivilegedOperationExecutor.class);
|
||||
mockNMStateStore = mock(NMStateStoreService.class);
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
|
||||
Context nmctx = mock(Context.class);
|
||||
when(nmctx.getNMStateStore()).thenReturn(mockNMStateStore);
|
||||
when(nmctx.getConf()).thenReturn(conf);
|
||||
runningContainersMap = new ConcurrentHashMap<>();
|
||||
when(nmctx.getContainers()).thenReturn(runningContainersMap);
|
||||
|
||||
|
@ -347,15 +350,17 @@ public class TestGpuResourceHandler {
|
|||
public void testAllocationStoredWithNULLStateStore() throws Exception {
|
||||
NMNullStateStoreService mockNMNULLStateStore = mock(NMNullStateStoreService.class);
|
||||
|
||||
Configuration conf = new YarnConfiguration();
|
||||
conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
|
||||
|
||||
Context nmnctx = mock(Context.class);
|
||||
when(nmnctx.getNMStateStore()).thenReturn(mockNMNULLStateStore);
|
||||
when(nmnctx.getConf()).thenReturn(conf);
|
||||
|
||||
GpuResourceHandlerImpl gpuNULLStateResourceHandler =
|
||||
new GpuResourceHandlerImpl(nmnctx, mockCGroupsHandler,
|
||||
mockPrivilegedExecutor);
|
||||
|
||||
Configuration conf = new YarnConfiguration();
|
||||
conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
|
||||
GpuDiscoverer.getInstance().initialize(conf);
|
||||
|
||||
gpuNULLStateResourceHandler.bootstrap(conf);
|
||||
|
|
|
@ -309,11 +309,45 @@ public class TestDockerContainerRuntime {
|
|||
envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
|
||||
|
||||
Assert.assertEquals(false, DockerLinuxContainerRuntime
|
||||
.isDockerContainerRequested(null));
|
||||
.isDockerContainerRequested(conf, null));
|
||||
Assert.assertEquals(true, DockerLinuxContainerRuntime
|
||||
.isDockerContainerRequested(envDockerType));
|
||||
.isDockerContainerRequested(conf, envDockerType));
|
||||
Assert.assertEquals(false, DockerLinuxContainerRuntime
|
||||
.isDockerContainerRequested(envOtherType));
|
||||
.isDockerContainerRequested(conf, envOtherType));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSelectDockerContainerTypeWithDockerAsDefault() {
|
||||
Map<String, String> envDockerType = new HashMap<>();
|
||||
Map<String, String> envOtherType = new HashMap<>();
|
||||
|
||||
conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "docker");
|
||||
envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
|
||||
envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
|
||||
|
||||
Assert.assertEquals(true, DockerLinuxContainerRuntime
|
||||
.isDockerContainerRequested(conf, null));
|
||||
Assert.assertEquals(true, DockerLinuxContainerRuntime
|
||||
.isDockerContainerRequested(conf, envDockerType));
|
||||
Assert.assertEquals(false, DockerLinuxContainerRuntime
|
||||
.isDockerContainerRequested(conf, envOtherType));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSelectDockerContainerTypeWithDefaultSet() {
|
||||
Map<String, String> envDockerType = new HashMap<>();
|
||||
Map<String, String> envOtherType = new HashMap<>();
|
||||
|
||||
conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "default");
|
||||
envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
|
||||
envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
|
||||
|
||||
Assert.assertEquals(false, DockerLinuxContainerRuntime
|
||||
.isDockerContainerRequested(conf, null));
|
||||
Assert.assertEquals(true, DockerLinuxContainerRuntime
|
||||
.isDockerContainerRequested(conf, envDockerType));
|
||||
Assert.assertEquals(false, DockerLinuxContainerRuntime
|
||||
.isDockerContainerRequested(conf, envOtherType));
|
||||
}
|
||||
|
||||
private PrivilegedOperation capturePrivilegedOperation()
|
||||
|
@ -421,6 +455,57 @@ public class TestDockerContainerRuntime {
|
|||
dockerCommands.get(counter));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDockerContainerLaunchWithDefaultImage()
|
||||
throws ContainerExecutionException, PrivilegedOperationException,
|
||||
IOException {
|
||||
conf.set(YarnConfiguration.NM_DOCKER_IMAGE_NAME, "busybox:1.2.3");
|
||||
env.remove(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE);
|
||||
|
||||
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
|
||||
mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf, nmContext);
|
||||
runtime.launchContainer(builder.build());
|
||||
|
||||
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
|
||||
List<String> args = op.getArguments();
|
||||
String dockerCommandFile = args.get(11);
|
||||
|
||||
List<String> dockerCommands = Files.readAllLines(Paths.get(
|
||||
dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
int expected = 13;
|
||||
int counter = 0;
|
||||
Assert.assertEquals(expected, dockerCommands.size());
|
||||
Assert.assertEquals("[docker-command-execution]",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" group-add=" + String.join(",", groups),
|
||||
dockerCommands.get(counter++));
|
||||
Assert
|
||||
.assertEquals(" image=busybox:1.2.3", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
" launch-command=bash,/test_container_work_dir/launch_container.sh",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" mounts="
|
||||
+ "/test_container_log_dir:/test_container_log_dir:rw,"
|
||||
+ "/test_application_local_dir:/test_application_local_dir:rw,"
|
||||
+ "/test_filecache_dir:/test_filecache_dir:ro,"
|
||||
+ "/test_user_filecache_dir:/test_user_filecache_dir:ro",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(
|
||||
" name=container_e11_1518975676334_14532816_01_000001",
|
||||
dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
|
||||
Assert.assertEquals(" workdir=/test_container_work_dir",
|
||||
dockerCommands.get(counter));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testContainerLaunchWithUserRemapping()
|
||||
throws ContainerExecutionException, PrivilegedOperationException,
|
||||
|
|
|
@ -114,6 +114,23 @@ The following properties should be set in yarn-site.xml:
|
|||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.runtime.linux.type</name>
|
||||
<value></value>
|
||||
<description>
|
||||
Optional. Sets the default container runtime to use.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.runtime.linux.docker.image-name</name>
|
||||
<value></value>
|
||||
<description>
|
||||
Optional. Default docker image to be used when the docker runtime is
|
||||
selected.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.runtime.linux.docker.allowed-container-networks</name>
|
||||
<value>host,none,bridge</value>
|
||||
|
|
Loading…
Reference in New Issue