YARN-6456. Added config to set default container runtimes.

Contributed by Craig Condit

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -1882,9 +1882,17 @@ public class YarnConfiguration extends Configuration {
   public static final String[] DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES
       = {"default"};
 
+  /** Default runtime to be used. */
+  public static final String LINUX_CONTAINER_RUNTIME_TYPE =
+      LINUX_CONTAINER_RUNTIME_PREFIX + "type";
+
   public static final String DOCKER_CONTAINER_RUNTIME_PREFIX =
       LINUX_CONTAINER_RUNTIME_PREFIX + "docker.";
 
+  /** Default docker image to be used. */
+  public static final String NM_DOCKER_IMAGE_NAME =
+      DOCKER_CONTAINER_RUNTIME_PREFIX + "image-name";
+
   /** Capabilities allowed (and added by default) for docker containers. **/
   public static final String NM_DOCKER_CONTAINER_CAPABILITIES =
       DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities";

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -1725,6 +1725,12 @@
     <value>default</value>
   </property>
 
+  <property>
+    <description>Default container runtime to use.</description>
+    <name>yarn.nodemanager.runtime.linux.type</name>
+    <value></value>
+  </property>
+
   <property>
     <description>This configuration setting determines the capabilities
       assigned to docker containers when they are launched. While these may not
@@ -1735,6 +1741,13 @@
     <value>CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE</value>
   </property>
 
+  <property>
+    <description>Default docker image to be used when the docker runtime is
+    selected.</description>
+    <name>yarn.nodemanager.runtime.linux.docker.image-name</name>
+    <value></value>
+  </property>
+
   <property>
     <description>This configuration setting determines if privileged docker
       containers are allowed on this cluster. Privileged containers are granted

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java

@@ -1544,6 +1544,7 @@ public class ContainerImpl implements Container {
       // TODO: Add containerWorkDir to the deletion service.
 
       if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+          container.daemonConf,
           container.getLaunchContext().getEnvironment())) {
         removeDockerContainer(container);
       }
@@ -1584,6 +1585,7 @@ public class ContainerImpl implements Container {
       // TODO: Add containerOuputDir to the deletion service.
 
       if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+          container.daemonConf,
           container.getLaunchContext().getEnvironment())) {
         removeDockerContainer(container);
       }
@@ -1858,6 +1860,7 @@ public class ContainerImpl implements Container {
       }
 
       if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+          container.daemonConf,
           container.getLaunchContext().getEnvironment())) {
         removeDockerContainer(container);
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java

@@ -799,6 +799,7 @@ public class ContainerLaunch implements Callable<Integer> {
         }
         // The Docker container may not have fully started, reap the container.
         if (DockerLinuxContainerRuntime.isDockerContainerRequested(
+            conf,
             container.getLaunchContext().getEnvironment())) {
           reapDockerContainerNoPid(user);
         }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java

@@ -48,6 +48,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
   public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus";
   public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
 
+  private Context nmContext;
   private GpuResourceAllocator gpuAllocator;
   private CGroupsHandler cGroupsHandler;
   private PrivilegedOperationExecutor privilegedOperationExecutor;
@@ -55,6 +56,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
   public GpuResourceHandlerImpl(Context nmContext,
       CGroupsHandler cGroupsHandler,
       PrivilegedOperationExecutor privilegedOperationExecutor) {
+    this.nmContext = nmContext;
     this.cGroupsHandler = cGroupsHandler;
     this.privilegedOperationExecutor = privilegedOperationExecutor;
     gpuAllocator = new GpuResourceAllocator(nmContext);
@@ -102,6 +104,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
     cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES,
         containerIdStr);
     if (!DockerLinuxContainerRuntime.isDockerContainerRequested(
+        nmContext.getConf(),
         container.getLaunchContext().getEnvironment())) {
       // Write to devices cgroup only for non-docker container. The reason is
       // docker engine runtime runc do the devices cgroups initialize in the

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java

@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -71,7 +72,10 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
   @Override
   public boolean isRuntimeRequested(Map<String, String> env) {
     String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
-    return type == null || type.equals("default");
+    if (type == null) {
+      type = conf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
+    }
+    return type == null || type.isEmpty() || type.equals("default");
   }
 
   @Override

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java

@@ -235,6 +235,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
   private Context nmContext;
   private DockerClient dockerClient;
   private PrivilegedOperationExecutor privilegedOperationExecutor;
+  private String defaultImageName;
   private Set<String> allowedNetworks = new HashSet<>();
   private String defaultNetwork;
   private CGroupsHandler cGroupsHandler;
@@ -254,17 +255,17 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
    * called {@code YARN_CONTAINER_RUNTIME_TYPE} whose value is {@code docker},
    * this method will return true.  Otherwise it will return false.
    *
+   * @param daemonConf the NodeManager daemon configuration
    * @param env the environment variable settings for the operation
    * @return whether a Docker container is requested
    */
-  public static boolean isDockerContainerRequested(
+  public static boolean isDockerContainerRequested(Configuration daemonConf,
       Map<String, String> env) {
-    if (env == null) {
+    String type = (env == null)
-      return false;
+        ? null : env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
+    if (type == null) {
+      type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
     }
-
-    String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
-
     return type != null && type.equals("docker");
   }
 
@@ -312,6 +313,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
     defaultROMounts.clear();
     defaultRWMounts.clear();
     defaultTmpfsMounts.clear();
+    defaultImageName = conf.getTrimmed(
+        YarnConfiguration.NM_DOCKER_IMAGE_NAME, "");
     allowedNetworks.addAll(Arrays.asList(
         conf.getTrimmedStrings(
             YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
@@ -325,8 +328,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
           + " is not in the set of allowed networks: " + allowedNetworks;
 
       if (LOG.isWarnEnabled()) {
-        LOG.warn(message + ". Please check "
+        LOG.warn(message + ". Please check configuration");
-            + "configuration");
       }
 
       throw new ContainerExecutionException(message);
@@ -369,7 +371,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
 
   @Override
   public boolean isRuntimeRequested(Map<String, String> env) {
-    return isDockerContainerRequested(env);
+    return isDockerContainerRequested(conf, env);
   }
 
   private Set<String> getDockerCapabilitiesFromConf() throws
@@ -789,6 +791,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
     String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
     boolean useEntryPoint = checkUseEntryPoint(environment);
 
+    if (imageName == null || imageName.isEmpty()) {
+      imageName = defaultImageName;
+    }
     if(network == null || network.isEmpty()) {
       network = defaultNetwork;
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java

@@ -80,8 +80,11 @@ public class TestGpuResourceHandler {
     mockPrivilegedExecutor = mock(PrivilegedOperationExecutor.class);
     mockNMStateStore = mock(NMStateStoreService.class);
 
+    Configuration conf = new Configuration();
+
     Context nmctx = mock(Context.class);
     when(nmctx.getNMStateStore()).thenReturn(mockNMStateStore);
+    when(nmctx.getConf()).thenReturn(conf);
     runningContainersMap = new ConcurrentHashMap<>();
     when(nmctx.getContainers()).thenReturn(runningContainersMap);
 
@@ -347,15 +350,17 @@ public class TestGpuResourceHandler {
   public void testAllocationStoredWithNULLStateStore() throws Exception {
     NMNullStateStoreService mockNMNULLStateStore = mock(NMNullStateStoreService.class);
 
+    Configuration conf = new YarnConfiguration();
+    conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
+
     Context nmnctx = mock(Context.class);
     when(nmnctx.getNMStateStore()).thenReturn(mockNMNULLStateStore);
+    when(nmnctx.getConf()).thenReturn(conf);
 
     GpuResourceHandlerImpl gpuNULLStateResourceHandler =
         new GpuResourceHandlerImpl(nmnctx, mockCGroupsHandler,
         mockPrivilegedExecutor);
 
-    Configuration conf = new YarnConfiguration();
-    conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
     GpuDiscoverer.getInstance().initialize(conf);
 
     gpuNULLStateResourceHandler.bootstrap(conf);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java

@@ -309,11 +309,45 @@ public class TestDockerContainerRuntime {
     envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
 
     Assert.assertEquals(false, DockerLinuxContainerRuntime
-        .isDockerContainerRequested(null));
+        .isDockerContainerRequested(conf, null));
     Assert.assertEquals(true, DockerLinuxContainerRuntime
-        .isDockerContainerRequested(envDockerType));
+        .isDockerContainerRequested(conf, envDockerType));
     Assert.assertEquals(false, DockerLinuxContainerRuntime
-        .isDockerContainerRequested(envOtherType));
+        .isDockerContainerRequested(conf, envOtherType));
+  }
+
+  @Test
+  public void testSelectDockerContainerTypeWithDockerAsDefault() {
+    Map<String, String> envDockerType = new HashMap<>();
+    Map<String, String> envOtherType = new HashMap<>();
+
+    conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "docker");
+    envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
+    envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
+
+    Assert.assertEquals(true, DockerLinuxContainerRuntime
+        .isDockerContainerRequested(conf, null));
+    Assert.assertEquals(true, DockerLinuxContainerRuntime
+        .isDockerContainerRequested(conf, envDockerType));
+    Assert.assertEquals(false, DockerLinuxContainerRuntime
+        .isDockerContainerRequested(conf, envOtherType));
+  }
+
+  @Test
+  public void testSelectDockerContainerTypeWithDefaultSet() {
+    Map<String, String> envDockerType = new HashMap<>();
+    Map<String, String> envOtherType = new HashMap<>();
+
+    conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "default");
+    envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
+    envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
+
+    Assert.assertEquals(false, DockerLinuxContainerRuntime
+        .isDockerContainerRequested(conf, null));
+    Assert.assertEquals(true, DockerLinuxContainerRuntime
+        .isDockerContainerRequested(conf, envDockerType));
+    Assert.assertEquals(false, DockerLinuxContainerRuntime
+        .isDockerContainerRequested(conf, envOtherType));
   }
 
   private PrivilegedOperation capturePrivilegedOperation()
@@ -421,6 +455,57 @@ public class TestDockerContainerRuntime {
         dockerCommands.get(counter));
   }
 
+  @Test
+  public void testDockerContainerLaunchWithDefaultImage()
+      throws ContainerExecutionException, PrivilegedOperationException,
+      IOException {
+    conf.set(YarnConfiguration.NM_DOCKER_IMAGE_NAME, "busybox:1.2.3");
+    env.remove(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE);
+
+    DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+        mockExecutor, mockCGroupsHandler);
+    runtime.initialize(conf, nmContext);
+    runtime.launchContainer(builder.build());
+
+    PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
+    List<String> args = op.getArguments();
+    String dockerCommandFile = args.get(11);
+
+    List<String> dockerCommands = Files.readAllLines(Paths.get(
+        dockerCommandFile), Charset.forName("UTF-8"));
+
+    int expected = 13;
+    int counter = 0;
+    Assert.assertEquals(expected, dockerCommands.size());
+    Assert.assertEquals("[docker-command-execution]",
+        dockerCommands.get(counter++));
+    Assert.assertEquals("  cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+        dockerCommands.get(counter++));
+    Assert.assertEquals("  cap-drop=ALL", dockerCommands.get(counter++));
+    Assert.assertEquals("  detach=true", dockerCommands.get(counter++));
+    Assert.assertEquals("  docker-command=run", dockerCommands.get(counter++));
+    Assert.assertEquals("  group-add=" + String.join(",", groups),
+        dockerCommands.get(counter++));
+    Assert
+        .assertEquals("  image=busybox:1.2.3", dockerCommands.get(counter++));
+    Assert.assertEquals(
+        "  launch-command=bash,/test_container_work_dir/launch_container.sh",
+        dockerCommands.get(counter++));
+    Assert.assertEquals("  mounts="
+            + "/test_container_log_dir:/test_container_log_dir:rw,"
+            + "/test_application_local_dir:/test_application_local_dir:rw,"
+            + "/test_filecache_dir:/test_filecache_dir:ro,"
+            + "/test_user_filecache_dir:/test_user_filecache_dir:ro",
+        dockerCommands.get(counter++));
+    Assert.assertEquals(
+        "  name=container_e11_1518975676334_14532816_01_000001",
+        dockerCommands.get(counter++));
+    Assert.assertEquals("  net=host", dockerCommands.get(counter++));
+    Assert.assertEquals("  user=" + uidGidPair, dockerCommands.get(counter++));
+    Assert.assertEquals("  workdir=/test_container_work_dir",
+        dockerCommands.get(counter));
+  }
+
   @Test
   public void testContainerLaunchWithUserRemapping()
       throws ContainerExecutionException, PrivilegedOperationException,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md

@@ -114,6 +114,23 @@ The following properties should be set in yarn-site.xml:
     </description>
   </property>
 
+  <property>
+    <name>yarn.nodemanager.runtime.linux.type</name>
+    <value></value>
+    <description>
+      Optional. Sets the default container runtime to use.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.runtime.linux.docker.image-name</name>
+    <value></value>
+    <description>
+      Optional. Default docker image to be used when the docker runtime is
+      selected.
+    </description>
+  </property>
+
   <property>
     <name>yarn.nodemanager.runtime.linux.docker.allowed-container-networks</name>
     <value>host,none,bridge</value>