YARN-4578. Directories that are mounted in docker containers need to be more restrictive/container-specific. Contributed by Sidharta Seethana.

hadoop-yarn-project/CHANGES.txt

@@ -140,6 +140,9 @@ Release 2.9.0 - UNRELEASED
     YARN-4584. RM startup failure when AM attempts greater than max-attempts.
     (Bibin A Chundatt via rohithsharmaks)
 
+    YARN-4578. Directories that are mounted in docker containers need to be more
+    restrictive/container-specific. (Sidharta Seethana via vvasudev)
+
 Release 2.8.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java

@@ -291,6 +291,8 @@ public int launchContainer(ContainerStartContext ctx) throws IOException {
     Path containerWorkDir = ctx.getContainerWorkDir();
     List<String> localDirs = ctx.getLocalDirs();
     List<String> logDirs = ctx.getLogDirs();
+    List<String> containerLocalDirs = ctx.getContainerLocalDirs();
+    List<String> containerLogDirs = ctx.getContainerLogDirs();
     Map<Path, List<String>> localizedResources = ctx.getLocalizedResources();
 
     verifyUsernamePattern(user);
@@ -375,6 +377,8 @@ public int launchContainer(ContainerStartContext ctx) throws IOException {
             .setExecutionAttribute(PID_FILE_PATH, pidFilePath)
             .setExecutionAttribute(LOCAL_DIRS, localDirs)
             .setExecutionAttribute(LOG_DIRS, logDirs)
+            .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
+            .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
             .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
 
         if (tcCommandFile != null) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java

@@ -253,6 +253,7 @@ public Integer call() {
             + dirsHandler.getDisksHealthReport(false));
       }
 
+      List<String> containerLocalDirs = new ArrayList<>(localDirs.size());
       try {
         // /////////// Write out the container-script in the nmPrivate space.
         List<Path> appDirs = new ArrayList<Path>(localDirs.size());
@@ -261,6 +262,14 @@ public Integer call() {
           Path userdir = new Path(usersdir, user);
           Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
           appDirs.add(new Path(appsdir, appIdStr));
+
+          String containerLocalDir = localDir + Path.SEPARATOR +
+              ContainerLocalizer.USERCACHE + Path.SEPARATOR + user
+              + Path.SEPARATOR
+              + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr
+              + Path.SEPARATOR;
+
+          containerLocalDirs.add(containerLocalDir);
         }
         containerScriptOutStream =
           lfs.create(nmPrivateContainerScriptPath,
@@ -317,6 +326,8 @@ public Integer call() {
             .setContainerWorkDir(containerWorkDir)
             .setLocalDirs(localDirs)
             .setLogDirs(logDirs)
+            .setContainerLocalDirs(containerLocalDirs)
+            .setContainerLogDirs(containerLogDirs)
             .build());
       }
     } catch (Throwable e) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java

@@ -223,6 +223,12 @@ public void launchContainer(ContainerRuntimeContext ctx)
     List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
     @SuppressWarnings("unchecked")
     List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
+    @SuppressWarnings("unchecked")
+    List<String> containerLocalDirs = ctx.getExecutionAttribute(
+        CONTAINER_LOCAL_DIRS);
+    @SuppressWarnings("unchecked")
+    List<String> containerLogDirs = ctx.getExecutionAttribute(
+        CONTAINER_LOG_DIRS);
     Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
         YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
         YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
@@ -235,10 +241,10 @@ public void launchContainer(ContainerRuntimeContext ctx)
         .setNetworkType("host")
         .setCapabilities(capabilities)
         .addMountLocation("/etc/passwd", "/etc/password:ro");
-    List<String> allDirs = new ArrayList<>(localDirs);
+    List<String> allDirs = new ArrayList<>(containerLocalDirs);
 
     allDirs.add(containerWorkDir.toString());
-    allDirs.addAll(logDirs);
+    allDirs.addAll(containerLogDirs);
     for (String dir: allDirs) {
       runCommand.addMountLocation(dir, dir);
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java

@@ -55,6 +55,10 @@ private LinuxContainerRuntimeConstants() {
       List.class, "local_dirs");
   public static final Attribute<List> LOG_DIRS = Attribute.attribute(
       List.class, "log_dirs");
+  public static final Attribute<List> CONTAINER_LOCAL_DIRS = Attribute
+      .attribute(List.class, "container_local_dirs");
+  public static final Attribute<List> CONTAINER_LOG_DIRS = Attribute.attribute(
+      List.class, "container_log_dirs");
   public static final Attribute<String> RESOURCES_OPTIONS = Attribute.attribute(
       String.class, "resources_options");
   public static final Attribute<String> TC_COMMAND_FILE = Attribute.attribute(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java

@@ -45,6 +45,8 @@ public final class ContainerStartContext {
   private final Path containerWorkDir;
   private final List<String> localDirs;
   private final List<String> logDirs;
+  private final List<String> containerLocalDirs;
+  private final List<String> containerLogDirs;
 
   public static final class Builder {
     private Container container;
@@ -56,6 +58,8 @@ public static final class Builder {
     private Path containerWorkDir;
     private List<String> localDirs;
     private List<String> logDirs;
+    private List<String> containerLocalDirs;
+    private List<String> containerLogDirs;
 
     public Builder() {
     }
@@ -107,6 +111,16 @@ public Builder setLogDirs(List<String> logDirs) {
       return this;
     }
 
+    public Builder setContainerLocalDirs(List<String> containerLocalDirs) {
+      this.containerLocalDirs = containerLocalDirs;
+      return this;
+    }
+
+    public Builder setContainerLogDirs(List<String> containerLogDirs) {
+      this.containerLogDirs = containerLogDirs;
+      return this;
+    }
+
     public ContainerStartContext build() {
       return new ContainerStartContext(this);
     }
@@ -122,6 +136,8 @@ private ContainerStartContext(Builder builder) {
     this.containerWorkDir = builder.containerWorkDir;
     this.localDirs = builder.localDirs;
     this.logDirs = builder.logDirs;
+    this.containerLocalDirs = builder.containerLocalDirs;
+    this.containerLogDirs = builder.containerLogDirs;
   }
 
   public Container getContainer() {
@@ -163,4 +179,12 @@ public List<String> getLocalDirs() {
   public List<String> getLogDirs() {
     return Collections.unmodifiableList(this.logDirs);
   }
+
+  public List<String> getContainerLocalDirs() {
+    return this.containerLocalDirs;
+  }
+
+  public List<String> getContainerLogDirs() {
+    return this.containerLogDirs;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java

@@ -81,6 +81,8 @@ public class TestDockerContainerRuntime {
   Path pidFilePath;
   List<String> localDirs;
   List<String> logDirs;
+  List<String> containerLocalDirs;
+  List<String> containerLogDirs;
   String resourcesOptions;
   ContainerRuntimeContext.Builder builder;
   String submittingUser = "anakin";
@@ -123,9 +125,13 @@ public void setup() {
     localDirs = new ArrayList<>();
     logDirs = new ArrayList<>();
     resourcesOptions = "cgroups=none";
+    containerLocalDirs = new ArrayList<>();
+    containerLogDirs = new ArrayList<>();
 
     localDirs.add("/test_local_dir");
     logDirs.add("/test_log_dir");
+    containerLocalDirs.add("/test_container_local_dir");
+    containerLogDirs.add("/test_container_log_dir");
 
     builder = new ContainerRuntimeContext
         .Builder(container);
@@ -141,6 +147,8 @@ public void setup() {
         .setExecutionAttribute(PID_FILE_PATH, pidFilePath)
         .setExecutionAttribute(LOCAL_DIRS, localDirs)
         .setExecutionAttribute(LOG_DIRS, logDirs)
+        .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
+        .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
         .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
   }
 
@@ -245,8 +253,8 @@ public void testDockerContainerLaunch()
         .append("bash %8$s/launch_container.sh");
 
     String expectedCommand = String.format(expectedCommandTemplate.toString(),
-        containerId, runAsUser, containerWorkDir, localDirs.get(0),
+        containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0),
-        containerWorkDir, logDirs.get(0), image, containerWorkDir);
+        containerWorkDir, containerLogDirs.get(0), image, containerWorkDir);
 
     List<String> dockerCommands = Files.readAllLines(Paths.get
             (dockerCommandFile), Charset.forName("UTF-8"));