Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5298. Mount usercache and NM filecache directories into Docker container. Contributed by Sidharta Seethana.

This commit is contained in:
Varun Vasudev 2016-07-14 16:11:07 +05:30
parent be26c1b9b0
commit 58e1850801
9 changed files with 136 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
View File

@ -229,6 +229,12 @@ public interface ApplicationConstants {
*/
LOCAL_DIRS("LOCAL_DIRS"),
/**
* $LOCAL_USER_DIRS
* Final, exported by NodeManager and non-modifiable by users.
*/
LOCAL_USER_DIRS("LOCAL_USER_DIRS"),
/**
* $LOG_DIRS
* Final, exported by NodeManager and non-modifiable by users.

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
View File

@ -52,7 +52,6 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler;
import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler;
import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler;
import org.apache.hadoop.yarn.util.ConverterUtils;
import java.io.File;
import java.io.IOException;
@ -311,6 +310,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
Path containerWorkDir = ctx.getContainerWorkDir();
List<String> localDirs = ctx.getLocalDirs();
List<String> logDirs = ctx.getLogDirs();
List<String> filecacheDirs = ctx.getFilecacheDirs();
List<String> userLocalDirs = ctx.getUserLocalDirs();
List<String> containerLocalDirs = ctx.getContainerLocalDirs();
List<String> containerLogDirs = ctx.getContainerLogDirs();
Map<Path, List<String>> localizedResources = ctx.getLocalizedResources();
@ -397,6 +398,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
.setExecutionAttribute(PID_FILE_PATH, pidFilePath)
.setExecutionAttribute(LOCAL_DIRS, localDirs)
.setExecutionAttribute(LOG_DIRS, logDirs)
.setExecutionAttribute(FILECACHE_DIRS, filecacheDirs)
.setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs)
.setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
.setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
.setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
View File

@ -230,6 +230,8 @@ public class ContainerLaunch implements Callable<Integer> {
pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
List<String> localDirs = dirsHandler.getLocalDirs();
List<String> logDirs = dirsHandler.getLogDirs();
List<String> filecacheDirs = getNMFilecacheDirs(localDirs);
List<String> userLocalDirs = getUserLocalDirs(localDirs);
List<String> containerLocalDirs = getContainerLocalDirs(localDirs);
List<String> containerLogDirs = getContainerLogDirs(logDirs);
@ -242,6 +244,7 @@ public class ContainerLaunch implements Callable<Integer> {
try {
// /////////// Write out the container-script in the nmPrivate space.
List<Path> appDirs = new ArrayList<Path>(localDirs.size());
for (String localDir : localDirs) {
Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
Path userdir = new Path(usersdir, user);
@ -258,7 +261,8 @@ public class ContainerLaunch implements Callable<Integer> {
new Path(containerWorkDir,
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
// Sanitize the container's environment
sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs,
sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs,
containerLogDirs,
localResources, nmPrivateClasspathJarDir);
// Write out the environment
@ -288,6 +292,8 @@ public class ContainerLaunch implements Callable<Integer> {
.setContainerWorkDir(containerWorkDir)
.setLocalDirs(localDirs)
.setLogDirs(logDirs)
.setFilecacheDirs(filecacheDirs)
.setUserLocalDirs(userLocalDirs)
.setContainerLocalDirs(containerLocalDirs)
.setContainerLogDirs(containerLogDirs)
.build());
@ -353,6 +359,35 @@ public class ContainerLaunch implements Callable<Integer> {
return containerLocalDirs;
}
protected List<String> getUserLocalDirs(List<String> localDirs) {
List<String> userLocalDirs = new ArrayList<>(localDirs.size());
String user = container.getUser();
for (String localDir : localDirs) {
String userLocalDir = localDir + Path.SEPARATOR +
ContainerLocalizer.USERCACHE + Path.SEPARATOR + user
+ Path.SEPARATOR;
userLocalDirs.add(userLocalDir);
}
return userLocalDirs;
}
protected List<String> getNMFilecacheDirs(List<String> localDirs) {
List<String> filecacheDirs = new ArrayList<>(localDirs.size());
for (String localDir : localDirs) {
String filecacheDir = localDir + Path.SEPARATOR +
ContainerLocalizer.FILECACHE;
filecacheDirs.add(filecacheDir);
}
return filecacheDirs;
}
protected Map<Path, List<String>> getLocalizedResources()
throws YarnException {
Map<Path, List<String>> localResources = container.getLocalizedResources();
@ -990,7 +1025,8 @@ public class ContainerLaunch implements Callable<Integer> {
}
public void sanitizeEnv(Map<String, String> environment, Path pwd,
List<Path> appDirs, List<String> containerLogDirs,
List<Path> appDirs, List<String> userLocalDirs, List<String>
containerLogDirs,
Map<Path, List<String>> resources,
Path nmPrivateClasspathJarDir) throws IOException {
/**
@ -1012,6 +1048,9 @@ public class ContainerLaunch implements Callable<Integer> {
environment.put(Environment.LOCAL_DIRS.name(),
StringUtils.join(",", appDirs));
environment.put(Environment.LOCAL_USER_DIRS.name(), StringUtils.join(",",
userLocalDirs));
environment.put(Environment.LOG_DIRS.name(),
StringUtils.join(",", containerLogDirs));

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
View File

@ -318,6 +318,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@SuppressWarnings("unchecked")
List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
@SuppressWarnings("unchecked")
List<String> filecacheDirs = ctx.getExecutionAttribute(FILECACHE_DIRS);
@SuppressWarnings("unchecked")
List<String> containerLocalDirs = ctx.getExecutionAttribute(
CONTAINER_LOCAL_DIRS);
@SuppressWarnings("unchecked")
@ -326,6 +328,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@SuppressWarnings("unchecked")
Map<Path, List<String>> localizedResources = ctx.getExecutionAttribute(
LOCALIZED_RESOURCES);
@SuppressWarnings("unchecked")
List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
@ -340,8 +345,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.addMountLocation("/etc/passwd", "/etc/password:ro");
List<String> allDirs = new ArrayList<>(containerLocalDirs);
allDirs.addAll(filecacheDirs);
allDirs.add(containerWorkDir.toString());
allDirs.addAll(containerLogDirs);
allDirs.addAll(userLocalDirs);
for (String dir: allDirs) {
runCommand.addMountLocation(dir, dir);
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java
View File

@ -55,6 +55,10 @@ public final class LinuxContainerRuntimeConstants {
List.class, "local_dirs");
public static final Attribute<List> LOG_DIRS = Attribute.attribute(
List.class, "log_dirs");
public static final Attribute<List> FILECACHE_DIRS = Attribute.attribute(
List.class, "filecache_dirs");
public static final Attribute<List> USER_LOCAL_DIRS = Attribute.attribute(
List.class, "user_local_dirs");
public static final Attribute<List> CONTAINER_LOCAL_DIRS = Attribute
.attribute(List.class, "container_local_dirs");
public static final Attribute<List> CONTAINER_LOG_DIRS = Attribute.attribute(

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java
View File

@ -45,6 +45,8 @@ public final class ContainerStartContext {
private final Path containerWorkDir;
private final List<String> localDirs;
private final List<String> logDirs;
private final List<String> filecacheDirs;
private final List<String> userLocalDirs;
private final List<String> containerLocalDirs;
private final List<String> containerLogDirs;
@ -58,6 +60,8 @@ public final class ContainerStartContext {
private Path containerWorkDir;
private List<String> localDirs;
private List<String> logDirs;
private List<String> filecacheDirs;
private List<String> userLocalDirs;
private List<String> containerLocalDirs;
private List<String> containerLogDirs;
@ -111,6 +115,16 @@ public final class ContainerStartContext {
return this;
}
public Builder setFilecacheDirs(List<String> filecacheDirs) {
this.filecacheDirs = filecacheDirs;
return this;
}
public Builder setUserLocalDirs(List<String> userLocalDirs) {
this.userLocalDirs = userLocalDirs;
return this;
}
public Builder setContainerLocalDirs(List<String> containerLocalDirs) {
this.containerLocalDirs = containerLocalDirs;
return this;
@ -136,6 +150,8 @@ public final class ContainerStartContext {
this.containerWorkDir = builder.containerWorkDir;
this.localDirs = builder.localDirs;
this.logDirs = builder.logDirs;
this.filecacheDirs = builder.filecacheDirs;
this.userLocalDirs = builder.userLocalDirs;
this.containerLocalDirs = builder.containerLocalDirs;
this.containerLogDirs = builder.containerLogDirs;
}
@ -180,11 +196,20 @@ public final class ContainerStartContext {
return Collections.unmodifiableList(this.logDirs);
}
public List<String> getFilecacheDirs() {
return Collections.unmodifiableList(this.filecacheDirs);
}
public List<String> getUserLocalDirs() {
return Collections.unmodifiableList(this.userLocalDirs);
}
public List<String> getContainerLocalDirs() {
return this.containerLocalDirs;
return Collections.unmodifiableList(this.containerLocalDirs);
}
public List<String> getContainerLogDirs() {
return this.containerLogDirs;
return Collections.unmodifiableList(this
.containerLogDirs);
}
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
View File

@ -180,6 +180,10 @@ public class TestLinuxContainerExecutorWithMocks {
.setContainerWorkDir(workDir)
.setLocalDirs(dirsHandler.getLocalDirs())
.setLogDirs(dirsHandler.getLogDirs())
.setFilecacheDirs(new ArrayList<>())
.setUserLocalDirs(new ArrayList<>())
.setContainerLocalDirs(new ArrayList<>())
.setContainerLogDirs(new ArrayList<>())
.build());
assertEquals(0, ret);
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
@ -345,6 +349,10 @@ public class TestLinuxContainerExecutorWithMocks {
.setContainerWorkDir(workDir)
.setLocalDirs(dirsHandler.getLocalDirs())
.setLogDirs(dirsHandler.getLogDirs())
.setFilecacheDirs(new ArrayList<>())
.setUserLocalDirs(new ArrayList<>())
.setContainerLocalDirs(new ArrayList<>())
.setContainerLogDirs(new ArrayList<>())
.build());
Assert.assertNotSame(0, ret);

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
View File

@ -455,6 +455,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
"target/test-dir");
Path pwd = new Path(testDir);
List<Path> appDirs = new ArrayList<Path>();
List<String> userLocalDirs = new ArrayList<>();
List<String> containerLogs = new ArrayList<String>();
Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
@ -465,8 +466,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Path nmp = new Path(testDir);
launch.sanitizeEnv(
userSetEnv, pwd, appDirs, containerLogs, resources, nmp);
launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
resources, nmp);
List<String> result =
getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));
@ -484,8 +485,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
launch = new ContainerLaunch(distContext, conf,
dispatcher, exec, null, container, dirsHandler, containerManager);
launch.sanitizeEnv(
userSetEnv, pwd, appDirs, containerLogs, resources, nmp);
launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
resources, nmp);
result =
getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
View File

@ -82,6 +82,8 @@ public class TestDockerContainerRuntime {
private Path pidFilePath;
private List<String> localDirs;
private List<String> logDirs;
private List<String> filecacheDirs;
private List<String> userLocalDirs;
private List<String> containerLocalDirs;
private List<String> containerLogDirs;
private Map<Path, List<String>> localizedResources;
@ -126,13 +128,17 @@ public class TestDockerContainerRuntime {
pidFilePath = new Path("/test_pid_file_path");
localDirs = new ArrayList<>();
logDirs = new ArrayList<>();
filecacheDirs = new ArrayList<>();
resourcesOptions = "cgroups=none";
userLocalDirs = new ArrayList<>();
containerLocalDirs = new ArrayList<>();
containerLogDirs = new ArrayList<>();
localizedResources = new HashMap<>();
localDirs.add("/test_local_dir");
logDirs.add("/test_log_dir");
filecacheDirs.add("/test_filecache_dir");
userLocalDirs.add("/test_user_local_dir");
containerLocalDirs.add("/test_container_local_dir");
containerLogDirs.add("/test_container_log_dir");
localizedResources.put(new Path("/test_local_dir/test_resource_file"),
@ -156,6 +162,8 @@ public class TestDockerContainerRuntime {
.setExecutionAttribute(PID_FILE_PATH, pidFilePath)
.setExecutionAttribute(LOCAL_DIRS, localDirs)
.setExecutionAttribute(LOG_DIRS, logDirs)
.setExecutionAttribute(FILECACHE_DIRS, filecacheDirs)
.setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs)
.setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs)
.setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs)
.setExecutionAttribute(LOCALIZED_RESOURCES, localizedResources)
@ -262,12 +270,15 @@ public class TestDockerContainerRuntime {
.append("-v %4$s:%4$s ")
.append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ")
.append("%7$s ")
.append("bash %8$s/launch_container.sh");
.append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
.append("bash %10$s/launch_container.sh");
String expectedCommand = String.format(expectedCommandTemplate.toString(),
containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0),
containerWorkDir, containerLogDirs.get(0), image, containerWorkDir);
String expectedCommand = String
.format(expectedCommandTemplate.toString(), containerId, runAsUser,
containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
image, containerWorkDir);
List<String> dockerCommands = Files.readAllLines(Paths.get
(dockerCommandFile), Charset.forName("UTF-8"));
@ -363,13 +374,15 @@ public class TestDockerContainerRuntime {
.append(getExpectedTestCapabilitiesArgumentString())
.append("-v /etc/passwd:/etc/password:ro ")
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("%7$s ")
.append("bash %8$s/launch_container.sh");
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
.append("bash %10$s/launch_container.sh");
String expectedCommand = String
.format(expectedCommandTemplate.toString(), containerId, runAsUser,
containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
containerLogDirs.get(0), image, containerWorkDir);
containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
image, containerWorkDir);
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
@ -416,13 +429,15 @@ public class TestDockerContainerRuntime {
.append(getExpectedTestCapabilitiesArgumentString())
.append("-v /etc/passwd:/etc/password:ro ")
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("%7$s ")
.append("bash %8$s/launch_container.sh");
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
.append("bash %10$s/launch_container.sh");
String expectedCommand = String
.format(expectedCommandTemplate.toString(), containerId, runAsUser,
containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
containerLogDirs.get(0), image, containerWorkDir);
containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
image, containerWorkDir);
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
@ -451,14 +466,15 @@ public class TestDockerContainerRuntime {
.append(getExpectedTestCapabilitiesArgumentString())
.append("-v /etc/passwd:/etc/password:ro ")
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("%7$s ")
.append("bash %8$s/launch_container.sh");
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
.append("bash %10$s/launch_container.sh");
expectedCommand = String
.format(expectedCommandTemplate.toString(), containerId, runAsUser,
containerWorkDir, containerLocalDirs.get(0), containerWorkDir,
containerLogDirs.get(0), image, containerWorkDir);
containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
image, containerWorkDir);
dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));