Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6447. Provide container sandbox policies for groups (gphillips via rkanter)

This commit is contained in:
Robert Kanter 2017-05-16 18:02:39 -07:00 committed by Xiaoyu Yao
parent 3defe7d85b
commit e774688409
3 changed files with 147 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1500,6 +1500,10 @@ public class YarnConfiguration extends Configuration {
public static final String YARN_CONTAINER_SANDBOX_POLICY =
YARN_CONTAINER_SANDBOX + ".policy";
/** Prefix for group to policy file mapping.*/
public static final String YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX =
YARN_CONTAINER_SANDBOX_POLICY + ".group.";
/** The group which will run by default without the java security manager.*/
public static final String YARN_CONTAINER_SANDBOX_WHITELIST_GROUP =
YARN_CONTAINER_SANDBOX + ".whitelist-group";

64
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
View File

@ -50,10 +50,13 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static org.apache.hadoop.fs.Path.SEPARATOR;
import static org.apache.hadoop.util.Shell.SYSPROP_HADOOP_HOME_DIR;
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
@ -93,13 +96,21 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
* Accepts canonical path to a java policy file on the local filesystem.
* This file will be loaded as the base policy, any additional container
* grants will be appended to this base file. If not specified, the default
* java.policy file provided with hadoop resources will be used.
* java.policy file provided with hadoop resources will be used.
* </li>
* <li>
* {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} :
* Optional setting to specify a YARN queue which will be exempt from the
* sand-boxing process.
* </li>
* <li>
* {@value
* YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX}$groupName :
* Optional setting to map groups to java policy files. The value is a path
* to the java policy file for $groupName. A user which is a member of
* multiple groups with different policies will receive the superset of all
* the permissions across their groups.
* </li>
* </ul>
*/
@InterfaceAudience.Private
@ -138,7 +149,7 @@ public class JavaSandboxLinuxContainerRuntime
this.configuration = conf;
this.sandboxMode =
SandboxMode.get(
this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX,
this.configuration.get(YARN_CONTAINER_SANDBOX,
YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
super.initialize(conf);
@ -211,8 +222,10 @@ public class JavaSandboxLinuxContainerRuntime
ctx.getExecutionAttribute(CONTAINER_RUN_CMDS);
Map<String, String> env =
ctx.getContainer().getLaunchContext().getEnvironment();
String username =
ctx.getExecutionAttribute(USER);
if(!isSandboxContainerWhitelisted(ctx, commands)) {
if(!isSandboxContainerWhitelisted(username, commands)) {
String tmpDirBase = configuration.get("hadoop.tmp.dir");
if (tmpDirBase == null) {
throw new ContainerExecutionException("hadoop.tmp.dir not set!");
@ -223,6 +236,8 @@ public class JavaSandboxLinuxContainerRuntime
String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
initializePolicyDir();
List<String> groupPolicyFiles =
getGroupPolicyFiles(configuration, ctx.getExecutionAttribute(USER));
Path policyFilePath = Files.createFile(
Paths.get(policyFileDir.toString(),
containerID + "-" + NMContainerPolicyUtils.POLICY_FILE),
@ -231,12 +246,12 @@ public class JavaSandboxLinuxContainerRuntime
containerPolicies.put(containerID, policyFilePath);
NMContainerPolicyUtils.generatePolicyFile(
policyOutputStream, localDirs, resources, configuration);
NMContainerPolicyUtils.generatePolicyFile(policyOutputStream,
localDirs, groupPolicyFiles, resources, configuration);
NMContainerPolicyUtils.appendSecurityFlags(
commands, env, policyFilePath, sandboxMode);
} catch (Exception e) {
} catch (IOException e) {
throw new ContainerExecutionException(e);
} finally {
IOUtils.cleanup(LOG, policyOutputStream);
@ -264,15 +279,32 @@ public class JavaSandboxLinuxContainerRuntime
return sandboxMode != SandboxMode.disabled;
}
private static List<String> getGroupPolicyFiles(Configuration conf,
String user) throws ContainerExecutionException {
Groups groups = Groups.getUserToGroupsMappingService(conf);
List<String> userGroups;
try {
userGroups = groups.getGroups(user);
} catch (IOException e) {
throw new ContainerExecutionException("Container user does not exist");
}
return userGroups.stream()
.map(group -> conf.get(YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX
+ group))
.filter(groupPolicy -> groupPolicy != null)
.collect(Collectors.toList());
}
/**
* Determine if the container should be whitelisted (i.e. exempt from the
* Java Security Manager).
* @param ctx The container runtime context for the requested container
* @param username The name of the user running the container
* @param commands The list of run commands for the container
* @return boolean value denoting whether the container should be whitelisted.
* @throws ContainerExecutionException If container user can not be resolved
*/
private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx,
private boolean isSandboxContainerWhitelisted(String username,
List<String> commands) throws ContainerExecutionException {
String whitelistGroup = configuration.get(
YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP);
@ -281,7 +313,7 @@ public class JavaSandboxLinuxContainerRuntime
boolean isWhitelisted = false;
try {
userGroups = groups.getGroups(ctx.getExecutionAttribute(USER));
userGroups = groups.getGroups(username);
} catch (IOException e) {
throw new ContainerExecutionException("Container user does not exist");
}
@ -399,8 +431,9 @@ public class JavaSandboxLinuxContainerRuntime
* base policy file or if it is unable to create a new policy file.
*/
static void generatePolicyFile(OutputStream policyOutStream,
List<String> localDirs, Map<org.apache.hadoop.fs.Path,
List<String>> resources, Configuration conf)
List<String> localDirs, List<String> groupPolicyPaths,
Map<org.apache.hadoop.fs.Path, List<String>> resources,
Configuration conf)
throws IOException {
String policyFilePath =
@ -414,13 +447,16 @@ public class JavaSandboxLinuxContainerRuntime
cacheDirs.add(path.getParent().toString());
}
if(policyFilePath == null) {
if (groupPolicyPaths != null) {
for(String policyPath : groupPolicyPaths) {
Files.copy(Paths.get(policyPath), policyOutStream);
}
} else if (policyFilePath == null) {
IOUtils.copyBytes(
NMContainerPolicyUtils.class.getResourceAsStream("/" + POLICY_FILE),
policyOutStream, conf, false);
} else {
Files.copy(Paths.get(policyFilePath), policyOutStream);
policyOutStream.flush();
}
Formatter filePermissionFormat = new Formatter(policyOutStream,
@ -484,7 +520,7 @@ public class JavaSandboxLinuxContainerRuntime
private static boolean validateJavaHome(String containerJavaHome)
throws ContainerExecutionException{
if (System.getenv(JAVA_HOME.name()) == null){
if (System.getenv(JAVA_HOME.name()) == null) {
throw new ContainerExecutionException(
"JAVA_HOME is not set for NodeManager");
}

106
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java
View File

@ -37,20 +37,29 @@ import org.junit.rules.ExpectedException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilePermission;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.net.SocketPermission;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.Permission;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Formatter;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.LOG;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.MULTI_COMMAND_REGEX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_APPEND_FLAG;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FLAG;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.SECURITY_FLAG;
@ -76,8 +85,9 @@ import static org.mockito.Mockito.when;
*/
public class TestJavaSandboxLinuxContainerRuntime {
private static final String HADOOP_HOME = "hadoop.home.dir";
private static String hadoopHomeDir = System.getProperty(HADOOP_HOME);
private final static String HADOOP_HOME = "hadoop.home.dir";
private final static String HADOOP_HOME_DIR = System.getProperty(HADOOP_HOME);
private final Properties baseProps = new Properties(System.getProperties());
@Rule
public ExpectedException exception = ExpectedException.none();
@ -101,11 +111,12 @@ public class TestJavaSandboxLinuxContainerRuntime {
private final static String WHITELIST_GROUP = "captains";
private final static String CONTAINER_ID = "container_1234567890";
private final static String APPLICATION_ID = "application_1234567890";
private File baseTestDirectory;
@Before
public void setup() throws Exception {
File baseTestDirectory = new File(System.getProperty("test.build.data",
baseTestDirectory = new File(System.getProperty("test.build.data",
System.getProperty("java.io.tmpdir", "target")),
TestJavaSandboxLinuxContainerRuntime.class.getName());
@ -114,10 +125,8 @@ public class TestJavaSandboxLinuxContainerRuntime {
conf = new Configuration();
conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES,
WHITELIST_USER + "=" + WHITELIST_GROUP + ";"
WHITELIST_USER + "=" + WHITELIST_GROUP + "," + NORMAL_GROUP + ";"
+ NORMAL_USER + "=" + NORMAL_GROUP + ";");
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
WHITELIST_GROUP);
conf.set("hadoop.tmp.dir", baseTestDirectory.getAbsolutePath());
Files.deleteIfExists(Paths.get(baseTestDirectory.getAbsolutePath(),
@ -151,21 +160,20 @@ public class TestJavaSandboxLinuxContainerRuntime {
runtimeContextBuilder = createRuntimeContext();
if (hadoopHomeDir == null) {
if (HADOOP_HOME_DIR == null) {
System.setProperty(HADOOP_HOME, policyFile.getParent());
}
OutputStream outStream = new FileOutputStream(policyFile);
JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
.generatePolicyFile(outStream, symLinks, resources, conf);
.generatePolicyFile(outStream, symLinks, null, resources, conf);
outStream.close();
System.setProperty("java.security.policy", policyFile.getCanonicalPath());
securityManager = new SecurityManager();
}
public ContainerRuntimeContext.Builder createRuntimeContext(){
public ContainerRuntimeContext.Builder createRuntimeContext(){
Container container = mock(Container.class);
ContainerLaunchContext ctx = mock(ContainerLaunchContext.class);
@ -194,6 +202,71 @@ public class TestJavaSandboxLinuxContainerRuntime {
return builder;
}
public static final String SOCKET_PERMISSION_FORMAT =
"grant { \n" +
" permission %1s \"%2s\", \"%3s\";\n" +
"};\n";
public static final String RUNTIME_PERMISSION_FORMAT =
"grant { \n" +
" permission %1s \"%2s\";\n" +
"};\n";
@Test
public void testGroupPolicies()
throws IOException, ContainerExecutionException {
// Generate new policy files each containing one grant
File openSocketPolicyFile =
File.createTempFile("openSocket", "policy", baseTestDirectory);
File classLoaderPolicyFile =
File.createTempFile("createClassLoader", "policy", baseTestDirectory);
Permission socketPerm = new SocketPermission("localhost:0", "listen");
Permission runtimePerm = new RuntimePermission("createClassLoader");
StringBuilder socketPermString = new StringBuilder();
Formatter openSocketPolicyFormatter = new Formatter(socketPermString);
openSocketPolicyFormatter.format(SOCKET_PERMISSION_FORMAT,
socketPerm.getClass().getName(), socketPerm.getName(),
socketPerm.getActions());
FileWriter socketPermWriter = new FileWriter(openSocketPolicyFile);
socketPermWriter.write(socketPermString.toString());
socketPermWriter.close();
StringBuilder classLoaderPermString = new StringBuilder();
Formatter classLoaderPolicyFormatter = new Formatter(classLoaderPermString);
classLoaderPolicyFormatter.format(RUNTIME_PERMISSION_FORMAT,
runtimePerm.getClass().getName(), runtimePerm.getName());
FileWriter classLoaderPermWriter = new FileWriter(classLoaderPolicyFile);
classLoaderPermWriter.write(classLoaderPermString.toString());
classLoaderPermWriter.close();
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX +
WHITELIST_GROUP, openSocketPolicyFile.toString());
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX +
NORMAL_GROUP, classLoaderPolicyFile.toString());
String[] inputCommand = {"$JAVA_HOME/bin/java jar MyJob.jar"};
List<String> commands = Arrays.asList(inputCommand);
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
runtime.prepareContainer(runtimeContextBuilder.build());
//pull generated policy from cmd
Matcher policyMatches = Pattern.compile(POLICY_APPEND_FLAG + "=?([^ ]+)")
.matcher(commands.get(0));
policyMatches.find();
String generatedPolicy = policyMatches.group(1);
//Test that generated policy file has included both policies
Assert.assertTrue(
Files.readAllLines(Paths.get(generatedPolicy)).contains(
classLoaderPermString.toString().split("\n")[1]));
Assert.assertTrue(
Files.readAllLines(Paths.get(generatedPolicy)).contains(
socketPermString.toString().split("\n")[1]));
}
@Test
public void testGrant() throws Exception {
FilePermission grantPermission =
@ -235,7 +308,6 @@ public class TestJavaSandboxLinuxContainerRuntime {
JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
.appendSecurityFlags(commands, env, policyFilePath,
JavaSandboxLinuxContainerRuntime.SandboxMode.permissive);
}
@Test
@ -247,6 +319,9 @@ public class TestJavaSandboxLinuxContainerRuntime {
};
List<String> commands = Arrays.asList(inputCommand);
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
WHITELIST_GROUP);
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
runtime.prepareContainer(runtimeContextBuilder.build());
@ -254,7 +329,6 @@ public class TestJavaSandboxLinuxContainerRuntime {
Assert.assertTrue("Command should not be modified when user is " +
"member of whitelisted group",
inputCommand[0].equals(commands.get(0)));
}
@Test
@ -265,6 +339,9 @@ public class TestJavaSandboxLinuxContainerRuntime {
};
List<String> commands = Arrays.asList(inputCommand);
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
WHITELIST_GROUP);
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
runtime.prepareContainer(runtimeContextBuilder.build());
@ -283,6 +360,9 @@ public class TestJavaSandboxLinuxContainerRuntime {
};
List<String> commands = Arrays.asList(inputCommand);
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
WHITELIST_GROUP);
runtimeContextBuilder.setExecutionAttribute(USER, NORMAL_USER);
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
runtime.prepareContainer(runtimeContextBuilder.build());
@ -373,6 +453,6 @@ public class TestJavaSandboxLinuxContainerRuntime {
@After
public void cleanup(){
System.setSecurityManager(null);
System.setProperties(baseProps);
}
}