From 18c494a00c8ead768f3a868b450dceea485559df Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Tue, 16 May 2017 18:02:39 -0700 Subject: [PATCH] YARN-6447. Provide container sandbox policies for groups (gphillips via rkanter) --- .../hadoop/yarn/conf/YarnConfiguration.java | 4 + .../JavaSandboxLinuxContainerRuntime.java | 64 ++++++++--- .../TestJavaSandboxLinuxContainerRuntime.java | 106 +++++++++++++++--- 3 files changed, 147 insertions(+), 27 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 82274fe74ab..5e4c826a426 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1500,6 +1500,10 @@ public class YarnConfiguration extends Configuration { public static final String YARN_CONTAINER_SANDBOX_POLICY = YARN_CONTAINER_SANDBOX + ".policy"; + /** Prefix for group to policy file mapping.*/ + public static final String YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX = + YARN_CONTAINER_SANDBOX_POLICY + ".group."; + /** The group which will run by default without the java security manager.*/ public static final String YARN_CONTAINER_SANDBOX_WHITELIST_GROUP = YARN_CONTAINER_SANDBOX + ".whitelist-group"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java index 1e5bf57ae62..0b858bc42c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java @@ -50,10 +50,13 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.apache.hadoop.fs.Path.SEPARATOR; import static org.apache.hadoop.util.Shell.SYSPROP_HADOOP_HOME_DIR; import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS; @@ -93,13 +96,21 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r * Accepts canonical path to a java policy file on the local filesystem. * This file will be loaded as the base policy, any additional container * grants will be appended to this base file. If not specified, the default - * java.policy file provided with hadoop resources will be used. + * java.policy file provided with hadoop resources will be used. * *
  • * {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} : * Optional setting to specify a YARN queue which will be exempt from the * sand-boxing process. *
    • + *
  • + * {@value + * YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX}$groupName : + * Optional setting to map groups to java policy files. The value is a path + * to the java policy file for $groupName. A user which is a member of + * multiple groups with different policies will receive the superset of all + * the permissions across their groups. + *
    • * */ @InterfaceAudience.Private @@ -138,7 +149,7 @@ public class JavaSandboxLinuxContainerRuntime this.configuration = conf; this.sandboxMode = SandboxMode.get( - this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX, + this.configuration.get(YARN_CONTAINER_SANDBOX, YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX)); super.initialize(conf); @@ -211,8 +222,10 @@ public class JavaSandboxLinuxContainerRuntime ctx.getExecutionAttribute(CONTAINER_RUN_CMDS); Map env = ctx.getContainer().getLaunchContext().getEnvironment(); + String username = + ctx.getExecutionAttribute(USER); - if(!isSandboxContainerWhitelisted(ctx, commands)) { + if(!isSandboxContainerWhitelisted(username, commands)) { String tmpDirBase = configuration.get("hadoop.tmp.dir"); if (tmpDirBase == null) { throw new ContainerExecutionException("hadoop.tmp.dir not set!"); @@ -223,6 +236,8 @@ public class JavaSandboxLinuxContainerRuntime String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR); initializePolicyDir(); + List groupPolicyFiles = + getGroupPolicyFiles(configuration, ctx.getExecutionAttribute(USER)); Path policyFilePath = Files.createFile( Paths.get(policyFileDir.toString(), containerID + "-" + NMContainerPolicyUtils.POLICY_FILE), @@ -231,12 +246,12 @@ public class JavaSandboxLinuxContainerRuntime containerPolicies.put(containerID, policyFilePath); - NMContainerPolicyUtils.generatePolicyFile( - policyOutputStream, localDirs, resources, configuration); + NMContainerPolicyUtils.generatePolicyFile(policyOutputStream, + localDirs, groupPolicyFiles, resources, configuration); NMContainerPolicyUtils.appendSecurityFlags( commands, env, policyFilePath, sandboxMode); - } catch (Exception e) { + } catch (IOException e) { throw new ContainerExecutionException(e); } finally { IOUtils.cleanup(LOG, policyOutputStream); @@ -264,15 +279,32 @@ public class JavaSandboxLinuxContainerRuntime return sandboxMode != SandboxMode.disabled; } + private static List getGroupPolicyFiles(Configuration conf, + String user) throws ContainerExecutionException { + Groups groups = Groups.getUserToGroupsMappingService(conf); + List userGroups; + try { + userGroups = groups.getGroups(user); + } catch (IOException e) { + throw new ContainerExecutionException("Container user does not exist"); + } + + return userGroups.stream() + .map(group -> conf.get(YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX + + group)) + .filter(groupPolicy -> groupPolicy != null) + .collect(Collectors.toList()); + } + /** * Determine if the container should be whitelisted (i.e. exempt from the * Java Security Manager). - * @param ctx The container runtime context for the requested container + * @param username The name of the user running the container * @param commands The list of run commands for the container * @return boolean value denoting whether the container should be whitelisted. * @throws ContainerExecutionException If container user can not be resolved */ - private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx, + private boolean isSandboxContainerWhitelisted(String username, List commands) throws ContainerExecutionException { String whitelistGroup = configuration.get( YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP); @@ -281,7 +313,7 @@ public class JavaSandboxLinuxContainerRuntime boolean isWhitelisted = false; try { - userGroups = groups.getGroups(ctx.getExecutionAttribute(USER)); + userGroups = groups.getGroups(username); } catch (IOException e) { throw new ContainerExecutionException("Container user does not exist"); } @@ -399,8 +431,9 @@ public class JavaSandboxLinuxContainerRuntime * base policy file or if it is unable to create a new policy file. */ static void generatePolicyFile(OutputStream policyOutStream, - List localDirs, Map> resources, Configuration conf) + List localDirs, List groupPolicyPaths, + Map> resources, + Configuration conf) throws IOException { String policyFilePath = @@ -414,13 +447,16 @@ public class JavaSandboxLinuxContainerRuntime cacheDirs.add(path.getParent().toString()); } - if(policyFilePath == null) { + if (groupPolicyPaths != null) { + for(String policyPath : groupPolicyPaths) { + Files.copy(Paths.get(policyPath), policyOutStream); + } + } else if (policyFilePath == null) { IOUtils.copyBytes( NMContainerPolicyUtils.class.getResourceAsStream("/" + POLICY_FILE), policyOutStream, conf, false); } else { Files.copy(Paths.get(policyFilePath), policyOutStream); - policyOutStream.flush(); } Formatter filePermissionFormat = new Formatter(policyOutStream, @@ -484,7 +520,7 @@ public class JavaSandboxLinuxContainerRuntime private static boolean validateJavaHome(String containerJavaHome) throws ContainerExecutionException{ - if (System.getenv(JAVA_HOME.name()) == null){ + if (System.getenv(JAVA_HOME.name()) == null) { throw new ContainerExecutionException( "JAVA_HOME is not set for NodeManager"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java index e10d0dd1232..bdd435eb2ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestJavaSandboxLinuxContainerRuntime.java @@ -37,20 +37,29 @@ import org.junit.rules.ExpectedException; import java.io.File; import java.io.FileOutputStream; import java.io.FilePermission; +import java.io.FileWriter; +import java.io.IOException; import java.io.OutputStream; +import java.net.SocketPermission; import java.nio.file.Files; import java.nio.file.Paths; +import java.security.Permission; import java.util.ArrayList; import java.util.Arrays; +import java.util.Formatter; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.LOG; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.MULTI_COMMAND_REGEX; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_APPEND_FLAG; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FLAG; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.SECURITY_FLAG; @@ -76,8 +85,9 @@ import static org.mockito.Mockito.when; */ public class TestJavaSandboxLinuxContainerRuntime { - private static final String HADOOP_HOME = "hadoop.home.dir"; - private static String hadoopHomeDir = System.getProperty(HADOOP_HOME); + private final static String HADOOP_HOME = "hadoop.home.dir"; + private final static String HADOOP_HOME_DIR = System.getProperty(HADOOP_HOME); + private final Properties baseProps = new Properties(System.getProperties()); @Rule public ExpectedException exception = ExpectedException.none(); @@ -101,11 +111,12 @@ public class TestJavaSandboxLinuxContainerRuntime { private final static String WHITELIST_GROUP = "captains"; private final static String CONTAINER_ID = "container_1234567890"; private final static String APPLICATION_ID = "application_1234567890"; + private File baseTestDirectory; @Before public void setup() throws Exception { - File baseTestDirectory = new File(System.getProperty("test.build.data", + baseTestDirectory = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir", "target")), TestJavaSandboxLinuxContainerRuntime.class.getName()); @@ -114,10 +125,8 @@ public class TestJavaSandboxLinuxContainerRuntime { conf = new Configuration(); conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES, - WHITELIST_USER + "=" + WHITELIST_GROUP + ";" + WHITELIST_USER + "=" + WHITELIST_GROUP + "," + NORMAL_GROUP + ";" + NORMAL_USER + "=" + NORMAL_GROUP + ";"); - conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP, - WHITELIST_GROUP); conf.set("hadoop.tmp.dir", baseTestDirectory.getAbsolutePath()); Files.deleteIfExists(Paths.get(baseTestDirectory.getAbsolutePath(), @@ -151,21 +160,20 @@ public class TestJavaSandboxLinuxContainerRuntime { runtimeContextBuilder = createRuntimeContext(); - if (hadoopHomeDir == null) { + if (HADOOP_HOME_DIR == null) { System.setProperty(HADOOP_HOME, policyFile.getParent()); } OutputStream outStream = new FileOutputStream(policyFile); JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils - .generatePolicyFile(outStream, symLinks, resources, conf); + .generatePolicyFile(outStream, symLinks, null, resources, conf); outStream.close(); System.setProperty("java.security.policy", policyFile.getCanonicalPath()); securityManager = new SecurityManager(); - } - public ContainerRuntimeContext.Builder createRuntimeContext(){ + public ContainerRuntimeContext.Builder createRuntimeContext(){ Container container = mock(Container.class); ContainerLaunchContext ctx = mock(ContainerLaunchContext.class); @@ -194,6 +202,71 @@ public class TestJavaSandboxLinuxContainerRuntime { return builder; } + public static final String SOCKET_PERMISSION_FORMAT = + "grant { \n" + + " permission %1s \"%2s\", \"%3s\";\n" + + "};\n"; + public static final String RUNTIME_PERMISSION_FORMAT = + "grant { \n" + + " permission %1s \"%2s\";\n" + + "};\n"; + + @Test + public void testGroupPolicies() + throws IOException, ContainerExecutionException { + // Generate new policy files each containing one grant + File openSocketPolicyFile = + File.createTempFile("openSocket", "policy", baseTestDirectory); + File classLoaderPolicyFile = + File.createTempFile("createClassLoader", "policy", baseTestDirectory); + Permission socketPerm = new SocketPermission("localhost:0", "listen"); + Permission runtimePerm = new RuntimePermission("createClassLoader"); + + StringBuilder socketPermString = new StringBuilder(); + Formatter openSocketPolicyFormatter = new Formatter(socketPermString); + openSocketPolicyFormatter.format(SOCKET_PERMISSION_FORMAT, + socketPerm.getClass().getName(), socketPerm.getName(), + socketPerm.getActions()); + FileWriter socketPermWriter = new FileWriter(openSocketPolicyFile); + socketPermWriter.write(socketPermString.toString()); + socketPermWriter.close(); + + StringBuilder classLoaderPermString = new StringBuilder(); + Formatter classLoaderPolicyFormatter = new Formatter(classLoaderPermString); + classLoaderPolicyFormatter.format(RUNTIME_PERMISSION_FORMAT, + runtimePerm.getClass().getName(), runtimePerm.getName()); + FileWriter classLoaderPermWriter = new FileWriter(classLoaderPolicyFile); + classLoaderPermWriter.write(classLoaderPermString.toString()); + classLoaderPermWriter.close(); + + conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX + + WHITELIST_GROUP, openSocketPolicyFile.toString()); + conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX + + NORMAL_GROUP, classLoaderPolicyFile.toString()); + + String[] inputCommand = {"$JAVA_HOME/bin/java jar MyJob.jar"}; + List commands = Arrays.asList(inputCommand); + + runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER); + runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands); + + runtime.prepareContainer(runtimeContextBuilder.build()); + + //pull generated policy from cmd + Matcher policyMatches = Pattern.compile(POLICY_APPEND_FLAG + "=?([^ ]+)") + .matcher(commands.get(0)); + policyMatches.find(); + String generatedPolicy = policyMatches.group(1); + + //Test that generated policy file has included both policies + Assert.assertTrue( + Files.readAllLines(Paths.get(generatedPolicy)).contains( + classLoaderPermString.toString().split("\n")[1])); + Assert.assertTrue( + Files.readAllLines(Paths.get(generatedPolicy)).contains( + socketPermString.toString().split("\n")[1])); + } + @Test public void testGrant() throws Exception { FilePermission grantPermission = @@ -235,7 +308,6 @@ public class TestJavaSandboxLinuxContainerRuntime { JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils .appendSecurityFlags(commands, env, policyFilePath, JavaSandboxLinuxContainerRuntime.SandboxMode.permissive); - } @Test @@ -247,6 +319,9 @@ public class TestJavaSandboxLinuxContainerRuntime { }; List commands = Arrays.asList(inputCommand); + conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP, + WHITELIST_GROUP); + runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER); runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands); runtime.prepareContainer(runtimeContextBuilder.build()); @@ -254,7 +329,6 @@ public class TestJavaSandboxLinuxContainerRuntime { Assert.assertTrue("Command should not be modified when user is " + "member of whitelisted group", inputCommand[0].equals(commands.get(0))); - } @Test @@ -265,6 +339,9 @@ public class TestJavaSandboxLinuxContainerRuntime { }; List commands = Arrays.asList(inputCommand); + conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP, + WHITELIST_GROUP); + runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER); runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands); runtime.prepareContainer(runtimeContextBuilder.build()); @@ -283,6 +360,9 @@ public class TestJavaSandboxLinuxContainerRuntime { }; List commands = Arrays.asList(inputCommand); + conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP, + WHITELIST_GROUP); + runtimeContextBuilder.setExecutionAttribute(USER, NORMAL_USER); runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands); runtime.prepareContainer(runtimeContextBuilder.build()); @@ -373,6 +453,6 @@ public class TestJavaSandboxLinuxContainerRuntime { @After public void cleanup(){ - System.setSecurityManager(null); + System.setProperties(baseProps); } } \ No newline at end of file