YARN-5280. Allow YARN containers to run with Java Security Manager (gphillips via rkanter)
This commit is contained in:
Robert Kanter
parent
899d5c4d49
commit
6f6dfe0202
|
|
@ -1465,6 +1465,26 @@ public static boolean isAclEnabled(Configuration conf) {
|
|||
public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK =
|
||||
"host";
|
||||
|
||||
/** The mode in which the Java Container Sandbox should run detailed by
|
||||
* the JavaSandboxLinuxContainerRuntime. */
|
||||
public static final String YARN_CONTAINER_SANDBOX =
|
||||
LINUX_CONTAINER_RUNTIME_PREFIX + "sandbox-mode";
|
||||
public static final String DEFAULT_YARN_CONTAINER_SANDBOX = "disabled";
|
||||
|
||||
/** Permissions for application local directories.*/
|
||||
public static final String YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS =
|
||||
YARN_CONTAINER_SANDBOX + ".local-dirs.permissions";
|
||||
public static final String DEFAULT_YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS =
|
||||
"read";
|
||||
|
||||
/** Location for non-default java policy file.*/
|
||||
public static final String YARN_CONTAINER_SANDBOX_POLICY =
|
||||
YARN_CONTAINER_SANDBOX + ".policy";
|
||||
|
||||
/** The group which will run by default without the java security manager.*/
|
||||
public static final String YARN_CONTAINER_SANDBOX_WHITELIST_GROUP =
|
||||
YARN_CONTAINER_SANDBOX + ".whitelist-group";
|
||||
|
||||
/** The path to the Linux container executor.*/
|
||||
public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH =
|
||||
NM_PREFIX + "linux-container-executor.path";
|
||||
|
|
|
|||
|
|
@ -1583,6 +1583,32 @@
|
|||
<value>host</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>The mode in which the Java Container Sandbox should run detailed by
|
||||
the JavaSandboxLinuxContainerRuntime.</description>
|
||||
<name>yarn.nodemanager.runtime.linux.sandbox-mode</name>
|
||||
<value>disabled</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>Permissions for application local directories.</description>
|
||||
<name>yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions</name>
|
||||
<value>read</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>Location for non-default java policy file.</description>
|
||||
<name>yarn.nodemanager.runtime.linux.sandbox-mode.policy</name>
|
||||
<value></value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>The group which will run by default without the java security
|
||||
manager.</description>
|
||||
<name>yarn.nodemanager.runtime.linux.sandbox-mode.whitelist-group</name>
|
||||
<value></value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>This flag determines whether memory limit will be set for the Windows Job
|
||||
Object of the containers launched by the default container executor.</description>
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@
|
|||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext;
|
||||
|
|
@ -148,6 +149,14 @@ public Path localizeClasspathJar(Path jarPath, Path target, String owner)
|
|||
public abstract void startLocalizer(LocalizerStartContext ctx)
|
||||
throws IOException, InterruptedException;
|
||||
|
||||
/**
|
||||
* Prepare the container prior to the launch environment being written.
|
||||
* @param ctx Encapsulates information necessary for launching containers.
|
||||
* @throws IOException if errors occur during container preparation
|
||||
*/
|
||||
public void prepareContainer(ContainerPrepareContext ctx) throws
|
||||
IOException{
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch the container on the node. This is a blocking call and returns only
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@
|
|||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
|
||||
|
|
@ -362,6 +363,28 @@ public void buildMainArgs(List<String> command, String user, String appId,
|
|||
localDirs);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void prepareContainer(ContainerPrepareContext ctx) throws IOException {
|
||||
|
||||
ContainerRuntimeContext.Builder builder =
|
||||
new ContainerRuntimeContext.Builder(ctx.getContainer());
|
||||
|
||||
builder.setExecutionAttribute(LOCALIZED_RESOURCES,
|
||||
ctx.getLocalizedResources())
|
||||
.setExecutionAttribute(USER, ctx.getUser())
|
||||
.setExecutionAttribute(CONTAINER_LOCAL_DIRS,
|
||||
ctx.getContainerLocalDirs())
|
||||
.setExecutionAttribute(CONTAINER_RUN_CMDS, ctx.getCommands())
|
||||
.setExecutionAttribute(CONTAINER_ID_STR,
|
||||
ctx.getContainer().getContainerId().toString());
|
||||
|
||||
try {
|
||||
linuxContainerRuntime.prepareContainer(builder.build());
|
||||
} catch (ContainerExecutionException e) {
|
||||
throw new IOException("Unable to prepare container: ", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int launchContainer(ContainerStartContext ctx) throws IOException {
|
||||
Container container = ctx.getContainer();
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@
|
|||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerPrepareContext;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
|
||||
|
|
@ -198,18 +199,14 @@ public Integer call() {
|
|||
|
||||
FileContext lfs = FileContext.getLocalFSFileContext();
|
||||
|
||||
Path nmPrivateContainerScriptPath =
|
||||
dirsHandler.getLocalPathForWrite(
|
||||
Path nmPrivateContainerScriptPath = dirsHandler.getLocalPathForWrite(
|
||||
getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
|
||||
+ CONTAINER_SCRIPT);
|
||||
Path nmPrivateTokensPath =
|
||||
dirsHandler.getLocalPathForWrite(
|
||||
getContainerPrivateDir(appIdStr, containerIdStr)
|
||||
+ Path.SEPARATOR
|
||||
Path nmPrivateTokensPath = dirsHandler.getLocalPathForWrite(
|
||||
getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
|
||||
+ String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
|
||||
containerIdStr));
|
||||
Path nmPrivateClasspathJarDir =
|
||||
dirsHandler.getLocalPathForWrite(
|
||||
Path nmPrivateClasspathJarDir = dirsHandler.getLocalPathForWrite(
|
||||
getContainerPrivateDir(appIdStr, containerIdStr));
|
||||
DataOutputStream containerScriptOutStream = null;
|
||||
DataOutputStream tokensOutStream = null;
|
||||
|
|
@ -224,7 +221,6 @@ public Integer call() {
|
|||
recordContainerWorkDir(containerID, containerWorkDir.toString());
|
||||
|
||||
String pidFileSubpath = getPidFileSubpath(appIdStr, containerIdStr);
|
||||
|
||||
// pid file should be in nm private dir so that it is not
|
||||
// accessible by users
|
||||
pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
|
||||
|
|
@ -240,11 +236,9 @@ public Integer call() {
|
|||
throw new IOException("Most of the disks failed. "
|
||||
+ dirsHandler.getDisksHealthReport(false));
|
||||
}
|
||||
|
||||
try {
|
||||
// /////////// Write out the container-script in the nmPrivate space.
|
||||
List<Path> appDirs = new ArrayList<Path>(localDirs.size());
|
||||
|
||||
for (String localDir : localDirs) {
|
||||
Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
|
||||
Path userdir = new Path(usersdir, user);
|
||||
|
|
@ -258,18 +252,22 @@ public Integer call() {
|
|||
// Set the token location too.
|
||||
environment.put(
|
||||
ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME,
|
||||
new Path(containerWorkDir,
|
||||
new Path(containerWorkDir,
|
||||
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
|
||||
// Sanitize the container's environment
|
||||
sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs,
|
||||
containerLogDirs,
|
||||
localResources, nmPrivateClasspathJarDir);
|
||||
containerLogDirs, localResources, nmPrivateClasspathJarDir);
|
||||
|
||||
exec.prepareContainer(new ContainerPrepareContext.Builder()
|
||||
.setContainer(container)
|
||||
.setLocalizedResources(localResources)
|
||||
.setUser(user)
|
||||
.setContainerLocalDirs(containerLocalDirs)
|
||||
.setCommands(launchContext.getCommands()).build());
|
||||
// Write out the environment
|
||||
exec.writeLaunchEnv(containerScriptOutStream, environment,
|
||||
localResources, launchContext.getCommands(),
|
||||
new Path(containerLogDirs.get(0)), user);
|
||||
|
||||
// /////////// End of writing out container-script
|
||||
|
||||
// /////////// Write out the container-tokens in the nmPrivate space.
|
||||
|
|
@ -295,8 +293,7 @@ public Integer call() {
|
|||
.setFilecacheDirs(filecacheDirs)
|
||||
.setUserLocalDirs(userLocalDirs)
|
||||
.setContainerLocalDirs(containerLocalDirs)
|
||||
.setContainerLogDirs(containerLogDirs)
|
||||
.build());
|
||||
.setContainerLogDirs(containerLogDirs).build());
|
||||
} catch (Throwable e) {
|
||||
LOG.warn("Failed to launch container.", e);
|
||||
dispatcher.getEventHandler().handle(new ContainerExitEvent(
|
||||
|
|
@ -308,7 +305,6 @@ public Integer call() {
|
|||
}
|
||||
|
||||
handleContainerExitCode(ret, containerLogDir);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -35,10 +35,10 @@
|
|||
|
||||
/**
|
||||
* This class is a {@link ContainerRuntime} implementation that delegates all
|
||||
* operations to either a {@link DefaultLinuxContainerRuntime} instance or a
|
||||
* {@link DockerLinuxContainerRuntime} instance, depending on whether the
|
||||
* {@link DockerLinuxContainerRuntime} instance believes the operation to be
|
||||
* requesting a Docker container.
|
||||
* operations to a {@link DefaultLinuxContainerRuntime} instance, a
|
||||
* {@link DockerLinuxContainerRuntime} instance, or a
|
||||
* {@link JavaSandboxLinuxContainerRuntime} instance depending on whether
|
||||
* each instance believes the operation to be within its scope.
|
||||
*
|
||||
* @see DockerLinuxContainerRuntime#isDockerContainerRequested
|
||||
*/
|
||||
|
|
@ -49,6 +49,7 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
.getLog(DelegatingLinuxContainerRuntime.class);
|
||||
private DefaultLinuxContainerRuntime defaultLinuxContainerRuntime;
|
||||
private DockerLinuxContainerRuntime dockerLinuxContainerRuntime;
|
||||
private JavaSandboxLinuxContainerRuntime javaSandboxLinuxContainerRuntime;
|
||||
|
||||
@Override
|
||||
public void initialize(Configuration conf)
|
||||
|
|
@ -61,15 +62,20 @@ public void initialize(Configuration conf)
|
|||
dockerLinuxContainerRuntime = new DockerLinuxContainerRuntime(
|
||||
privilegedOperationExecutor);
|
||||
dockerLinuxContainerRuntime.initialize(conf);
|
||||
javaSandboxLinuxContainerRuntime = new JavaSandboxLinuxContainerRuntime(
|
||||
privilegedOperationExecutor);
|
||||
javaSandboxLinuxContainerRuntime.initialize(conf);
|
||||
}
|
||||
|
||||
private LinuxContainerRuntime pickContainerRuntime(Container container) {
|
||||
Map<String, String> env = container.getLaunchContext().getEnvironment();
|
||||
private LinuxContainerRuntime pickContainerRuntime(
|
||||
Map<String, String> environment){
|
||||
LinuxContainerRuntime runtime;
|
||||
|
||||
if (DockerLinuxContainerRuntime.isDockerContainerRequested(env)){
|
||||
if (DockerLinuxContainerRuntime.isDockerContainerRequested(environment)){
|
||||
runtime = dockerLinuxContainerRuntime;
|
||||
} else {
|
||||
} else if (javaSandboxLinuxContainerRuntime.isSandboxContainerRequested()) {
|
||||
runtime = javaSandboxLinuxContainerRuntime;
|
||||
} else {
|
||||
runtime = defaultLinuxContainerRuntime;
|
||||
}
|
||||
|
||||
|
|
@ -81,12 +87,14 @@ private LinuxContainerRuntime pickContainerRuntime(Container container) {
|
|||
return runtime;
|
||||
}
|
||||
|
||||
private LinuxContainerRuntime pickContainerRuntime(Container container) {
|
||||
return pickContainerRuntime(container.getLaunchContext().getEnvironment());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void prepareContainer(ContainerRuntimeContext ctx)
|
||||
throws ContainerExecutionException {
|
||||
Container container = ctx.getContainer();
|
||||
LinuxContainerRuntime runtime = pickContainerRuntime(container);
|
||||
|
||||
LinuxContainerRuntime runtime = pickContainerRuntime(ctx.getContainer());
|
||||
runtime.prepareContainer(ctx);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,495 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.nodemanager.
|
||||
containermanager.linux.runtime;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.io.IOUtils;
|
||||
import org.apache.hadoop.security.Groups;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import java.io.FilePermission;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.DirectoryStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.attribute.FileAttribute;
|
||||
import java.nio.file.attribute.PosixFilePermission;
|
||||
import java.nio.file.attribute.PosixFilePermissions;
|
||||
import java.security.AllPermission;
|
||||
import java.util.Formatter;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.apache.hadoop.fs.Path.SEPARATOR;
|
||||
import static org.apache.hadoop.util.Shell.SYSPROP_HADOOP_HOME_DIR;
|
||||
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOCALIZED_RESOURCES;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER;
|
||||
/**
|
||||
* <p>This class extends the {@link DefaultLinuxContainerRuntime} specifically
|
||||
* for containers which run Java commands. It generates a new java security
|
||||
* policy file per container and modifies the java command to enable the
|
||||
* Java Security Manager with the generated policy.</p>
|
||||
*
|
||||
* The behavior of the {@link JavaSandboxLinuxContainerRuntime} can be modified
|
||||
* using the following settings:
|
||||
*
|
||||
* <ul>
|
||||
* <li>
|
||||
* {@value YarnConfiguration#YARN_CONTAINER_SANDBOX} :
|
||||
* This yarn-site.xml setting has three options:
|
||||
* <ul>
|
||||
* <li>disabled - Default behavior. {@link LinuxContainerRuntime}
|
||||
* is disabled</li>
|
||||
* <li>permissive - JVM containers will run with Java Security Manager
|
||||
* enabled. Non-JVM containers will run normally</li>
|
||||
* <li>enforcing - JVM containers will run with Java Security Manager
|
||||
* enabled. Non-JVM containers will be prevented from executing and an
|
||||
* {@link ContainerExecutionException} will be thrown.</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* <li>
|
||||
* {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS} :
|
||||
* Determines the file permissions for the application directories. The
|
||||
* permissions come in the form of comma separated values
|
||||
* (e.g. read,write,execute,delete). Defaults to {@code read} for read-only.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_POLICY} :
|
||||
* Accepts canonical path to a java policy file on the local filesystem.
|
||||
* This file will be loaded as the base policy, any additional container
|
||||
* grants will be appended to this base file. If not specified, the default
|
||||
* java.policy file provided with hadoop resources will be used.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} :
|
||||
* Optional setting to specify a YARN queue which will be exempt from the
|
||||
* sand-boxing process.
|
||||
* </li>
|
||||
* </ul>
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Unstable
|
||||
public class JavaSandboxLinuxContainerRuntime
|
||||
extends DefaultLinuxContainerRuntime {
|
||||
private static final Log LOG =
|
||||
LogFactory.getLog(DefaultLinuxContainerRuntime.class);
|
||||
private Configuration configuration;
|
||||
private SandboxMode sandboxMode;
|
||||
|
||||
public static final String POLICY_FILE_DIR = "nm-sandbox-policies";
|
||||
|
||||
private static Path policyFileDir;
|
||||
private static final FileAttribute<Set<PosixFilePermission>> POLICY_ATTR =
|
||||
PosixFilePermissions.asFileAttribute(
|
||||
PosixFilePermissions.fromString("rwxr-xr-x"));
|
||||
|
||||
private Map<String, Path> containerPolicies = new HashMap<>();
|
||||
|
||||
/**
|
||||
* Create an instance using the given {@link PrivilegedOperationExecutor}
|
||||
* instance for performing operations.
|
||||
*
|
||||
* @param privilegedOperationExecutor the {@link PrivilegedOperationExecutor}
|
||||
* instance
|
||||
*/
|
||||
public JavaSandboxLinuxContainerRuntime(
|
||||
PrivilegedOperationExecutor privilegedOperationExecutor) {
|
||||
super(privilegedOperationExecutor);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initialize(Configuration conf)
|
||||
throws ContainerExecutionException {
|
||||
this.configuration = conf;
|
||||
this.sandboxMode =
|
||||
SandboxMode.get(
|
||||
this.configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX,
|
||||
YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX));
|
||||
|
||||
initializePolicyDir();
|
||||
|
||||
super.initialize(conf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the Java Security Policy directory. Either creates the
|
||||
* directory if it doesn't exist, or clears the contents of the directory if
|
||||
* already created.
|
||||
* @throws ContainerExecutionException If unable to resolve policy directory
|
||||
*/
|
||||
private void initializePolicyDir() throws ContainerExecutionException {
|
||||
String hadoopTempDir = configuration.get("hadoop.tmp.dir");
|
||||
if (hadoopTempDir == null) {
|
||||
throw new ContainerExecutionException("hadoop.tmp.dir not set!");
|
||||
}
|
||||
policyFileDir = Paths.get(hadoopTempDir, POLICY_FILE_DIR);
|
||||
//Delete any existing policy files if the directory has already been created
|
||||
if(Files.exists(policyFileDir)){
|
||||
try (DirectoryStream<Path> stream =
|
||||
Files.newDirectoryStream(policyFileDir)){
|
||||
for(Path policyFile : stream){
|
||||
Files.delete(policyFile);
|
||||
}
|
||||
}catch(IOException e){
|
||||
throw new ContainerExecutionException("Unable to initialize policy "
|
||||
+ "directory: " + e);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
policyFileDir = Files.createDirectories(
|
||||
Paths.get(hadoopTempDir, POLICY_FILE_DIR), POLICY_ATTR);
|
||||
} catch (IOException e) {
|
||||
throw new ContainerExecutionException("Unable to create policy file " +
|
||||
"directory: " + e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prior to environment from being written locally need to generate
|
||||
* policy file which limits container access to a small set of directories.
|
||||
* Additionally the container run command needs to be modified to include
|
||||
* flags to enable the java security manager with the generated policy.
|
||||
* <br>
|
||||
* The Java Sandbox will be circumvented if the user is a member of the
|
||||
* group specified in:
|
||||
* {@value YarnConfiguration#YARN_CONTAINER_SANDBOX_WHITELIST_GROUP} and if
|
||||
* they do not include the JVM flag:
|
||||
* {@value NMContainerPolicyUtils#SECURITY_FLAG}
|
||||
*
|
||||
* @param ctx The {@link ContainerRuntimeContext} containing container
|
||||
* setup properties.
|
||||
* @throws ContainerExecutionException Exception thrown if temporary policy
|
||||
* file directory can't be created, or if any exceptions occur during policy
|
||||
* file parsing and generation.
|
||||
*/
|
||||
@Override
|
||||
public void prepareContainer(ContainerRuntimeContext ctx)
|
||||
throws ContainerExecutionException {
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<String> localDirs =
|
||||
ctx.getExecutionAttribute(CONTAINER_LOCAL_DIRS);
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<org.apache.hadoop.fs.Path, List<String>> resources =
|
||||
ctx.getExecutionAttribute(LOCALIZED_RESOURCES);
|
||||
@SuppressWarnings("unchecked")
|
||||
List<String> commands =
|
||||
ctx.getExecutionAttribute(CONTAINER_RUN_CMDS);
|
||||
Map<String, String> env =
|
||||
ctx.getContainer().getLaunchContext().getEnvironment();
|
||||
|
||||
if(!isSandboxContainerWhitelisted(ctx, commands)) {
|
||||
String tmpDirBase = configuration.get("hadoop.tmp.dir");
|
||||
if (tmpDirBase == null) {
|
||||
throw new ContainerExecutionException("hadoop.tmp.dir not set!");
|
||||
}
|
||||
|
||||
OutputStream policyOutputStream = null;
|
||||
try {
|
||||
String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
|
||||
|
||||
Path policyFilePath = Files.createFile(
|
||||
Paths.get(policyFileDir.toString(),
|
||||
containerID + "-" + NMContainerPolicyUtils.POLICY_FILE),
|
||||
POLICY_ATTR);
|
||||
policyOutputStream = Files.newOutputStream(policyFilePath);
|
||||
|
||||
containerPolicies.put(containerID, policyFilePath);
|
||||
|
||||
NMContainerPolicyUtils.generatePolicyFile(
|
||||
policyOutputStream, localDirs, resources, configuration);
|
||||
NMContainerPolicyUtils.appendSecurityFlags(
|
||||
commands, env, policyFilePath, sandboxMode);
|
||||
|
||||
} catch (Exception e) {
|
||||
throw new ContainerExecutionException(e);
|
||||
} finally {
|
||||
IOUtils.cleanup(LOG, policyOutputStream);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void launchContainer(ContainerRuntimeContext ctx)
|
||||
throws ContainerExecutionException {
|
||||
try {
|
||||
super.launchContainer(ctx);
|
||||
} finally {
|
||||
deletePolicyFiles(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if JVMSandboxLinuxContainerRuntime should be used. This is
|
||||
* decided based on the value of
|
||||
* {@value YarnConfiguration#YARN_CONTAINER_SANDBOX}
|
||||
* @return true if Sandbox is requested, false otherwise
|
||||
*/
|
||||
boolean isSandboxContainerRequested() {
|
||||
return sandboxMode != SandboxMode.disabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if the container should be whitelisted (i.e. exempt from the
|
||||
* Java Security Manager).
|
||||
* @param ctx The container runtime context for the requested container
|
||||
* @param commands The list of run commands for the container
|
||||
* @return boolean value denoting whether the container should be whitelisted.
|
||||
* @throws ContainerExecutionException If container user can not be resolved
|
||||
*/
|
||||
private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx,
|
||||
List<String> commands) throws ContainerExecutionException {
|
||||
String whitelistGroup = configuration.get(
|
||||
YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP);
|
||||
Groups groups = Groups.getUserToGroupsMappingService(configuration);
|
||||
List<String> userGroups;
|
||||
boolean isWhitelisted = false;
|
||||
|
||||
try {
|
||||
userGroups = groups.getGroups(ctx.getExecutionAttribute(USER));
|
||||
} catch (IOException e) {
|
||||
throw new ContainerExecutionException("Container user does not exist");
|
||||
}
|
||||
|
||||
if(whitelistGroup != null && userGroups.contains(whitelistGroup)) {
|
||||
// If any command has security flag, whitelisting is disabled
|
||||
for(String cmd : commands) {
|
||||
if(cmd.contains(NMContainerPolicyUtils.SECURITY_FLAG)){
|
||||
isWhitelisted = false;
|
||||
break;
|
||||
} else {
|
||||
isWhitelisted = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return isWhitelisted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes policy files for container specified by parameter. Additionally
|
||||
* this method will age off any stale policy files generated by
|
||||
* {@link JavaSandboxLinuxContainerRuntime}
|
||||
* @param ctx Container context for files to be deleted
|
||||
* @throws ContainerExecutionException if unable to access or delete policy
|
||||
* files or generated policy file directory
|
||||
*/
|
||||
private void deletePolicyFiles(ContainerRuntimeContext ctx)
|
||||
throws ContainerExecutionException {
|
||||
try {
|
||||
Files.delete(containerPolicies.remove(
|
||||
ctx.getExecutionAttribute(CONTAINER_ID_STR)));
|
||||
} catch (IOException e) {
|
||||
throw new ContainerExecutionException("Unable to delete policy file: "
|
||||
+ e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enumeration of the modes the JavaSandboxLinuxContainerRuntime can use.
|
||||
* See {@link JavaSandboxLinuxContainerRuntime} for details on the
|
||||
* behavior of each setting.
|
||||
*/
|
||||
public enum SandboxMode {
|
||||
enforcing("enforcing"),
|
||||
permissive("permissive"),
|
||||
disabled("disabled");
|
||||
|
||||
private final String mode;
|
||||
SandboxMode(String mode){
|
||||
this.mode = mode;
|
||||
}
|
||||
|
||||
public static SandboxMode get(String mode) {
|
||||
|
||||
if(enforcing.mode.equals(mode)) {
|
||||
return enforcing;
|
||||
} else if(permissive.mode.equals(mode)) {
|
||||
return permissive;
|
||||
} else {
|
||||
return disabled;
|
||||
}
|
||||
}
|
||||
|
||||
public String toString(){
|
||||
return mode;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Static utility class defining String constants and static methods for the
|
||||
* use of the {@link JavaSandboxLinuxContainerRuntime}.
|
||||
*/
|
||||
static final class NMContainerPolicyUtils{
|
||||
|
||||
static final String POLICY_FILE = "java.policy";
|
||||
static final String SECURITY_DEBUG = " -Djava.security.debug=all";
|
||||
static final String SECURITY_FLAG = "-Djava.security.manager";
|
||||
static final String POLICY_APPEND_FLAG = "-Djava.security.policy=";
|
||||
static final String POLICY_FLAG = POLICY_APPEND_FLAG + "=";
|
||||
static final String JAVA_CMD = "/bin/java ";
|
||||
static final String JVM_SECURITY_CMD =
|
||||
JAVA_CMD + SECURITY_FLAG + " " + POLICY_FLAG;
|
||||
|
||||
static final String STRIP_POLICY_FLAG = POLICY_APPEND_FLAG + "[^ ]+";
|
||||
static final String CONTAINS_JAVA_CMD = "\\$" + JAVA_HOME + JAVA_CMD + ".*";
|
||||
static final String CHAINED_COMMAND_REGEX =
|
||||
"^.*(&&.+$)|(\\|\\|.+$).*$"; //Matches any occurrences of '||' or '&&'
|
||||
static final String CLEAN_CMD_REGEX =
|
||||
"(" + SECURITY_FLAG + ")|" +
|
||||
"(" + STRIP_POLICY_FLAG + ")";
|
||||
|
||||
static final String FILE_PERMISSION_FORMAT = " permission "
|
||||
+ FilePermission.class.getCanonicalName()
|
||||
+ " \"%1$s" + SEPARATOR + "-\", \"%2$s\";%n";
|
||||
static final String HADOOP_HOME_PERMISSION = "%ngrant codeBase \"file:"
|
||||
+ Paths.get(System.getProperty(SYSPROP_HADOOP_HOME_DIR))
|
||||
+ SEPARATOR + "-\" {%n" +
|
||||
" permission " + AllPermission.class.getCanonicalName() + ";%n};%n";
|
||||
static final Logger LOG =
|
||||
Logger.getLogger(NMContainerPolicyUtils.class);
|
||||
|
||||
/**
|
||||
* Write new policy file to policyOutStream which will include read access
|
||||
* to localize resources. Optionally a default policyFilePath can be
|
||||
* specified to append a custom policy implementation to the new policy file
|
||||
* @param policyOutStream OutputStream pointing to java.policy file
|
||||
* @param localDirs Container local directories
|
||||
* @param resources List of local container resources
|
||||
* @param conf YARN configuration
|
||||
* @throws IOException - If policy file generation is unable to read the
|
||||
* base policy file or if it is unable to create a new policy file.
|
||||
*/
|
||||
static void generatePolicyFile(OutputStream policyOutStream,
|
||||
List<String> localDirs, Map<org.apache.hadoop.fs.Path,
|
||||
List<String>> resources, Configuration conf)
|
||||
throws IOException {
|
||||
|
||||
String policyFilePath =
|
||||
conf.get(YarnConfiguration.YARN_CONTAINER_SANDBOX_POLICY);
|
||||
String filePermissions =
|
||||
conf.get(YarnConfiguration.YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS,
|
||||
YarnConfiguration.DEFAULT_YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS);
|
||||
|
||||
Set<String> cacheDirs = new HashSet<>();
|
||||
for(org.apache.hadoop.fs.Path path : resources.keySet()) {
|
||||
cacheDirs.add(path.getParent().toString());
|
||||
}
|
||||
|
||||
if(policyFilePath == null) {
|
||||
IOUtils.copyBytes(
|
||||
NMContainerPolicyUtils.class.getResourceAsStream("/" + POLICY_FILE),
|
||||
policyOutStream, conf, false);
|
||||
} else {
|
||||
Files.copy(Paths.get(policyFilePath), policyOutStream);
|
||||
policyOutStream.flush();
|
||||
}
|
||||
|
||||
Formatter filePermissionFormat = new Formatter(policyOutStream,
|
||||
StandardCharsets.UTF_8.name());
|
||||
filePermissionFormat.format(HADOOP_HOME_PERMISSION);
|
||||
filePermissionFormat.format("grant {%n");
|
||||
for(String localDir : localDirs) {
|
||||
filePermissionFormat.format(
|
||||
FILE_PERMISSION_FORMAT, localDir, filePermissions);
|
||||
}
|
||||
for(String cacheDir : cacheDirs) {
|
||||
filePermissionFormat.format(
|
||||
FILE_PERMISSION_FORMAT, cacheDir, filePermissions);
|
||||
}
|
||||
filePermissionFormat.format("};%n");
|
||||
filePermissionFormat.flush();
|
||||
}
|
||||
|
||||
/**
|
||||
* Modify command to enable the Java Security Manager and specify
|
||||
* java.policy file. Will modify the passed commands to strip any
|
||||
* existing java security configurations. Expects a java command to be the
|
||||
* first and only executable provided in enforcing mode. In passive mode
|
||||
* any commands with '||' or '&&' will not be modified.
|
||||
* @param commands List of container commands
|
||||
* @param env Container environment variables
|
||||
* @param policyPath Path to the container specific policy file
|
||||
* @param sandboxMode (enforcing, permissive, disabled) Determines
|
||||
* whether non-java containers will be launched
|
||||
* @throws ContainerExecutionException - Exception thrown if
|
||||
* JVM Sandbox enabled in 'enforcing' mode and a non-java command is
|
||||
* provided in the list of commands
|
||||
*/
|
||||
static void appendSecurityFlags(List<String> commands,
|
||||
Map<String, String> env, Path policyPath, SandboxMode sandboxMode)
|
||||
throws ContainerExecutionException {
|
||||
|
||||
for(int i = 0; i < commands.size(); i++){
|
||||
String command = commands.get(i);
|
||||
if(validateJavaHome(env.get(JAVA_HOME.name()))
|
||||
&& command.matches(CONTAINS_JAVA_CMD)
|
||||
&& !command.matches(CHAINED_COMMAND_REGEX)){
|
||||
command = command.replaceAll(CLEAN_CMD_REGEX, "");
|
||||
String securityString = JVM_SECURITY_CMD + policyPath + " ";
|
||||
if(LOG.isDebugEnabled()) {
|
||||
securityString += SECURITY_DEBUG;
|
||||
}
|
||||
commands.set(i, command.replaceFirst(JAVA_CMD, securityString));
|
||||
} else if (sandboxMode == SandboxMode.enforcing){
|
||||
throw new ContainerExecutionException(
|
||||
"Only JVM containers permitted in YARN sandbox mode (enforcing). "
|
||||
+ "The following command can not be executed securely: " + command);
|
||||
} else if (sandboxMode == SandboxMode.permissive){
|
||||
LOG.warn("The container will run without the java security manager"
|
||||
+ " due to an unsupported container command. The command"
|
||||
+ " will be permitted to run in Sandbox permissive mode: "
|
||||
+ command);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean validateJavaHome(String containerJavaHome)
|
||||
throws ContainerExecutionException{
|
||||
if (System.getenv(JAVA_HOME.name()) == null){
|
||||
throw new ContainerExecutionException(
|
||||
"JAVA_HOME is not set for NodeManager");
|
||||
}
|
||||
if (containerJavaHome == null) {
|
||||
throw new ContainerExecutionException(
|
||||
"JAVA_HOME is not set for container");
|
||||
}
|
||||
return System.getenv(JAVA_HOME.name()).equals(containerJavaHome);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -67,6 +67,8 @@ private LinuxContainerRuntimeConstants() {
|
|||
String.class, "resources_options");
|
||||
public static final Attribute<String> TC_COMMAND_FILE = Attribute.attribute(
|
||||
String.class, "tc_command_file");
|
||||
public static final Attribute<List> CONTAINER_RUN_CMDS = Attribute.attribute(
|
||||
List.class, "container_run_cmds");
|
||||
public static final Attribute<String> CGROUP_RELATIVE_PATH = Attribute
|
||||
.attribute(String.class, "cgroup_relative_path");
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,119 @@
|
|||
/*
|
||||
* *
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
* /
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.nodemanager.executor;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Encapsulates information required for preparing containers.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Unstable
|
||||
public final class ContainerPrepareContext {
|
||||
private final Container container;
|
||||
private final Map<Path, List<String>> localizedResources;
|
||||
private final String user;
|
||||
private final List<String> containerLocalDirs;
|
||||
private final List<String> commands;
|
||||
|
||||
/**
|
||||
* Builder for ContainerPrepareContext.
|
||||
*/
|
||||
public static final class Builder {
|
||||
private Container container;
|
||||
private Map<Path, List<String>> localizedResources;
|
||||
private String user;
|
||||
private List<String> containerLocalDirs;
|
||||
private List<String> commands;
|
||||
|
||||
public Builder() {
|
||||
}
|
||||
|
||||
public ContainerPrepareContext.Builder setContainer(Container container) {
|
||||
this.container = container;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ContainerPrepareContext.Builder setLocalizedResources(Map<Path,
|
||||
List<String>> localizedResources) {
|
||||
this.localizedResources = localizedResources;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ContainerPrepareContext.Builder setUser(String user) {
|
||||
this.user = user;
|
||||
return this;
|
||||
}
|
||||
public ContainerPrepareContext.Builder setContainerLocalDirs(
|
||||
List<String> containerLocalDirs) {
|
||||
this.containerLocalDirs = containerLocalDirs;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ContainerPrepareContext build() {
|
||||
return new ContainerPrepareContext(this);
|
||||
}
|
||||
|
||||
public ContainerPrepareContext.Builder setCommands(List<String> commands) {
|
||||
this.commands = commands;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
private ContainerPrepareContext(ContainerPrepareContext.Builder builder) {
|
||||
this.container = builder.container;
|
||||
this.localizedResources = builder.localizedResources;
|
||||
this.user = builder.user;
|
||||
this.containerLocalDirs = builder.containerLocalDirs;
|
||||
this.commands = builder.commands;
|
||||
}
|
||||
|
||||
public Container getContainer() {
|
||||
return this.container;
|
||||
}
|
||||
|
||||
public Map<Path, List<String>> getLocalizedResources() {
|
||||
if (this.localizedResources != null) {
|
||||
return Collections.unmodifiableMap(this.localizedResources);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public String getUser() {
|
||||
return this.user;
|
||||
}
|
||||
|
||||
public List<String> getContainerLocalDirs() {
|
||||
return Collections.unmodifiableList(this.containerLocalDirs);
|
||||
}
|
||||
|
||||
public List<String> getCommands(){
|
||||
return this.commands;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Standard extensions get all permissions by default
|
||||
grant codeBase "file:${{java.ext.dirs}}/*" {
|
||||
permission java.security.AllPermission;
|
||||
};
|
||||
|
||||
// default permissions granted to all domains
|
||||
grant {
|
||||
permission java.lang.RuntimePermission "accessDeclaredMembers";
|
||||
|
||||
permission java.util.PropertyPermission "java.version", "read";
|
||||
permission java.util.PropertyPermission "java.vendor", "read";
|
||||
permission java.util.PropertyPermission "java.vendor.url", "read";
|
||||
permission java.util.PropertyPermission "java.class.version", "read";
|
||||
permission java.util.PropertyPermission "os.name", "read";
|
||||
permission java.util.PropertyPermission "os.version", "read";
|
||||
permission java.util.PropertyPermission "os.arch", "read";
|
||||
permission java.util.PropertyPermission "file.separator", "read";
|
||||
permission java.util.PropertyPermission "path.separator", "read";
|
||||
permission java.util.PropertyPermission "line.separator", "read";
|
||||
|
||||
permission java.util.PropertyPermission "java.specification.version", "read";
|
||||
permission java.util.PropertyPermission "java.specification.vendor", "read";
|
||||
permission java.util.PropertyPermission "java.specification.name", "read";
|
||||
|
||||
permission java.util.PropertyPermission "java.vm.specification.version", "read";
|
||||
permission java.util.PropertyPermission "java.vm.specification.vendor", "read";
|
||||
permission java.util.PropertyPermission "java.vm.specification.name", "read";
|
||||
permission java.util.PropertyPermission "java.vm.version", "read";
|
||||
permission java.util.PropertyPermission "java.vm.vendor", "read";
|
||||
permission java.util.PropertyPermission "java.vm.name", "read";
|
||||
|
||||
//additional hadoop permissions
|
||||
permission java.util.PropertyPermission "awt.Toolkit", "read";
|
||||
permission java.util.PropertyPermission "file.encoding", "read";
|
||||
permission java.util.PropertyPermission "file.encoding.pkg", "read";
|
||||
permission java.util.PropertyPermission "hadoop.metrics.log.level", "read";
|
||||
permission java.util.PropertyPermission "hadoop.root.logger", "read";
|
||||
permission java.util.PropertyPermission "java.awt.graphicsenv" ,"read";
|
||||
permission java.util.PropertyPermission "java.awt.printerjob", "read";
|
||||
permission java.util.PropertyPermission "java.class.path", "read";
|
||||
permission java.util.PropertyPermission "yarn.app.container.log.dir", "read";
|
||||
permission java.util.PropertyPermission "yarn.app.container.log.filesize", "read";
|
||||
permission java.lang.RuntimePermission "loadLibrary.gplcompression";
|
||||
permission javax.security.auth.AuthPermission "getSubject";
|
||||
};
|
||||
|
|
@ -0,0 +1,364 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.nodemanager.
|
||||
containermanager.linux.runtime;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.ExpectedException;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FilePermission;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.apache.hadoop.yarn.api.ApplicationConstants.Environment.JAVA_HOME;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CHAINED_COMMAND_REGEX;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CLEAN_CMD_REGEX;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.CONTAINS_JAVA_CMD;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FILE;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.POLICY_FLAG;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils.SECURITY_FLAG;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.JavaSandboxLinuxContainerRuntime.POLICY_FILE_DIR;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.APPID;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_LOCAL_DIRS;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_RUN_CMDS;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_WORK_DIR;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.FILECACHE_DIRS;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOCALIZED_RESOURCES;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOCAL_DIRS;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.LOG_DIRS;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.RUN_AS_USER;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER;
|
||||
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.USER_LOCAL_DIRS;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Test policy file generation and policy enforcement for the
|
||||
* {@link JavaSandboxLinuxContainerRuntime}.
|
||||
*/
|
||||
public class TestJavaSandboxLinuxContainerRuntime {
|
||||
|
||||
private static final String HADOOP_HOME = "hadoop.home.dir";
|
||||
private static String hadoopHomeDir = System.getProperty(HADOOP_HOME);
|
||||
|
||||
@Rule
|
||||
public ExpectedException exception = ExpectedException.none();
|
||||
|
||||
private static File grantFile, denyFile, policyFile,
|
||||
grantDir, denyDir, containerDir;
|
||||
private static java.nio.file.Path policyFilePath;
|
||||
private static SecurityManager securityManager;
|
||||
private Map<Path, List<String>> resources;
|
||||
private Map<String, String> env;
|
||||
private List<String> whitelistGroup;
|
||||
|
||||
private PrivilegedOperationExecutor mockExecutor;
|
||||
private JavaSandboxLinuxContainerRuntime runtime;
|
||||
private ContainerRuntimeContext.Builder runtimeContextBuilder;
|
||||
private Configuration conf;
|
||||
|
||||
private final static String NORMAL_USER = System.getProperty("user.name");
|
||||
private final static String NORMAL_GROUP = "normalGroup";
|
||||
private final static String WHITELIST_USER = "picard";
|
||||
private final static String WHITELIST_GROUP = "captains";
|
||||
private final static String CONTAINER_ID = "container_1234567890";
|
||||
private final static String APPLICATION_ID = "application_1234567890";
|
||||
|
||||
@Before
|
||||
public void setup() throws Exception {
|
||||
|
||||
File baseTestDirectory = new File(System.getProperty("test.build.data",
|
||||
System.getProperty("java.io.tmpdir", "target")),
|
||||
TestJavaSandboxLinuxContainerRuntime.class.getName());
|
||||
|
||||
whitelistGroup = new ArrayList<>();
|
||||
whitelistGroup.add(WHITELIST_GROUP);
|
||||
|
||||
conf = new Configuration();
|
||||
conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES,
|
||||
WHITELIST_USER + "=" + WHITELIST_GROUP + ";"
|
||||
+ NORMAL_USER + "=" + NORMAL_GROUP + ";");
|
||||
conf.set(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP,
|
||||
WHITELIST_GROUP);
|
||||
conf.set("hadoop.tmp.dir", baseTestDirectory.getAbsolutePath());
|
||||
|
||||
Files.deleteIfExists(Paths.get(baseTestDirectory.getAbsolutePath(),
|
||||
POLICY_FILE_DIR, CONTAINER_ID + "-" + POLICY_FILE));
|
||||
|
||||
mockExecutor = mock(PrivilegedOperationExecutor.class);
|
||||
runtime = new JavaSandboxLinuxContainerRuntime(mockExecutor);
|
||||
runtime.initialize(conf);
|
||||
|
||||
resources = new HashMap<>();
|
||||
grantDir = new File(baseTestDirectory, "grantDir");
|
||||
denyDir = new File(baseTestDirectory, "denyDir");
|
||||
containerDir = new File(baseTestDirectory,
|
||||
APPLICATION_ID + Path.SEPARATOR + CONTAINER_ID);
|
||||
grantDir.mkdirs();
|
||||
denyDir.mkdirs();
|
||||
containerDir.mkdirs();
|
||||
|
||||
grantFile = File.createTempFile("grantFile", "tmp", grantDir);
|
||||
denyFile = File.createTempFile("denyFile", "tmp", denyDir);
|
||||
|
||||
List<String> symLinks = new ArrayList<>();
|
||||
symLinks.add(grantFile.getName());
|
||||
resources.put(new Path(grantFile.getCanonicalPath()), symLinks);
|
||||
|
||||
env = new HashMap();
|
||||
env.put(JAVA_HOME.name(), System.getenv(JAVA_HOME.name()));
|
||||
|
||||
policyFile = File.createTempFile("java", "policy", containerDir);
|
||||
policyFilePath = Paths.get(policyFile.getAbsolutePath());
|
||||
|
||||
runtimeContextBuilder = createRuntimeContext();
|
||||
|
||||
if (hadoopHomeDir == null) {
|
||||
System.setProperty(HADOOP_HOME, policyFile.getParent());
|
||||
}
|
||||
|
||||
OutputStream outStream = new FileOutputStream(policyFile);
|
||||
JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
|
||||
.generatePolicyFile(outStream, symLinks, resources, conf);
|
||||
outStream.close();
|
||||
|
||||
System.setProperty("java.security.policy", policyFile.getCanonicalPath());
|
||||
securityManager = new SecurityManager();
|
||||
|
||||
}
|
||||
|
||||
public ContainerRuntimeContext.Builder createRuntimeContext(){
|
||||
|
||||
Container container = mock(Container.class);
|
||||
ContainerLaunchContext ctx = mock(ContainerLaunchContext.class);
|
||||
|
||||
when(container.getLaunchContext()).thenReturn(ctx);
|
||||
when(ctx.getEnvironment()).thenReturn(env);
|
||||
|
||||
ContainerRuntimeContext.Builder builder =
|
||||
new ContainerRuntimeContext.Builder(container);
|
||||
|
||||
List<String> localDirs = new ArrayList<>();
|
||||
|
||||
builder.setExecutionAttribute(LOCALIZED_RESOURCES, resources)
|
||||
.setExecutionAttribute(RUN_AS_USER, NORMAL_USER)
|
||||
.setExecutionAttribute(CONTAINER_ID_STR, CONTAINER_ID)
|
||||
.setExecutionAttribute(APPID, APPLICATION_ID)
|
||||
.setExecutionAttribute(CONTAINER_WORK_DIR,
|
||||
new Path(containerDir.toString()))
|
||||
.setExecutionAttribute(LOCAL_DIRS, localDirs)
|
||||
.setExecutionAttribute(LOG_DIRS, localDirs)
|
||||
.setExecutionAttribute(FILECACHE_DIRS, localDirs)
|
||||
.setExecutionAttribute(USER_LOCAL_DIRS, localDirs)
|
||||
.setExecutionAttribute(CONTAINER_LOCAL_DIRS, localDirs)
|
||||
.setExecutionAttribute(CONTAINER_RUN_CMDS, localDirs);
|
||||
|
||||
return builder;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGrant() throws Exception {
|
||||
FilePermission grantPermission =
|
||||
new FilePermission(grantFile.getAbsolutePath(), "read");
|
||||
securityManager.checkPermission(grantPermission);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeny() throws Exception {
|
||||
FilePermission denyPermission =
|
||||
new FilePermission(denyFile.getAbsolutePath(), "read");
|
||||
exception.expect(java.security.AccessControlException.class);
|
||||
securityManager.checkPermission(denyPermission);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEnforcingMode() throws ContainerExecutionException {
|
||||
String[] nonJavaCommands = {
|
||||
"bash malicious_script.sh",
|
||||
"python malicious_script.py"
|
||||
};
|
||||
|
||||
List<String> commands = Arrays.asList(nonJavaCommands);
|
||||
exception.expect(ContainerExecutionException.class);
|
||||
JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
|
||||
.appendSecurityFlags(commands, env, policyFilePath,
|
||||
JavaSandboxLinuxContainerRuntime.SandboxMode.enforcing);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPermissiveMode() throws ContainerExecutionException {
|
||||
String[] nonJavaCommands = {
|
||||
"bash non-java-script.sh",
|
||||
"python non-java-script.py"
|
||||
};
|
||||
|
||||
List<String> commands = Arrays.asList(nonJavaCommands);
|
||||
JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
|
||||
.appendSecurityFlags(commands, env, policyFilePath,
|
||||
JavaSandboxLinuxContainerRuntime.SandboxMode.permissive);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDisabledSandboxWithWhitelist()
|
||||
throws ContainerExecutionException {
|
||||
|
||||
String[] inputCommand = {
|
||||
"java jar MyJob.jar"
|
||||
};
|
||||
List<String> commands = Arrays.asList(inputCommand);
|
||||
|
||||
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
|
||||
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
|
||||
runtime.prepareContainer(runtimeContextBuilder.build());
|
||||
|
||||
Assert.assertTrue("Command should not be modified when user is " +
|
||||
"member of whitelisted group",
|
||||
inputCommand[0].equals(commands.get(0)));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEnabledSandboxWithWhitelist()
|
||||
throws ContainerExecutionException{
|
||||
String[] inputCommand = {
|
||||
"$JAVA_HOME/bin/java jar -Djava.security.manager MyJob.jar"
|
||||
};
|
||||
List<String> commands = Arrays.asList(inputCommand);
|
||||
|
||||
runtimeContextBuilder.setExecutionAttribute(USER, WHITELIST_USER);
|
||||
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
|
||||
runtime.prepareContainer(runtimeContextBuilder.build());
|
||||
|
||||
Assert.assertTrue("Command should be modified to include " +
|
||||
"policy file in whitelisted Sandbox mode",
|
||||
commands.get(0).contains(SECURITY_FLAG)
|
||||
&& commands.get(0).contains(POLICY_FLAG));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeniedWhitelistGroup() throws ContainerExecutionException {
|
||||
|
||||
String[] inputCommand = {
|
||||
"$JAVA_HOME/bin/java jar MyJob.jar"
|
||||
};
|
||||
List<String> commands = Arrays.asList(inputCommand);
|
||||
|
||||
runtimeContextBuilder.setExecutionAttribute(USER, NORMAL_USER);
|
||||
runtimeContextBuilder.setExecutionAttribute(CONTAINER_RUN_CMDS, commands);
|
||||
runtime.prepareContainer(runtimeContextBuilder.build());
|
||||
|
||||
Assert.assertTrue("Java security manager must be enabled for "
|
||||
+ "unauthorized users",
|
||||
commands.get(0).contains(SECURITY_FLAG));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testChainedCmdRegex(){
|
||||
Assert.assertTrue("cmd1 && cmd2 || cmd3".matches(CHAINED_COMMAND_REGEX));
|
||||
Assert.assertFalse("cmd1 &> logfile".matches(CHAINED_COMMAND_REGEX));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testContainsJavaRegex(){
|
||||
String[] javaCmds = {
|
||||
"$JAVA_HOME/bin/java -cp App.jar AppClass",
|
||||
"$JAVA_HOME/bin/java -jar App.jar AppClass &> logfile"
|
||||
};
|
||||
String[] nonJavaCmds = {
|
||||
"$JAVA_HOME/bin/jajavava -cp App.jar AppClass",
|
||||
"/nm/app/container/usercache/badjava -cp Bad.jar ChaosClass"
|
||||
};
|
||||
for(String javaCmd : javaCmds) {
|
||||
Assert.assertTrue(javaCmd.matches(CONTAINS_JAVA_CMD));
|
||||
}
|
||||
for(String nonJavaCmd : nonJavaCmds) {
|
||||
Assert.assertFalse(nonJavaCmd.matches(CONTAINS_JAVA_CMD));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCleanCmdRegex(){
|
||||
String[] securityManagerCmds = {
|
||||
"/usr/bin/java -Djava.security.manager -cp $CLASSPATH $MainClass",
|
||||
"-Djava.security.manager -Djava.security.policy==testpolicy keepThis"
|
||||
};
|
||||
String[] cleanedCmdsResult = {
|
||||
"/usr/bin/java -cp $CLASSPATH $MainClass",
|
||||
"keepThis"
|
||||
};
|
||||
for(int i = 0; i < securityManagerCmds.length; i++){
|
||||
Assert.assertEquals(
|
||||
securityManagerCmds[i].replaceAll(CLEAN_CMD_REGEX, "").trim(),
|
||||
cleanedCmdsResult[i]);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppendSecurityFlags() throws ContainerExecutionException {
|
||||
String securityString = "-Djava.security.manager -Djava.security.policy=="
|
||||
+ policyFile.getAbsolutePath();
|
||||
String[] badCommands = {
|
||||
"$JAVA_HOME/bin/java -Djava.security.manager "
|
||||
+ "-Djava.security.policy=/home/user/java.policy",
|
||||
"$JAVA_HOME/bin/java -cp MyApp.jar MrAppMaster"
|
||||
};
|
||||
String[] cleanCommands = {
|
||||
"$JAVA_HOME/bin/java " + securityString,
|
||||
"$JAVA_HOME/bin/java " + securityString
|
||||
+ " -cp MyApp.jar MrAppMaster"
|
||||
};
|
||||
|
||||
List<String> commands = Arrays.asList(badCommands);
|
||||
JavaSandboxLinuxContainerRuntime.NMContainerPolicyUtils
|
||||
.appendSecurityFlags(commands, env, policyFilePath,
|
||||
JavaSandboxLinuxContainerRuntime.SandboxMode.enforcing);
|
||||
|
||||
for(int i = 0; i < commands.size(); i++) {
|
||||
Assert.assertTrue(commands.get(i).trim().equals(cleanCommands[i].trim()));
|
||||
}
|
||||
}
|
||||
|
||||
@After
|
||||
public void cleanup(){
|
||||
System.setSecurityManager(null);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue