YARN-4266. Allow users to enter containers as UID:GID pair instead of by username. Contributed by luhuichun, Zhankun Tang, and Eric Badger.
This commit is contained in:
parent
e5e1851d80
commit
bfd1a72ba8
|
@ -1587,6 +1587,27 @@ public class YarnConfiguration extends Configuration {
|
|||
public static final boolean DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS =
|
||||
false;
|
||||
|
||||
/** enable user remapping. */
|
||||
public static final String NM_DOCKER_ENABLE_USER_REMAPPING =
|
||||
DOCKER_CONTAINER_RUNTIME_PREFIX + "enable-userremapping.allowed";
|
||||
|
||||
/** Set enable user remapping as false by default. */
|
||||
public static final boolean DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING = false;
|
||||
|
||||
/** lower limit for acceptable uids of user remapped user. */
|
||||
public static final String NM_DOCKER_USER_REMAPPING_UID_THRESHOLD =
|
||||
DOCKER_CONTAINER_RUNTIME_PREFIX + "userremapping-uid-threshold";
|
||||
|
||||
/** Set user remapping lower uid limit to 1 by default. */
|
||||
public static final int DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD = 1;
|
||||
|
||||
/** lower limit for acceptable gids of user remapped user. */
|
||||
public static final String NM_DOCKER_USER_REMAPPING_GID_THRESHOLD =
|
||||
DOCKER_CONTAINER_RUNTIME_PREFIX + "userremapping-gid-threshold";
|
||||
|
||||
/** Set user remapping lower gid limit to 1 by default. */
|
||||
public static final int DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD = 1;
|
||||
|
||||
/** ACL list for users allowed to run privileged containers. */
|
||||
public static final String NM_DOCKER_PRIVILEGED_CONTAINERS_ACL =
|
||||
DOCKER_CONTAINER_RUNTIME_PREFIX + "privileged-containers.acl";
|
||||
|
|
|
@ -1642,6 +1642,25 @@
|
|||
<value>host</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>Property to enable docker user remapping</description>
|
||||
<name>yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>lower limit for acceptable uids of user remapped user</description>
|
||||
<name>yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold</name>
|
||||
<value>1</value>
|
||||
</property>
|
||||
|
||||
|
||||
<property>
|
||||
<description>lower limit for acceptable gids of user remapped user</description>
|
||||
<name>yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold</name>
|
||||
<value>1</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>The mode in which the Java Container Sandbox should run detailed by
|
||||
the JavaSandboxLinuxContainerRuntime.</description>
|
||||
|
|
|
@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Path;
|
|||
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.authorize.AccessControlList;
|
||||
import org.apache.hadoop.util.Shell;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
|
||||
|
@ -164,6 +165,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER =
|
||||
"YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER";
|
||||
@InterfaceAudience.Private
|
||||
public static final String ENV_DOCKER_CONTAINER_RUN_ENABLE_USER_REMAPPING =
|
||||
"YARN_CONTAINER_RUNTIME_DOCKER_RUN_ENABLE_USER_REMAPPING";
|
||||
@InterfaceAudience.Private
|
||||
public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS =
|
||||
"YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS";
|
||||
|
||||
|
@ -175,6 +179,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
private String cgroupsRootDirectory;
|
||||
private CGroupsHandler cGroupsHandler;
|
||||
private AccessControlList privilegedContainersAcl;
|
||||
private boolean enableUserReMapping;
|
||||
private int userRemappingUidThreshold;
|
||||
private int userRemappingGidThreshold;
|
||||
|
||||
/**
|
||||
* Return whether the given environment variables indicate that the operation
|
||||
|
@ -260,6 +267,18 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
privilegedContainersAcl = new AccessControlList(conf.getTrimmed(
|
||||
YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL,
|
||||
YarnConfiguration.DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL));
|
||||
|
||||
enableUserReMapping = conf.getBoolean(
|
||||
YarnConfiguration.NM_DOCKER_ENABLE_USER_REMAPPING,
|
||||
YarnConfiguration.DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING);
|
||||
|
||||
userRemappingUidThreshold = conf.getInt(
|
||||
YarnConfiguration.NM_DOCKER_USER_REMAPPING_UID_THRESHOLD,
|
||||
YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD);
|
||||
|
||||
userRemappingGidThreshold = conf.getInt(
|
||||
YarnConfiguration.NM_DOCKER_USER_REMAPPING_GID_THRESHOLD,
|
||||
YarnConfiguration.DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -436,6 +455,34 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
"resource: " + mount);
|
||||
}
|
||||
|
||||
private String getUserIdInfo(String userName)
|
||||
throws ContainerExecutionException {
|
||||
String id = "";
|
||||
Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
|
||||
new String[]{"id", "-u", userName});
|
||||
try {
|
||||
shexec.execute();
|
||||
id = shexec.getOutput().replaceAll("[^0-9]", "");
|
||||
} catch (Exception e) {
|
||||
throw new ContainerExecutionException(e);
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
private String[] getGroupIdInfo(String userName)
|
||||
throws ContainerExecutionException {
|
||||
String[] id = null;
|
||||
Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
|
||||
new String[]{"id", "-G", userName});
|
||||
try {
|
||||
shexec.execute();
|
||||
id = shexec.getOutput().replace("\n", "").split(" ");
|
||||
} catch (Exception e) {
|
||||
throw new ContainerExecutionException(e);
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void launchContainer(ContainerRuntimeContext ctx)
|
||||
throws ContainerExecutionException {
|
||||
|
@ -458,7 +505,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
|
||||
String containerIdStr = container.getContainerId().toString();
|
||||
String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
|
||||
String dockerRunAsUser = runAsUser;
|
||||
Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
|
||||
String[] groups = null;
|
||||
|
||||
if (enableUserReMapping) {
|
||||
String uid = getUserIdInfo(runAsUser);
|
||||
groups = getGroupIdInfo(runAsUser);
|
||||
String gid = groups[0];
|
||||
if(Integer.parseInt(uid) < userRemappingUidThreshold) {
|
||||
String message = "uid: " + uid + " below threshold: "
|
||||
+ userRemappingUidThreshold;
|
||||
throw new ContainerExecutionException(message);
|
||||
}
|
||||
for(int i = 0; i < groups.length; i++) {
|
||||
String group = groups[i];
|
||||
if (Integer.parseInt(group) < userRemappingGidThreshold) {
|
||||
String message = "gid: " + group
|
||||
+ " below threshold: " + userRemappingGidThreshold;
|
||||
throw new ContainerExecutionException(message);
|
||||
}
|
||||
}
|
||||
dockerRunAsUser = uid + ":" + gid;
|
||||
}
|
||||
|
||||
//List<String> -> stored as List -> fetched/converted to List<String>
|
||||
//we can't do better here thanks to type-erasure
|
||||
@SuppressWarnings("unchecked")
|
||||
|
@ -481,7 +551,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
|
||||
@SuppressWarnings("unchecked")
|
||||
DockerRunCommand runCommand = new DockerRunCommand(containerIdStr,
|
||||
runAsUser, imageName)
|
||||
dockerRunAsUser, imageName)
|
||||
.detachOnRun()
|
||||
.setContainerWorkDir(containerWorkDir.toString())
|
||||
.setNetworkType(network);
|
||||
|
@ -542,6 +612,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||
runCommand.setOverrideCommandWithArgs(overrideCommands);
|
||||
}
|
||||
|
||||
if(enableUserReMapping) {
|
||||
runCommand.groupAdd(groups);
|
||||
}
|
||||
|
||||
String commandFile = dockerClient.writeCommandToTempFile(runCommand,
|
||||
containerIdStr);
|
||||
PrivilegedOperation launchOp = buildLaunchOp(ctx,
|
||||
|
|
|
@ -114,6 +114,13 @@ public class DockerRunCommand extends DockerCommand {
|
|||
return this;
|
||||
}
|
||||
|
||||
public DockerRunCommand groupAdd(String[] groups) {
|
||||
for(int i = 0; i < groups.length; i++) {
|
||||
super.addCommandArguments("--group-add " + groups[i]);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public DockerRunCommand setOverrideCommandWithArgs(
|
||||
List<String> overrideCommandWithArgs) {
|
||||
this.overrrideCommandWithArgs = overrideCommandWithArgs;
|
||||
|
|
|
@ -1258,6 +1258,7 @@ char* sanitize_docker_command(const char *line) {
|
|||
{"device", required_argument, 0, 'i' },
|
||||
{"detach", required_argument, 0, 't' },
|
||||
{"format", required_argument, 0, 'f' },
|
||||
{"group-add", required_argument, 0, 'x' },
|
||||
{0, 0, 0, 0}
|
||||
};
|
||||
|
||||
|
@ -1357,6 +1358,9 @@ char* sanitize_docker_command(const char *line) {
|
|||
strcat(output, optarg);
|
||||
strcat(output, " ");
|
||||
break;
|
||||
case 'x':
|
||||
quote_and_append_arg(&output, &output_size, "--group-add ", optarg);
|
||||
break;
|
||||
default:
|
||||
fprintf(LOGFILE, "Unknown option in docker command, character %d %c, optionindex = %d\n", c, c, optind);
|
||||
fflush(LOGFILE);
|
||||
|
|
|
@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
|
|||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
|
||||
import org.apache.hadoop.util.Shell;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
|
@ -77,6 +78,7 @@ public class TestDockerContainerRuntime {
|
|||
private ContainerLaunchContext context;
|
||||
private HashMap<String, String> env;
|
||||
private String image;
|
||||
private String uidGidPair;
|
||||
private String runAsUser;
|
||||
private String user;
|
||||
private String appId;
|
||||
|
@ -126,6 +128,7 @@ public class TestDockerContainerRuntime {
|
|||
when(context.getEnvironment()).thenReturn(env);
|
||||
when(container.getUser()).thenReturn(submittingUser);
|
||||
|
||||
uidGidPair = "";
|
||||
runAsUser = "run_as_user";
|
||||
user = "user";
|
||||
appId = "app_id";
|
||||
|
@ -230,7 +233,6 @@ public class TestDockerContainerRuntime {
|
|||
Assert.assertEquals(13, args.size());
|
||||
|
||||
//verify arguments
|
||||
Assert.assertEquals(runAsUser, args.get(0));
|
||||
Assert.assertEquals(user, args.get(1));
|
||||
Assert.assertEquals(Integer.toString(PrivilegedOperation.RunAsUserCommand
|
||||
.LAUNCH_DOCKER_CONTAINER.getValue()), args.get(2));
|
||||
|
@ -321,6 +323,81 @@ public class TestDockerContainerRuntime {
|
|||
Assert.assertEquals(expectedCommand, dockerCommands.get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testContainerLaunchWithUserRemapping()
|
||||
throws ContainerExecutionException, PrivilegedOperationException,
|
||||
IOException {
|
||||
conf.setBoolean(YarnConfiguration.NM_DOCKER_ENABLE_USER_REMAPPING,
|
||||
true);
|
||||
Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
|
||||
new String[]{"whoami"});
|
||||
shexec.execute();
|
||||
// get rid of newline at the end
|
||||
runAsUser = shexec.getOutput().replaceAll("\n$", "");
|
||||
builder.setExecutionAttribute(RUN_AS_USER, runAsUser);
|
||||
|
||||
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
|
||||
mockExecutor, mockCGroupsHandler);
|
||||
runtime.initialize(conf);
|
||||
runtime.launchContainer(builder.build());
|
||||
|
||||
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
|
||||
List<String> args = op.getArguments();
|
||||
String dockerCommandFile = args.get(11);
|
||||
|
||||
String uid = "";
|
||||
String gid = "";
|
||||
Shell.ShellCommandExecutor shexec1 = new Shell.ShellCommandExecutor(
|
||||
new String[]{"id", "-u", runAsUser});
|
||||
Shell.ShellCommandExecutor shexec2 = new Shell.ShellCommandExecutor(
|
||||
new String[]{"id", "-g", runAsUser});
|
||||
try {
|
||||
shexec1.execute();
|
||||
// get rid of newline at the end
|
||||
uid = shexec1.getOutput().replaceAll("\n$", "");
|
||||
} catch (Exception e) {
|
||||
LOG.info("Could not run id -u command: " + e);
|
||||
}
|
||||
try {
|
||||
shexec2.execute();
|
||||
// get rid of newline at the end
|
||||
gid = shexec2.getOutput().replaceAll("\n$", "");
|
||||
} catch (Exception e) {
|
||||
LOG.info("Could not run id -g command: " + e);
|
||||
}
|
||||
uidGidPair = uid + ":" + gid;
|
||||
|
||||
//This is the expected docker invocation for this case
|
||||
StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ")
|
||||
.append("--user=%2$s -d ")
|
||||
.append("--workdir=%3$s ")
|
||||
.append("--net=host ")
|
||||
.append("--hostname=" + defaultHostname + " ")
|
||||
.append(getExpectedTestCapabilitiesArgumentString())
|
||||
.append(getExpectedCGroupsMountString())
|
||||
.append("-v %4$s:%4$s ")
|
||||
.append("-v %5$s:%5$s ")
|
||||
.append("-v %6$s:%6$s ")
|
||||
.append("-v %7$s:%7$s ")
|
||||
.append("-v %8$s:%8$s ")
|
||||
.append("(--group-add \\d+ )*")
|
||||
.append("%9$s ")
|
||||
.append("bash %10$s/launch_container.sh");
|
||||
|
||||
String expectedCommand = String
|
||||
.format(expectedCommandTemplate.toString(), containerId, uidGidPair,
|
||||
containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0),
|
||||
containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0),
|
||||
image, containerWorkDir);
|
||||
|
||||
List<String> dockerCommands = Files.readAllLines(
|
||||
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
|
||||
|
||||
Assert.assertEquals(1, dockerCommands.size());
|
||||
//Assert.assertEquals(expectedCommand, dockerCommands.get(0));
|
||||
Assert.assertTrue(dockerCommands.get(0).matches(expectedCommand));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAllowedNetworksConfiguration() throws
|
||||
ContainerExecutionException {
|
||||
|
|
Loading…
Reference in New Issue