Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8962. Add ability to use interactive shell with normal yarn container. Contributed by Eric Yang

This commit is contained in:
Billie Rinaldi 2018-12-12 18:18:57 -08:00
parent 37eb919c59
commit 72e7c6a489
8 changed files with 281 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
View File

@ -44,7 +44,7 @@ public enum OperationType {
INITIALIZE_CONTAINER(""), //no CLI switch supported yet
LAUNCH_CONTAINER(""), //no CLI switch supported yet
SIGNAL_CONTAINER(""), //no CLI switch supported yet
EXEC_CONTAINER("--run-docker"), //no CLI switch supported yet
EXEC_CONTAINER("--exec-container"), //no CLI switch supported yet
DELETE_AS_USER(""), //no CLI switch supported yet
LAUNCH_DOCKER_CONTAINER(""), //no CLI switch supported yet
TC_MODIFY_STATE("--tc-modify-state"),

100
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
View File

@ -33,6 +33,7 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
@ -41,6 +42,14 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -57,6 +66,8 @@
public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
private static final Logger LOG =
LoggerFactory.getLogger(DefaultLinuxContainerRuntime.class);
private static final String TMP_FILE_PREFIX = "yarn.";
private static final String TMP_FILE_SUFFIX = ".cmd";
private final PrivilegedOperationExecutor privilegedOperationExecutor;
private Configuration conf;
@ -198,8 +209,93 @@ public String[] getIpAndHost(Container container) {
}
@Override
public IOStreamPair execContainer(ContainerExecContext containerExecContext)
public IOStreamPair execContainer(ContainerExecContext ctx)
throws ContainerExecutionException {
throw new ContainerExecutionException("Unsupported operation.");
IOStreamPair output;
try {
PrivilegedOperation privOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.EXEC_CONTAINER);
String commandFile = writeCommandToTempFile(ctx);
privOp.appendArgs(commandFile);
privOp.disableFailureLogging();
output =
privilegedOperationExecutor.executePrivilegedInteractiveOperation(
null, privOp);
} catch (PrivilegedOperationException e) {
throw new ContainerExecutionException(
"Execute container interactive shell failed", e.getExitCode(),
e.getOutput(), e.getErrorOutput());
} catch (InterruptedException ie) {
LOG.warn("InterruptedException executing command: ", ie);
throw new ContainerExecutionException(ie.getMessage());
}
return output;
}
private String writeCommandToTempFile(ContainerExecContext ctx)
throws ContainerExecutionException {
Container container = ctx.getContainer();
File cmdDir = null;
String appId = container.getContainerId().getApplicationAttemptId()
.getApplicationId().toString();
String containerId = container.getContainerId().toString();
String filePrefix = containerId.toString();
try {
String cmdDirPath = ctx.getLocalDirsHandlerService().getLocalPathForWrite(
ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR +
appId + Path.SEPARATOR + filePrefix + Path.SEPARATOR).toString();
cmdDir = new File(cmdDirPath);
if (!cmdDir.mkdirs() && !cmdDir.exists()) {
throw new IOException("Cannot create container private directory "
+ cmdDir);
}
File commandFile = File.createTempFile(TMP_FILE_PREFIX + filePrefix,
TMP_FILE_SUFFIX, cmdDir);
try (
Writer writer = new OutputStreamWriter(
new FileOutputStream(commandFile.toString()), "UTF-8");
PrintWriter printWriter = new PrintWriter(writer);
) {
Map<String, List<String>> cmd = new HashMap<String, List<String>>();
// command = exec
List<String> exec = new ArrayList<String>();
exec.add("exec");
cmd.put("command", exec);
// user = foobar
List<String> user = new ArrayList<String>();
user.add(container.getUser());
cmd.put("user", user);
// launch-command = bash,-i
List<String> commands = new ArrayList<String>();
commands.add("/bin/bash");
commands.add("-ir");
cmd.put("launch-command", commands);
// workdir = ../nm-local-dir/usercache/appcache/appid/containerid
List<String> workdir = new ArrayList<String>();
workdir.add(container.getWorkDir());
cmd.put("workdir", workdir);
// generate cmd file
printWriter.println("[command-execution]");
for (Map.Entry<String, List<String>> entry :
cmd.entrySet()) {
if (entry.getKey().contains("=")) {
throw new ContainerExecutionException(
"'=' found in entry for docker command file, key = " + entry
.getKey() + "; value = " + entry.getValue());
}
if (entry.getValue().contains("\n")) {
throw new ContainerExecutionException(
"'\\n' found in entry for docker command file, key = " + entry
.getKey() + "; value = " + entry.getValue());
}
printWriter.println(" " + entry.getKey() + "=" + StringUtils
.join(",", entry.getValue()));
}
return commandFile.toString();
}
} catch (IOException e) {
LOG.warn("Unable to write command to " + cmdDir);
throw new ContainerExecutionException(e);
}
}
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerExecContext.java
View File

@ -22,6 +22,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
/**
@ -34,6 +35,7 @@ public final class ContainerExecContext {
private final String user;
private final String appId;
private final Container container;
private final LocalDirsHandlerService localDirsHandler;
/**
* Builder for ContainerExecContext.
@ -42,6 +44,7 @@ public static final class Builder {
private String user;
private String appId;
private Container container;
private LocalDirsHandlerService localDirsHandler;
public Builder() {
}
@ -64,12 +67,19 @@ public Builder setAppId(String appId) {
public ContainerExecContext build() {
return new ContainerExecContext(this);
}
public Builder setNMLocalPath(
LocalDirsHandlerService ldhs) {
this.localDirsHandler = ldhs;
return this;
}
}
private ContainerExecContext(Builder builder) {
this.container = builder.container;
this.user = builder.user;
this.appId = builder.appId;
this.localDirsHandler = builder.localDirsHandler;
}
public String getUser() {
@ -83,4 +93,8 @@ public String getAppId() {
public Container getContainer() {
return this.container;
}
public LocalDirsHandlerService getLocalDirsHandlerService() {
return this.localDirsHandler;
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
View File

@ -112,6 +112,7 @@ public void onConnect(Session session) {
ContainerExecContext execContext = new ContainerExecContext
.Builder()
.setContainer(container)
.setNMLocalPath(nmContext.getLocalDirsHandler())
.build();
pair = exec.execContainer(execContext);
} catch (Exception e) {

110
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
View File

@ -1374,28 +1374,12 @@ char **construct_docker_command(const char *command_file) {
}
int run_docker(const char *command_file) {
struct configuration command_config = {0, NULL};
int ret = read_config(command_file, &command_config);
if (ret != 0) {
free_configuration(&command_config);
return INVALID_COMMAND_FILE;
}
char *value = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (value != NULL && strcasecmp(value, "exec") == 0) {
free(value);
free_configuration(&command_config);
return run_docker_with_pty(command_file);
}
free_configuration(&command_config);
free(value);
char **args = construct_docker_command(command_file);
char* docker_binary = get_docker_binary(&CFG);
char *docker_binary = get_docker_binary(&CFG);
int exit_code = -1;
if (execvp(docker_binary, args) != 0) {
fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s",
fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s\n",
docker_binary, strerror(errno));
fflush(LOGFILE);
fflush(ERRORFILE);
@ -1409,12 +1393,55 @@ int run_docker(const char *command_file) {
return exit_code;
}
int run_docker_with_pty(const char *command_file) {
int exec_container(const char *command_file) {
int exit_code = -1;
char **args = construct_docker_command(command_file);
char* docker_binary = get_docker_binary(&CFG);
struct configuration command_config = {0, NULL};
char **args = NULL;
char **env = NULL;
char *workdir = NULL;
char *docker_binary = get_docker_binary(&CFG);
char *binary = NULL;
int fdm, fds, rc;
char input[4000];
int docker = 0;
char *user = NULL;
int ret = read_config(command_file, &command_config);
if (ret != 0) {
free_configuration(&command_config);
return INVALID_COMMAND_FILE;
}
char *value = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (value != NULL && strcasecmp(value, "exec") == 0) {
args = construct_docker_command(command_file);
binary = docker_binary;
docker = 1;
} else {
value = get_configuration_value("command", COMMAND_FILE_SECTION, &command_config);
if (value != NULL && strcasecmp(value, "exec") == 0) {
args = get_configuration_values_delimiter("launch-command",
COMMAND_FILE_SECTION, &command_config, ",");
if (args == NULL) {
goto cleanup;
}
binary = strdup(args[0]);
workdir = get_configuration_value("workdir", COMMAND_FILE_SECTION, &command_config);
if (workdir == NULL) {
goto cleanup;
}
env = (char **) alloc_and_clear_memory(3, sizeof(char *));
env[0] = make_string("PWD=%s", workdir);
env[1] = make_string("TERM=%s", "xterm-256color");
env[2] = NULL;
user = get_configuration_value("user", COMMAND_FILE_SECTION, &command_config);
if (user == NULL) {
goto cleanup;
}
} else {
goto cleanup;
}
}
fdm = posix_openpt(O_RDWR);
if (fdm < 0) {
@ -1513,6 +1540,9 @@ int run_docker_with_pty(const char *command_file) {
// Set raw mode on the slave side of the PTY
new_term_settings = slave_orig_term_settings;
cfmakeraw (&new_term_settings);
if (!docker) {
new_term_settings.c_lflag |= ECHO;
}
tcsetattr (fds, TCSANOW, &new_term_settings);
// The slave side of the PTY becomes the standard input and outputs of the child process
@ -1537,24 +1567,48 @@ int run_docker_with_pty(const char *command_file) {
close(fds);
// Make the current process a new session leader
if (docker) {
setsid();
} else {
exit_code = set_user(user);
if (exit_code!=0) {
goto cleanup;
}
}
// As the child is a session leader, set the controlling terminal to be the slave side of the PTY
// (Mandatory for programs like the shell to make them manage correctly their outputs)
ioctl(0, TIOCSCTTY, 1);
if (execvp(docker_binary, args) != 0) {
fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s",
docker_binary, strerror(errno));
free(docker_binary);
free_values(args);
if (docker) {
ret = execvp(binary, args);
} else {
if (change_user(user_detail->pw_uid, user_detail->pw_gid) != 0) {
exit_code = DOCKER_EXEC_FAILED;
goto cleanup;
}
ret = chdir(workdir);
if (ret != 0) {
exit_code = DOCKER_EXEC_FAILED;
goto cleanup;
}
ret = execve(binary, args, env);
}
if (ret != 0) {
fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s\n",
binary, strerror(errno));
exit_code = DOCKER_EXEC_FAILED;
} else {
free_values(args);
exit_code = 0;
}
}
cleanup:
free(docker_binary);
free(binary);
free(user);
free(workdir);
free_values(args);
free_configuration(&command_config);
return exit_code;
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
View File

@ -51,7 +51,8 @@ enum operations {
RUN_AS_USER_LIST = 12,
REMOVE_DOCKER_CONTAINER = 13,
INSPECT_DOCKER_CONTAINER = 14,
RUN_AS_USER_SYNC_YARN_SYSFS = 15
RUN_AS_USER_SYNC_YARN_SYSFS = 15,
EXEC_CONTAINER = 16
};
#define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group"
@ -71,6 +72,7 @@ enum operations {
#define MOUNT_CGROUP_SUPPORT_ENABLED_KEY "feature.mount-cgroup.enabled"
#define YARN_SYSFS_SUPPORT_ENABLED_KEY "feature.yarn.sysfs.enabled"
#define TMP_DIR "tmp"
#define COMMAND_FILE_SECTION "command-execution"
extern struct passwd *user_detail;
@ -296,6 +298,11 @@ int run_docker_with_pty(const char *command_file);
*/
int exec_docker_command(char *docker_command, char **argv, int argc);
/**
* Exec a container terminal.
*/
int exec_container(const char *command_file);
/** Check if yarn sysfs is enabled in configuration. */
int is_yarn_sysfs_support_enabled();

31
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
View File

@ -54,12 +54,14 @@ static void display_usage(FILE *stream) {
if(is_docker_support_enabled()) {
fprintf(stream,
" container-executor --run-docker <command-file>\n"
" container-executor --exec-container <command-file>\n"
" container-executor --remove-docker-container [hierarchy] "
"<container_id>\n"
" container-executor --inspect-docker-container <container_id>\n");
} else {
fprintf(stream,
"[DISABLED] container-executor --run-docker <command-file>\n"
"[DISABLED] container-executor --exec-container <command-file>\n"
"[DISABLED] container-executor --remove-docker-container [hierarchy] "
"<container_id>\n"
"[DISABLED] container-executor --inspect-docker-container "
@ -247,7 +249,7 @@ static struct {
const char *target_dir;
int container_pid;
int signal;
const char *docker_command_file;
const char *command_file;
} cmd_input;
static int validate_run_as_user_commands(int argc, char **argv, int *operation);
@ -348,6 +350,22 @@ static int validate_arguments(int argc, char **argv , int *operation) {
}
}
if (strcmp("--exec-container", argv[1]) == 0) {
if(is_docker_support_enabled()) {
if (argc != 3) {
display_usage(stdout);
return INVALID_ARGUMENT_NUMBER;
}
optind++;
cmd_input.command_file = argv[optind++];
*operation = EXEC_CONTAINER;
return 0;
} else {
display_feature_disabled_message("docker");
return FEATURE_DISABLED;
}
}
if (strcmp("--run-docker", argv[1]) == 0) {
if(is_docker_support_enabled()) {
if (argc != 3) {
@ -355,7 +373,7 @@ static int validate_arguments(int argc, char **argv , int *operation) {
return INVALID_ARGUMENT_NUMBER;
}
optind++;
cmd_input.docker_command_file = argv[optind++];
cmd_input.command_file = argv[optind++];
*operation = RUN_DOCKER;
return 0;
} else {
@ -469,7 +487,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
cmd_input.local_dirs = argv[optind++];
// good log dirs as a comma separated list
cmd_input.log_dirs = argv[optind++];
cmd_input.docker_command_file = argv[optind++];
cmd_input.command_file = argv[optind++];
//network isolation through tc
if ((argc == 15 && !cmd_input.https) || (argc == 17 && cmd_input.https)) {
if(is_tc_support_enabled()) {
@ -624,8 +642,11 @@ int main(int argc, char **argv) {
case TRAFFIC_CONTROL_READ_STATS:
exit_code = traffic_control_read_stats(cmd_input.traffic_control_command_file);
break;
case EXEC_CONTAINER:
exit_code = exec_container(cmd_input.command_file);
break;
case RUN_DOCKER:
exit_code = run_docker(cmd_input.docker_command_file);
exit_code = run_docker(cmd_input.command_file);
break;
case REMOVE_DOCKER_CONTAINER:
exit_code = remove_docker_container(argv + optind, argc - optind);
@ -674,7 +695,7 @@ int main(int argc, char **argv) {
cmd_input.pid_file,
split(cmd_input.local_dirs),
split(cmd_input.log_dirs),
cmd_input.docker_command_file);
cmd_input.command_file);
break;
case RUN_AS_USER_LAUNCH_CONTAINER:
if (cmd_input.traffic_control_command_file != NULL) {

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
View File

@ -1472,6 +1472,53 @@ void test_cleaning_docker_cgroups() {
}
}
void test_exec_container() {
int ret = -1;
char* filename = TEST_ROOT "/exec_container.cmd";
FILE *file = fopen(filename, "w");
if (file == NULL) {
printf("FAIL: Could not write to command file: %s\n", filename);
exit(1);
}
// Test missing user
fprintf(file, "[command-execution]\n");
fprintf(file, "workdir=/tmp/container_1541184499854_0001_01_000001\n");
fprintf(file, "launch-command=/bin/bash,-ir\n");
fprintf(file, "command=exec\n");
fclose(file);
ret = exec_container(filename);
if (ret!=-1) {
printf("FAIL: broken command file should not pass.\n");
exit(1);
}
// Test missing workdir
file = fopen(filename, "w");
fprintf(file, "[command-execution]\n");
fprintf(file, "launch-command=/bin/bash,-ir\n");
fprintf(file, "user=test\n");
fprintf(file, "command=exec\n");
fclose(file);
ret = exec_container(filename);
if (ret!=-1) {
printf("FAIL: broken command file should not pass.\n");
exit(1);
}
// Test missing launch-command
file = fopen(filename, "w");
fprintf(file, "[command-execution]\n");
fprintf(file, "workdir=/tmp/container_1541184499854_0001_01_000001\n");
fprintf(file, "user=test\n");
fprintf(file, "command=exec\n");
fclose(file);
ret = exec_container(filename);
if (ret!=-1) {
printf("FAIL: broken command file should not pass.\n");
exit(1);
}
}
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@ -1612,6 +1659,9 @@ int main(int argc, char **argv) {
printf("\nTesting yarn sysfs\n");
test_yarn_sysfs();
printf("\nTesting exec_container()\n");
test_exec_container();
test_check_user(0);
test_cleaning_docker_cgroups();