Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6988. container-executor fails for docker when command length > 4096 B. Contributed by Eric Badger

This commit is contained in:
Jason Lowe 2017-08-22 16:32:45 -05:00
parent 5cb8a92bdb
commit ff9cbcbbf4
2 changed files with 35 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
View File

@ -1308,8 +1308,10 @@ char* parse_docker_command_file(const char* command_file) {
int run_docker(const char *command_file) { int run_docker(const char *command_file) {
char* docker_command = parse_docker_command_file(command_file); char* docker_command = parse_docker_command_file(command_file);
char* docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg); char* docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg);
char* docker_command_with_binary = calloc(sizeof(char), EXECUTOR_PATH_MAX); size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command);
char* docker_command_with_binary = calloc(sizeof(char), command_size);
snprintf(docker_command_with_binary, command_size, "%s %s", docker_binary, docker_command);
char **args = extract_values_delim(docker_command_with_binary, " "); char **args = extract_values_delim(docker_command_with_binary, " ");
int exit_code = -1; int exit_code = -1;
@ -1455,16 +1457,24 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
char *script_file_dest = NULL; char *script_file_dest = NULL;
char *cred_file_dest = NULL; char *cred_file_dest = NULL;
char *exit_code_file = NULL; char *exit_code_file = NULL;
char docker_command_with_binary[EXECUTOR_PATH_MAX]; char *docker_command_with_binary = NULL;
char docker_wait_command[EXECUTOR_PATH_MAX]; char *docker_wait_command = NULL;
char docker_logs_command[EXECUTOR_PATH_MAX]; char *docker_logs_command = NULL;
char docker_inspect_command[EXECUTOR_PATH_MAX]; char *docker_inspect_command = NULL;
char docker_rm_command[EXECUTOR_PATH_MAX]; char *docker_rm_command = NULL;
int container_file_source =-1; int container_file_source =-1;
int cred_file_source = -1; int cred_file_source = -1;
int BUFFER_SIZE = 4096; int BUFFER_SIZE = 4096;
char buffer[BUFFER_SIZE]; char buffer[BUFFER_SIZE];
size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
docker_command_with_binary = calloc(sizeof(char), command_size);
docker_wait_command = calloc(sizeof(char), command_size);
docker_logs_command = calloc(sizeof(char), command_size);
docker_inspect_command = calloc(sizeof(char), command_size);
docker_rm_command = calloc(sizeof(char), command_size);
char *docker_command = parse_docker_command_file(command_file); char *docker_command = parse_docker_command_file(command_file);
char *docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg); char *docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg);
if (docker_binary == NULL) { if (docker_binary == NULL) {
@ -1509,7 +1519,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
goto cleanup; goto cleanup;
} }
snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command); snprintf(docker_command_with_binary, command_size, "%s %s", docker_binary, docker_command);
fprintf(LOGFILE, "Launching docker container...\n"); fprintf(LOGFILE, "Launching docker container...\n");
FILE* start_docker = popen(docker_command_with_binary, "r"); FILE* start_docker = popen(docker_command_with_binary, "r");
@ -1522,7 +1532,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
goto cleanup; goto cleanup;
} }
snprintf(docker_inspect_command, EXECUTOR_PATH_MAX, snprintf(docker_inspect_command, command_size,
"%s inspect --format {{.State.Pid}} %s", "%s inspect --format {{.State.Pid}} %s",
docker_binary, container_id); docker_binary, container_id);
@ -1567,7 +1577,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
goto cleanup; goto cleanup;
} }
snprintf(docker_wait_command, EXECUTOR_PATH_MAX, snprintf(docker_wait_command, command_size,
"%s wait %s", docker_binary, container_id); "%s wait %s", docker_binary, container_id);
fprintf(LOGFILE, "Waiting for docker container to finish...\n"); fprintf(LOGFILE, "Waiting for docker container to finish...\n");
@ -1581,7 +1591,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
if(exit_code != 0) { if(exit_code != 0) {
fprintf(ERRORFILE, "Docker container exit code was not zero: %d\n", fprintf(ERRORFILE, "Docker container exit code was not zero: %d\n",
exit_code); exit_code);
snprintf(docker_logs_command, EXECUTOR_PATH_MAX, "%s logs --tail=250 %s", snprintf(docker_logs_command, command_size, "%s logs --tail=250 %s",
docker_binary, container_id); docker_binary, container_id);
FILE* logs = popen(docker_logs_command, "r"); FILE* logs = popen(docker_logs_command, "r");
if(logs != NULL) { if(logs != NULL) {
@ -1611,7 +1621,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
} }
fprintf(LOGFILE, "Removing docker container post-exit...\n"); fprintf(LOGFILE, "Removing docker container post-exit...\n");
snprintf(docker_rm_command, EXECUTOR_PATH_MAX, snprintf(docker_rm_command, command_size,
"%s rm %s", docker_binary, container_id); "%s rm %s", docker_binary, container_id);
FILE* rm_docker = popen(docker_rm_command, "w"); FILE* rm_docker = popen(docker_rm_command, "w");
if (pclose (rm_docker) != 0) if (pclose (rm_docker) != 0)
@ -1643,6 +1653,11 @@ cleanup:
free(exit_code_file); free(exit_code_file);
free(script_file_dest); free(script_file_dest);
free(cred_file_dest); free(cred_file_dest);
free(docker_command_with_binary);
free(docker_wait_command);
free(docker_logs_command);
free(docker_inspect_command);
free(docker_rm_command);
return exit_code; return exit_code;
} }

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
View File

@ -98,6 +98,14 @@ enum operations {
#define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled" #define TC_SUPPORT_ENABLED_KEY "feature.tc.enabled"
#define TMP_DIR "tmp" #define TMP_DIR "tmp"
/* Macros for min/max. */
#ifndef MIN
#define MIN(a,b) (((a)<(b))?(a):(b))
#endif /* MIN */
#ifndef MAX
#define MAX(a,b) (((a)>(b))?(a):(b))
#endif /* MAX */
extern struct passwd *user_detail; extern struct passwd *user_detail;
// the log file for messages // the log file for messages