Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7999. Added file cache initializer for Linux container-executor. 

Contributed by Jason Lowe
This commit is contained in:
Eric Yang 2018-03-14 12:21:57 -04:00
parent a34d83d2fd
commit a82be7754d
2 changed files with 43 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
View File

@ -534,6 +534,18 @@ char *get_user_directory(const char *nm_root, const char *user) {
return concatenate(USER_DIR_PATTERN, "user_dir_path", 2, nm_root, user); return concatenate(USER_DIR_PATTERN, "user_dir_path", 2, nm_root, user);
} }
/**
* Get the user private filecache directory of a particular user
*/
char *get_user_filecache_directory(const char *nm_root, const char *user) {
int result = check_nm_local_dir(nm_uid, nm_root);
if (result != 0) {
return NULL;
}
return concatenate(USER_FILECACHE_DIR_PATTERN, "user_filecache_dir_path", 2,
nm_root, user);
}
/** /**
* Check node manager local dir permission. * Check node manager local dir permission.
*/ */
@ -1422,6 +1434,29 @@ int create_local_dirs(const char * user, const char *app_id,
return exit_code; return exit_code;
} }
// create the user file directory on all disks
int create_user_filecache_dirs(const char * user, char* const* local_dirs) {
int rc = 0;
const mode_t permissions = S_IRWXU | S_IXGRP;
for(char* const* ldir_p = local_dirs; *ldir_p != 0; ++ldir_p) {
char* filecache_dir = get_user_filecache_directory(*ldir_p, user);
if (filecache_dir == NULL) {
fprintf(LOGFILE, "Couldn't get user filecache directory for %s.\n", user);
rc = INITIALIZE_USER_FAILED;
break;
}
if (0 != mkdir(filecache_dir, permissions) && EEXIST != errno) {
fprintf(LOGFILE, "Failed to create directory %s - %s\n", filecache_dir,
strerror(errno));
free(filecache_dir);
rc = INITIALIZE_USER_FAILED;
break;
}
free(filecache_dir);
}
return rc;
}
int launch_docker_container_as_user(const char * user, const char *app_id, int launch_docker_container_as_user(const char * user, const char *app_id,
const char *container_id, const char *work_dir, const char *container_id, const char *work_dir,
const char *script_name, const char *cred_file, const char *script_name, const char *cred_file,
@ -1476,6 +1511,13 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
goto cleanup; goto cleanup;
} }
exit_code = create_user_filecache_dirs(user, local_dirs);
if (exit_code != 0) {
fprintf(ERRORFILE, "Could not create user filecache directory");
fflush(ERRORFILE);
goto cleanup;
}
docker_command = construct_docker_command(command_file); docker_command = construct_docker_command(command_file);
docker_binary = get_docker_binary(&CFG); docker_binary = get_docker_binary(&CFG);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
View File

@ -52,6 +52,7 @@ enum operations {
#define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group" #define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group"
#define USER_DIR_PATTERN "%s/usercache/%s" #define USER_DIR_PATTERN "%s/usercache/%s"
#define USER_FILECACHE_DIR_PATTERN "%s/usercache/%s/filecache"
#define NM_APP_DIR_PATTERN USER_DIR_PATTERN "/appcache/%s" #define NM_APP_DIR_PATTERN USER_DIR_PATTERN "/appcache/%s"
#define CONTAINER_DIR_PATTERN NM_APP_DIR_PATTERN "/%s" #define CONTAINER_DIR_PATTERN NM_APP_DIR_PATTERN "/%s"
#define CONTAINER_SCRIPT "launch_container.sh" #define CONTAINER_SCRIPT "launch_container.sh"