Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8538. Fixed memory leaks in container-executor and test cases. 

Contributed by Billie Rinaldi
This commit is contained in:
Eric Yang 2018-07-16 17:38:49 -04:00
parent 228508a0ee
commit d82edec3c0
11 changed files with 307 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
View File

@ -58,6 +58,7 @@ void free_section(struct section *section) {
section->name = NULL;
}
section->size = 0;
free(section);
}
//clean up method for freeing configuration
@ -466,6 +467,7 @@ static void merge_sections(struct section *section1, struct section *section2, c
section1->size += section2->size;
if (free_second_section) {
free(section2->name);
free(section2->kv_pairs);
memset(section2, 0, sizeof(*section2));
free(section2);
}
@ -708,6 +710,7 @@ char *get_config_path(const char *argv0) {
const char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
char *conf_file = resolve_config_path(orig_conf_file, executable_file);
free(executable_file);
if (conf_file == NULL) {
fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
View File

@ -133,6 +133,8 @@ if (ERRORFILE != NULL) {
fclose(ERRORFILE);
ERRORFILE = NULL;
}
free_executor_configurations();
}
/** Validates the current container-executor setup. Causes program exit

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
View File

@ -83,6 +83,8 @@ char* get_cgroups_path_to_write(
}
cleanup:
free((void *) cgroups_root);
free((void *) yarn_hierarchy_name);
if (failed) {
if (buffer) {
free(buffer);

226
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
View File

@ -143,9 +143,9 @@ int check_trusted_image(const struct configuration *command_config, const struct
fprintf(ERRORFILE, "image: %s is not trusted.\n", image_name);
ret = INVALID_DOCKER_IMAGE_TRUST;
}
free(image_name);
free_and_exit:
free(image_name);
free_values(privileged_registry);
return ret;
}
@ -195,7 +195,8 @@ static int add_param_to_command_if_allowed(const struct configuration *command_c
if (strcmp(key, "net") != 0) {
if (check_trusted_image(command_config, executor_cfg) != 0) {
fprintf(ERRORFILE, "Disable %s for untrusted image\n", key);
return INVALID_DOCKER_IMAGE_TRUST;
ret = INVALID_DOCKER_IMAGE_TRUST;
goto free_and_exit;
}
}
@ -225,13 +226,13 @@ static int add_param_to_command_if_allowed(const struct configuration *command_c
dst = strndup(values[i], tmp_ptr - values[i]);
pattern = strdup(permitted_values[j] + 6);
ret = execute_regex_match(pattern, dst);
free(dst);
free(pattern);
} else {
ret = strncmp(values[i], permitted_values[j], tmp_ptr - values[i]);
}
}
if (ret == 0) {
free(dst);
free(pattern);
permitted = 1;
break;
}
@ -379,6 +380,7 @@ int get_docker_command(const char *command_file, const struct configuration *con
ret = read_config(command_file, &command_config);
if (ret != 0) {
free_configuration(&command_config);
return INVALID_COMMAND_FILE;
}
@ -392,36 +394,41 @@ int get_docker_command(const char *command_file, const struct configuration *con
ret = add_to_args(args, docker);
free(docker);
if (ret != 0) {
free_configuration(&command_config);
return BUFFER_TOO_SMALL;
}
ret = add_docker_config_param(&command_config, args);
if (ret != 0) {
free_configuration(&command_config);
return BUFFER_TOO_SMALL;
}
char *command = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
free_configuration(&command_config);
if (strcmp(DOCKER_INSPECT_COMMAND, command) == 0) {
return get_docker_inspect_command(command_file, conf, args);
ret = get_docker_inspect_command(command_file, conf, args);
} else if (strcmp(DOCKER_KILL_COMMAND, command) == 0) {
return get_docker_kill_command(command_file, conf, args);
ret = get_docker_kill_command(command_file, conf, args);
} else if (strcmp(DOCKER_LOAD_COMMAND, command) == 0) {
return get_docker_load_command(command_file, conf, args);
ret = get_docker_load_command(command_file, conf, args);
} else if (strcmp(DOCKER_PULL_COMMAND, command) == 0) {
return get_docker_pull_command(command_file, conf, args);
ret = get_docker_pull_command(command_file, conf, args);
} else if (strcmp(DOCKER_RM_COMMAND, command) == 0) {
return get_docker_rm_command(command_file, conf, args);
ret = get_docker_rm_command(command_file, conf, args);
} else if (strcmp(DOCKER_RUN_COMMAND, command) == 0) {
return get_docker_run_command(command_file, conf, args);
ret = get_docker_run_command(command_file, conf, args);
} else if (strcmp(DOCKER_STOP_COMMAND, command) == 0) {
return get_docker_stop_command(command_file, conf, args);
ret = get_docker_stop_command(command_file, conf, args);
} else if (strcmp(DOCKER_VOLUME_COMMAND, command) == 0) {
return get_docker_volume_command(command_file, conf, args);
ret = get_docker_volume_command(command_file, conf, args);
} else if (strcmp(DOCKER_START_COMMAND, command) == 0) {
return get_docker_start_command(command_file, conf, args);
ret = get_docker_start_command(command_file, conf, args);
} else {
return UNKNOWN_DOCKER_COMMAND;
ret = UNKNOWN_DOCKER_COMMAND;
}
free(command);
return ret;
}
// check if a key is permitted in the configuration
@ -456,7 +463,7 @@ int get_docker_volume_command(const char *command_file, const struct configurati
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_VOLUME_COMMAND, &command_config);
if (ret != 0) {
return ret;
goto cleanup;
}
sub_command = get_configuration_value("sub-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
@ -533,6 +540,7 @@ int get_docker_volume_command(const char *command_file, const struct configurati
}
cleanup:
free_configuration(&command_config);
free(driver);
free(volume_name);
free(sub_command);
@ -548,18 +556,19 @@ int get_docker_inspect_command(const char *command_file, const struct configurat
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_INSPECT_COMMAND, &command_config);
if (ret != 0) {
return ret;
goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
return INVALID_DOCKER_CONTAINER_NAME;
ret = INVALID_DOCKER_CONTAINER_NAME;
goto free_and_exit;
}
format = get_configuration_value("format", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (format == NULL) {
free(container_name);
return INVALID_DOCKER_INSPECT_FORMAT;
ret = INVALID_DOCKER_INSPECT_FORMAT;
goto free_and_exit;
}
for (i = 0; i < 2; ++i) {
if (strcmp(format, valid_format_strings[i]) == 0) {
@ -569,9 +578,8 @@ int get_docker_inspect_command(const char *command_file, const struct configurat
}
if (valid_format != 1) {
fprintf(ERRORFILE, "Invalid format option '%s' not permitted\n", format);
free(container_name);
free(format);
return INVALID_DOCKER_INSPECT_FORMAT;
ret = INVALID_DOCKER_INSPECT_FORMAT;
goto free_and_exit;
}
ret = add_to_args(args, DOCKER_INSPECT_COMMAND);
@ -588,14 +596,12 @@ int get_docker_inspect_command(const char *command_file, const struct configurat
if (ret != 0) {
goto free_and_exit;
}
free(format);
free(container_name);
return 0;
free_and_exit:
free_configuration(&command_config);
free(format);
free(container_name);
return BUFFER_TOO_SMALL;
return ret;
}
int get_docker_load_command(const char *command_file, const struct configuration *conf, args *args) {
@ -604,12 +610,13 @@ int get_docker_load_command(const char *command_file, const struct configuration
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_LOAD_COMMAND, &command_config);
if (ret != 0) {
return ret;
goto free_and_exit;
}
image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (image_name == NULL) {
return INVALID_DOCKER_IMAGE_NAME;
ret = INVALID_DOCKER_IMAGE_NAME;
goto free_and_exit;
}
ret = add_to_args(args, DOCKER_LOAD_COMMAND);
@ -617,14 +624,14 @@ int get_docker_load_command(const char *command_file, const struct configuration
char *tmp_buffer = make_string("--i=%s", image_name);
ret = add_to_args(args, tmp_buffer);
free(tmp_buffer);
free(image_name);
if (ret != 0) {
return BUFFER_TOO_SMALL;
ret = BUFFER_TOO_SMALL;
}
return 0;
}
free_and_exit:
free(image_name);
return BUFFER_TOO_SMALL;
free_configuration(&command_config);
return ret;
}
static int validate_docker_image_name(const char *image_name) {
@ -638,26 +645,23 @@ int get_docker_pull_command(const char *command_file, const struct configuration
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_PULL_COMMAND, &command_config);
if (ret != 0) {
return ret;
goto free_pull;
}
image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (image_name == NULL || validate_docker_image_name(image_name) != 0) {
return INVALID_DOCKER_IMAGE_NAME;
ret = INVALID_DOCKER_IMAGE_NAME;
goto free_pull;
}
ret = add_to_args(args, DOCKER_PULL_COMMAND);
if (ret == 0) {
ret = add_to_args(args, image_name);
free(image_name);
if (ret != 0) {
goto free_pull;
}
return 0;
}
free_pull:
free(image_name);
return BUFFER_TOO_SMALL;
free_configuration(&command_config);
return ret;
}
int get_docker_rm_command(const char *command_file, const struct configuration *conf, args *args) {
@ -666,25 +670,26 @@ int get_docker_rm_command(const char *command_file, const struct configuration *
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_RM_COMMAND, &command_config);
if (ret != 0) {
return ret;
goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
return INVALID_DOCKER_CONTAINER_NAME;
ret = INVALID_DOCKER_CONTAINER_NAME;
goto free_and_exit;
}
ret = add_to_args(args, DOCKER_RM_COMMAND);
if (ret == 0) {
ret = add_to_args(args, container_name);
free(container_name);
if (ret != 0) {
return BUFFER_TOO_SMALL;
ret = BUFFER_TOO_SMALL;
}
return 0;
}
free_and_exit:
free(container_name);
return BUFFER_TOO_SMALL;
free_configuration(&command_config);
return ret;
}
int get_docker_stop_command(const char *command_file, const struct configuration *conf,
@ -696,12 +701,13 @@ int get_docker_stop_command(const char *command_file, const struct configuration
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_STOP_COMMAND, &command_config);
if (ret != 0) {
return ret;
goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
return INVALID_DOCKER_CONTAINER_NAME;
ret = INVALID_DOCKER_CONTAINER_NAME;
goto free_and_exit;
}
ret = add_to_args(args, DOCKER_STOP_COMMAND);
@ -726,7 +732,9 @@ int get_docker_stop_command(const char *command_file, const struct configuration
ret = add_to_args(args, container_name);
}
free_and_exit:
free(value);
free(container_name);
free_configuration(&command_config);
return ret;
}
@ -739,12 +747,13 @@ int get_docker_kill_command(const char *command_file, const struct configuration
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_KILL_COMMAND, &command_config);
if (ret != 0) {
return ret;
goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
return INVALID_DOCKER_CONTAINER_NAME;
ret = INVALID_DOCKER_CONTAINER_NAME;
goto free_and_exit;
}
ret = add_to_args(args, DOCKER_KILL_COMMAND);
@ -770,7 +779,9 @@ int get_docker_kill_command(const char *command_file, const struct configuration
ret = add_to_args(args, container_name);
}
free_and_exit:
free(value);
free(container_name);
free_configuration(&command_config);
return ret;
}
@ -780,12 +791,13 @@ int get_docker_start_command(const char *command_file, const struct configuratio
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_START_COMMAND, &command_config);
if (ret != 0) {
return ret;
goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
return INVALID_DOCKER_CONTAINER_NAME;
ret = INVALID_DOCKER_CONTAINER_NAME;
goto free_and_exit;
}
ret = add_to_args(args, DOCKER_START_COMMAND);
@ -795,6 +807,7 @@ int get_docker_start_command(const char *command_file, const struct configuratio
ret = add_to_args(args, container_name);
free_and_exit:
free(container_name);
free_configuration(&command_config);
return ret;
}
@ -1092,7 +1105,9 @@ static int add_mounts(const struct configuration *command_config, const struct c
char **permitted_rw_mounts = get_configuration_values_delimiter("docker.allowed.rw-mounts",
CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ",");
char **values = get_configuration_values_delimiter(key, DOCKER_COMMAND_FILE_SECTION, command_config, ",");
const char *container_executor_cfg_path = normalize_mount(get_config_path(""), 0);
char *config_path = get_config_path("");
const char *container_executor_cfg_path = normalize_mount(config_path, 0);
free(config_path);
int i = 0, permitted_rw = 0, permitted_ro = 0, ret = 0;
if (ro != 0) {
ro_suffix = ":ro";
@ -1172,6 +1187,8 @@ static int add_mounts(const struct configuration *command_config, const struct c
ret = BUFFER_TOO_SMALL;
goto free_and_exit;
}
free(mount_src);
mount_src = NULL;
}
}
@ -1325,42 +1342,41 @@ int get_docker_run_command(const char *command_file, const struct configuration
char *tmp_buffer = NULL;
char **launch_command = NULL;
char *privileged = NULL;
char *no_new_privileges_enabled = NULL;
struct configuration command_config = {0, NULL};
ret = read_and_verify_command_file(command_file, DOCKER_RUN_COMMAND, &command_config);
if (ret != 0) {
return ret;
goto free_and_exit;
}
container_name = get_configuration_value("name", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (container_name == NULL || validate_container_name(container_name) != 0) {
if (container_name != NULL) {
free(container_name);
}
return INVALID_DOCKER_CONTAINER_NAME;
ret = INVALID_DOCKER_CONTAINER_NAME;
goto free_and_exit;
}
user = get_configuration_value("user", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (user == NULL) {
return INVALID_DOCKER_USER_NAME;
ret = INVALID_DOCKER_USER_NAME;
goto free_and_exit;
}
image = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (image == NULL || validate_docker_image_name(image) != 0) {
if (image != NULL) {
free(image);
}
return INVALID_DOCKER_IMAGE_NAME;
ret = INVALID_DOCKER_IMAGE_NAME;
goto free_and_exit;
}
ret = add_to_args(args, DOCKER_RUN_COMMAND);
if(ret != 0) {
reset_args(args);
return BUFFER_TOO_SMALL;
ret = BUFFER_TOO_SMALL;
goto free_and_exit;
}
tmp_buffer = make_string("--name=%s", container_name);
ret = add_to_args(args, tmp_buffer);
free(tmp_buffer);
if (ret != 0) {
reset_args(args);
return BUFFER_TOO_SMALL;
ret = BUFFER_TOO_SMALL;
goto free_and_exit;
}
privileged = get_configuration_value("privileged", DOCKER_COMMAND_FILE_SECTION, &command_config);
@ -1370,111 +1386,95 @@ int get_docker_run_command(const char *command_file, const struct configuration
ret = add_to_args(args, user_buffer);
free(user_buffer);
if (ret != 0) {
reset_args(args);
return BUFFER_TOO_SMALL;
ret = BUFFER_TOO_SMALL;
goto free_and_exit;
}
char *no_new_privileges_enabled =
no_new_privileges_enabled =
get_configuration_value("docker.no-new-privileges.enabled",
CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
if (no_new_privileges_enabled != NULL &&
strcasecmp(no_new_privileges_enabled, "True") == 0) {
ret = add_to_args(args, "--security-opt=no-new-privileges");
if (ret != 0) {
reset_args(args);
return BUFFER_TOO_SMALL;
ret = BUFFER_TOO_SMALL;
goto free_and_exit;
}
}
free(no_new_privileges_enabled);
}
free(privileged);
ret = detach_container(&command_config, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = rm_container_on_exit(&command_config, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = set_container_workdir(&command_config, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = set_network(&command_config, conf, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = set_pid_namespace(&command_config, conf, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = add_ro_mounts(&command_config, conf, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = add_rw_mounts(&command_config, conf, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = set_cgroup_parent(&command_config, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = set_privileged(&command_config, conf, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = set_capabilities(&command_config, conf, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = set_hostname(&command_config, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = set_group_add(&command_config, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = set_devices(&command_config, conf, args);
if (ret != 0) {
reset_args(args);
return ret;
goto free_and_exit;
}
ret = set_env(&command_config, args);
if (ret != 0) {
return BUFFER_TOO_SMALL;
goto free_and_exit;
}
ret = add_to_args(args, image);
if (ret != 0) {
reset_args(args);
return BUFFER_TOO_SMALL;
goto free_and_exit;
}
launch_command = get_configuration_values_delimiter("launch-command", DOCKER_COMMAND_FILE_SECTION, &command_config,
@ -1483,13 +1483,21 @@ int get_docker_run_command(const char *command_file, const struct configuration
for (i = 0; launch_command[i] != NULL; ++i) {
ret = add_to_args(args, launch_command[i]);
if (ret != 0) {
free_values(launch_command);
ret = BUFFER_TOO_SMALL;
goto free_and_exit;
}
}
}
free_and_exit:
if (ret != 0) {
reset_args(args);
return BUFFER_TOO_SMALL;
}
}
free(user);
free(image);
free(privileged);
free(no_new_privileges_enabled);
free(container_name);
free_values(launch_command);
}
free(tmp_buffer);
return 0;
free_configuration(&command_config);
return ret;
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
View File

@ -73,6 +73,8 @@ TEST_F(TestCGroupsModule, test_cgroups_get_path_without_define_root) {
char* path = get_cgroups_path_to_write("devices", "deny", "container_1");
ASSERT_TRUE(NULL == path) << "Should fail.\n";
free_executor_configurations();
}
TEST_F(TestCGroupsModule, test_cgroups_get_path_without_define_yarn_hierarchy) {
@ -92,6 +94,8 @@ TEST_F(TestCGroupsModule, test_cgroups_get_path_without_define_yarn_hierarchy) {
char* path = get_cgroups_path_to_write("devices", "deny", "container_1");
ASSERT_TRUE(NULL == path) << "Should fail.\n";
free_executor_configurations();
}
TEST_F(TestCGroupsModule, test_cgroups_get_path_succeeded) {
@ -117,5 +121,9 @@ TEST_F(TestCGroupsModule, test_cgroups_get_path_succeeded) {
ASSERT_STREQ(EXPECTED, path)
<< "Return cgroup-path-to-write is not expected\n";
free(path);
free_executor_configurations();
}
} // namespace ContainerExecutor

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/fpga/test-fpga-module.cc
View File

@ -84,6 +84,13 @@ static int mock_update_cgroups_parameters(
return 0;
}
static void clear_cgroups_parameters_invoked() {
for (std::vector<const char*>::size_type i = 0; i < cgroups_parameters_invoked.size(); i++) {
free((void *) cgroups_parameters_invoked[i]);
}
cgroups_parameters_invoked.clear();
}
static void verify_param_updated_to_cgroups(
int argc, const char** argv) {
ASSERT_EQ(argc, cgroups_parameters_invoked.size());
@ -133,6 +140,9 @@ static void test_fpga_module_enabled_disabled(int enabled) {
EXPECTED_RC = -1;
}
ASSERT_EQ(EXPECTED_RC, rc);
clear_cgroups_parameters_invoked();
free_executor_configurations();
}
TEST_F(TestFpgaModule, test_verify_fpga_module_calls_cgroup_parameter) {
@ -146,7 +156,7 @@ TEST_F(TestFpgaModule, test_verify_fpga_module_calls_cgroup_parameter) {
container_id };
/* Test case 1: block 2 devices */
cgroups_parameters_invoked.clear();
clear_cgroups_parameters_invoked();
int rc = handle_fpga_request(&mock_update_cgroups_parameters,
"fpga", 5, argv);
ASSERT_EQ(0, rc) << "Should success.\n";
@ -157,7 +167,7 @@ TEST_F(TestFpgaModule, test_verify_fpga_module_calls_cgroup_parameter) {
verify_param_updated_to_cgroups(8, expected_cgroups_argv);
/* Test case 2: block 0 devices */
cgroups_parameters_invoked.clear();
clear_cgroups_parameters_invoked();
char* argv_1[] = { (char*) "--module-fpga", (char*) "--container_id", container_id };
rc = handle_fpga_request(&mock_update_cgroups_parameters,
"fpga", 3, argv_1);
@ -167,7 +177,7 @@ TEST_F(TestFpgaModule, test_verify_fpga_module_calls_cgroup_parameter) {
verify_param_updated_to_cgroups(0, NULL);
/* Test case 3: block 2 non-sequential devices */
cgroups_parameters_invoked.clear();
clear_cgroups_parameters_invoked();
char* argv_2[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "1,3",
(char*) "--container_id", container_id };
rc = handle_fpga_request(&mock_update_cgroups_parameters,
@ -178,6 +188,9 @@ TEST_F(TestFpgaModule, test_verify_fpga_module_calls_cgroup_parameter) {
const char* expected_cgroups_argv_2[] = { "devices", "deny", container_id, "c 246:1 rwm",
"devices", "deny", container_id, "c 246:3 rwm"};
verify_param_updated_to_cgroups(8, expected_cgroups_argv_2);
clear_cgroups_parameters_invoked();
free_executor_configurations();
}
TEST_F(TestFpgaModule, test_illegal_cli_parameters) {
@ -193,6 +206,7 @@ TEST_F(TestFpgaModule, test_illegal_cli_parameters) {
ASSERT_NE(0, rc) << "Should fail.\n";
// Illegal container id - 2
clear_cgroups_parameters_invoked();
char* argv_1[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "0,1",
(char*) "--container_id", (char*) "container_1" };
rc = handle_fpga_request(&mock_update_cgroups_parameters,
@ -200,10 +214,14 @@ TEST_F(TestFpgaModule, test_illegal_cli_parameters) {
ASSERT_NE(0, rc) << "Should fail.\n";
// Illegal container id - 3
clear_cgroups_parameters_invoked();
char* argv_2[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "0,1" };
rc = handle_fpga_request(&mock_update_cgroups_parameters,
"fpga", 3, argv_2);
ASSERT_NE(0, rc) << "Should fail.\n";
clear_cgroups_parameters_invoked();
free_executor_configurations();
}
TEST_F(TestFpgaModule, test_fpga_module_disabled) {

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
View File

@ -84,6 +84,13 @@ static int mock_update_cgroups_parameters(
return 0;
}
static void clear_cgroups_parameters_invoked() {
for (std::vector<const char*>::size_type i = 0; i < cgroups_parameters_invoked.size(); i++) {
free((void *) cgroups_parameters_invoked[i]);
}
cgroups_parameters_invoked.clear();
}
static void verify_param_updated_to_cgroups(
int argc, const char** argv) {
ASSERT_EQ(argc, cgroups_parameters_invoked.size());
@ -133,6 +140,9 @@ static void test_gpu_module_enabled_disabled(int enabled) {
EXPECTED_RC = -1;
}
ASSERT_EQ(EXPECTED_RC, rc);
clear_cgroups_parameters_invoked();
free_executor_configurations();
}
TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
@ -146,7 +156,7 @@ TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
container_id };
/* Test case 1: block 2 devices */
cgroups_parameters_invoked.clear();
clear_cgroups_parameters_invoked();
int rc = handle_gpu_request(&mock_update_cgroups_parameters,
"gpu", 5, argv);
ASSERT_EQ(0, rc) << "Should success.\n";
@ -157,7 +167,7 @@ TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
verify_param_updated_to_cgroups(8, expected_cgroups_argv);
/* Test case 2: block 0 devices */
cgroups_parameters_invoked.clear();
clear_cgroups_parameters_invoked();
char* argv_1[] = { (char*) "--module-gpu", (char*) "--container_id", container_id };
rc = handle_gpu_request(&mock_update_cgroups_parameters,
"gpu", 3, argv_1);
@ -167,7 +177,7 @@ TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
verify_param_updated_to_cgroups(0, NULL);
/* Test case 3: block 2 non-sequential devices */
cgroups_parameters_invoked.clear();
clear_cgroups_parameters_invoked();
char* argv_2[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "1,3",
(char*) "--container_id", container_id };
rc = handle_gpu_request(&mock_update_cgroups_parameters,
@ -178,6 +188,9 @@ TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
const char* expected_cgroups_argv_2[] = { "devices", "deny", container_id, "c 195:1 rwm",
"devices", "deny", container_id, "c 195:3 rwm"};
verify_param_updated_to_cgroups(8, expected_cgroups_argv_2);
clear_cgroups_parameters_invoked();
free_executor_configurations();
}
TEST_F(TestGpuModule, test_illegal_cli_parameters) {
@ -193,6 +206,7 @@ TEST_F(TestGpuModule, test_illegal_cli_parameters) {
ASSERT_NE(0, rc) << "Should fail.\n";
// Illegal container id - 2
clear_cgroups_parameters_invoked();
char* argv_1[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "0,1",
(char*) "--container_id", (char*) "container_1" };
rc = handle_gpu_request(&mock_update_cgroups_parameters,
@ -200,10 +214,14 @@ TEST_F(TestGpuModule, test_illegal_cli_parameters) {
ASSERT_NE(0, rc) << "Should fail.\n";
// Illegal container id - 3
clear_cgroups_parameters_invoked();
char* argv_2[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "0,1" };
rc = handle_gpu_request(&mock_update_cgroups_parameters,
"gpu", 3, argv_2);
ASSERT_NE(0, rc) << "Should fail.\n";
clear_cgroups_parameters_invoked();
free_executor_configurations();
}
TEST_F(TestGpuModule, test_gpu_module_disabled) {

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_configuration.cc
View File

@ -51,6 +51,7 @@ namespace ContainerExecutor {
virtual void TearDown() {
free_configuration(&new_config_format);
free_configuration(&old_config_format);
free_configuration(&mixed_config_format);
return;
}
@ -84,12 +85,12 @@ namespace ContainerExecutor {
ASSERT_STREQ("/var/run/yarn", split_values[0]);
ASSERT_STREQ("/tmp/mydir", split_values[1]);
ASSERT_EQ(NULL, split_values[2]);
free(split_values);
free_values(split_values);
split_values = get_configuration_values_delimiter("allowed.system.users",
"", &old_config_format, "%");
ASSERT_STREQ("nobody,daemon", split_values[0]);
ASSERT_EQ(NULL, split_values[1]);
free(split_values);
free_values(split_values);
}
TEST_F(TestConfiguration, test_get_configuration_values) {
@ -105,13 +106,13 @@ namespace ContainerExecutor {
split_values = get_configuration_values("yarn.local.dirs", "", &old_config_format);
ASSERT_STREQ("/var/run/yarn%/tmp/mydir", split_values[0]);
ASSERT_EQ(NULL, split_values[1]);
free(split_values);
free_values(split_values);
split_values = get_configuration_values("allowed.system.users", "",
&old_config_format);
ASSERT_STREQ("nobody", split_values[0]);
ASSERT_STREQ("daemon", split_values[1]);
ASSERT_EQ(NULL, split_values[2]);
free(split_values);
free_values(split_values);
}
TEST_F(TestConfiguration, test_get_configuration_value) {
@ -149,21 +150,28 @@ namespace ContainerExecutor {
char *value = NULL;
value = get_section_value("yarn.nodemanager.linux-container-executor.group", executor_cfg);
ASSERT_STREQ("yarn", value);
free(value);
value = get_section_value("feature.docker.enabled", executor_cfg);
ASSERT_STREQ("1", value);
free(value);
value = get_section_value("feature.tc.enabled", executor_cfg);
ASSERT_STREQ("0", value);
free(value);
value = get_section_value("min.user.id", executor_cfg);
ASSERT_STREQ("1000", value);
free(value);
value = get_section_value("docker.binary", executor_cfg);
ASSERT_STREQ("/usr/bin/docker", value);
free(value);
char **list = get_section_values("allowed.system.users", executor_cfg);
ASSERT_STREQ("nobody", list[0]);
ASSERT_STREQ("daemon", list[1]);
free_values(list);
list = get_section_values("banned.users", executor_cfg);
ASSERT_STREQ("root", list[0]);
ASSERT_STREQ("testuser1", list[1]);
ASSERT_STREQ("testuser2", list[2]);
free_values(list);
}
TEST_F(TestConfiguration, test_get_section_values_delimiter) {
@ -176,12 +184,16 @@ namespace ContainerExecutor {
free(value);
value = get_section_value("key2", section);
ASSERT_EQ(NULL, value);
free(value);
split_values = get_section_values_delimiter(NULL, section, "%");
ASSERT_EQ(NULL, split_values);
free_values(split_values);
split_values = get_section_values_delimiter("split-key", NULL, "%");
ASSERT_EQ(NULL, split_values);
free_values(split_values);
split_values = get_section_values_delimiter("split-key", section, NULL);
ASSERT_EQ(NULL, split_values);
free_values(split_values);
split_values = get_section_values_delimiter("split-key", section, "%");
ASSERT_FALSE(split_values == NULL);
ASSERT_STREQ("val1,val2,val3", split_values[0]);
@ -192,6 +204,7 @@ namespace ContainerExecutor {
ASSERT_STREQ("perc-val1", split_values[0]);
ASSERT_STREQ("perc-val2", split_values[1]);
ASSERT_TRUE(split_values[2] == NULL);
free_values(split_values);
}
TEST_F(TestConfiguration, test_get_section_values) {
@ -201,13 +214,16 @@ namespace ContainerExecutor {
section = get_configuration_section("section-1", &new_config_format);
value = get_section_value(NULL, section);
ASSERT_EQ(NULL, value);
free(value);
value = get_section_value("key1", NULL);
ASSERT_EQ(NULL, value);
free(value);
value = get_section_value("key1", section);
ASSERT_STREQ("value1", value);
free(value);
value = get_section_value("key2", section);
ASSERT_EQ(NULL, value);
free(value);
split_values = get_section_values("split-key", section);
ASSERT_FALSE(split_values == NULL);
ASSERT_STREQ("val1", split_values[0]);
@ -235,14 +251,16 @@ namespace ContainerExecutor {
section = get_configuration_section("split-section", &new_config_format);
value = get_section_value(NULL, section);
ASSERT_EQ(NULL, value);
free(value);
value = get_section_value("key3", NULL);
ASSERT_EQ(NULL, value);
free(value);
value = get_section_value("key3", section);
ASSERT_STREQ("value3", value);
free(value);
value = get_section_value("key4", section);
ASSERT_STREQ("value4", value);
free(value);
}
TEST_F(TestConfiguration, test_get_configuration_section) {
@ -343,6 +361,7 @@ namespace ContainerExecutor {
oss.str("");
oss << "value" << i;
ASSERT_STREQ(oss.str().c_str(), value);
free((void *) value);
}
remove(sample_file_name.c_str());
free_configuration(&cfg);
@ -372,6 +391,7 @@ namespace ContainerExecutor {
oss.str("");
oss << "value" << i;
ASSERT_STREQ(oss.str().c_str(), value);
free((void *) value);
}
remove(sample_file_name.c_str());
free_configuration(&cfg);
@ -415,18 +435,22 @@ namespace ContainerExecutor {
char *value = NULL;
value = get_section_value("key1", executor_cfg);
ASSERT_STREQ("value1", value);
free(value);
value = get_section_value("key2", executor_cfg);
ASSERT_STREQ("value2", value);
ASSERT_EQ(2, executor_cfg->size);
free(value);
executor_cfg = get_configuration_section("section-1",
&mixed_config_format);
value = get_section_value("key3", executor_cfg);
ASSERT_STREQ("value3", value);
free(value);
value = get_section_value("key1", executor_cfg);
ASSERT_STREQ("value4", value);
ASSERT_EQ(2, executor_cfg->size);
ASSERT_EQ(2, mixed_config_format.size);
ASSERT_STREQ("", mixed_config_format.sections[0]->name);
ASSERT_STREQ("section-1", mixed_config_format.sections[1]->name);
free(value);
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc
View File

@ -51,6 +51,7 @@ namespace ContainerExecutor {
ASSERT_STREQ(oss.str().c_str(), splits[i-1]);
}
ASSERT_EQ(NULL, splits[count]);
free(split_string);
free_values(splits);
split_string = (char *) calloc(str.length() + 1, sizeof(char));
@ -59,6 +60,7 @@ namespace ContainerExecutor {
ASSERT_TRUE(splits != NULL);
ASSERT_TRUE(splits[1] == NULL);
ASSERT_STREQ(str.c_str(), splits[0]);
free(split_string);
free_values(splits);
splits = split_delimiter(NULL, ",");
@ -82,6 +84,7 @@ namespace ContainerExecutor {
ASSERT_STREQ(oss.str().c_str(), splits[i-1]);
}
ASSERT_EQ(NULL, splits[count]);
free(split_string);
free_values(splits);
str = "1,2,3,4,5,6,7,8,9,10,11";
@ -91,6 +94,8 @@ namespace ContainerExecutor {
ASSERT_TRUE(splits != NULL);
ASSERT_TRUE(splits[1] == NULL);
ASSERT_STREQ(str.c_str(), splits[0]);
free(split_string);
free_values(splits);
return;
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
View File

@ -59,6 +59,7 @@
ASSERT_EQ(1, numbers[0]);
ASSERT_EQ(-1, numbers[3]);
ASSERT_EQ(0, numbers[5]);
free(numbers);
input = "3";
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
@ -66,28 +67,33 @@
ASSERT_EQ(0, rc) << "Should succeeded\n";
ASSERT_EQ(1, n_numbers);
ASSERT_EQ(3, numbers[0]);
free(numbers);
input = "";
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_EQ(0, rc) << "Should succeeded\n";
ASSERT_EQ(0, n_numbers);
free(numbers);
input = ",,";
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_EQ(0, rc) << "Should succeeded\n";
ASSERT_EQ(0, n_numbers);
free(numbers);
input = "1,2,aa,bb";
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_TRUE(0 != rc) << "Should failed\n";
free(numbers);
input = "1,2,3,-12312312312312312312321311231231231";
rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
std::cout << "Testing input=" << input << "\n";
ASSERT_TRUE(0 != rc) << "Should failed\n";
free(numbers);
}
TEST_F(TestStringUtils, test_validate_container_id) {

128
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
View File

@ -119,23 +119,22 @@ namespace ContainerExecutor {
struct args tmp = ARGS_INITIAL_VALUE;
std::vector<std::pair<std::string, std::string> >::const_iterator itr;
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
reset_args(&tmp);
write_command_file(itr->first);
int ret = (*docker_func)(docker_command_file.c_str(), &container_executor_cfg, &tmp);
ASSERT_EQ(0, ret) << "error message: " << get_docker_error_message(ret) << " for input " << itr->first;
char *actual = flatten(&tmp);
ASSERT_STREQ(itr->second.c_str(), actual);
reset_args(&tmp);
free(actual);
}
std::vector<std::pair<std::string, int> >::const_iterator itr2;
for (itr2 = bad_file_cmd_vec.begin(); itr2 != bad_file_cmd_vec.end(); ++itr2) {
reset_args(&tmp);
write_command_file(itr2->first);
int ret = (*docker_func)(docker_command_file.c_str(), &container_executor_cfg, &tmp);
ASSERT_EQ(itr2->second, ret) << " for " << itr2->first << std::endl;
}
reset_args(&tmp);
}
int ret = (*docker_func)("unknown-file", &container_executor_cfg, &tmp);
ASSERT_EQ(static_cast<int>(INVALID_COMMAND_FILE), ret);
reset_args(&tmp);
@ -147,7 +146,6 @@ namespace ContainerExecutor {
for(itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
struct configuration cfg;
struct args buff = ARGS_INITIAL_VALUE;
reset_args(&buff);
write_command_file(itr->first);
int ret = read_config(docker_command_file.c_str(), &cfg);
if(ret == 0) {
@ -155,7 +153,9 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
reset_args(&buff);
free(actual);
free_configuration(&cfg);
}
}
}
@ -445,7 +445,6 @@ namespace ContainerExecutor {
TEST_F(TestDockerUtil, test_set_network) {
struct configuration container_cfg;
struct args buff = ARGS_INITIAL_VALUE;
reset_args(&buff);
int ret = 0;
std::string container_executor_cfg_contents = "[docker]\n docker.allowed.networks=sdn1,bridge";
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
@ -464,7 +463,6 @@ namespace ContainerExecutor {
}
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
struct configuration cmd_cfg;
reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@ -474,7 +472,9 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
reset_args(&buff);
free(actual);
free_configuration(&cmd_cfg);
}
struct configuration cmd_cfg_1;
write_command_file("[docker-command-execution]\n docker-command=run\n net=sdn2");
@ -482,10 +482,11 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_network(&cmd_cfg_1, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_NETWORK, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&container_cfg);
container_executor_cfg_contents = "[docker]\n";
write_container_executor_cfg(container_executor_cfg_contents);
@ -493,10 +494,12 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_network(&cmd_cfg_1, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_NETWORK, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg_1);
free_configuration(&container_cfg);
}
TEST_F(TestDockerUtil, test_set_pid_namespace) {
@ -529,7 +532,6 @@ namespace ContainerExecutor {
FAIL();
}
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@ -539,10 +541,11 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
reset_args(&buff);
free(actual);
free_configuration(&cmd_cfg);
}
for (itr2 = bad_file_cmd_vec.begin(); itr2 != bad_file_cmd_vec.end(); ++itr2) {
reset_args(&buff);
write_command_file(itr2->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@ -551,7 +554,10 @@ namespace ContainerExecutor {
ret = set_pid_namespace(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
}
free_configuration(&container_cfg);
}
// check default case and when it's turned off
@ -575,6 +581,7 @@ namespace ContainerExecutor {
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
free(actual);
free_configuration(&cmd_cfg);
}
bad_file_cmd_vec.clear();
bad_file_cmd_vec.push_back(std::make_pair<std::string, int>(
@ -584,7 +591,6 @@ namespace ContainerExecutor {
"[docker-command-execution]\n docker-command=run\n pid=host",
static_cast<int>(PID_HOST_DISABLED)));
for (itr2 = bad_file_cmd_vec.begin(); itr2 != bad_file_cmd_vec.end(); ++itr2) {
reset_args(&buff);
write_command_file(itr2->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@ -593,7 +599,10 @@ namespace ContainerExecutor {
ret = set_pid_namespace(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
}
free_configuration(&container_cfg);
}
}
@ -633,6 +642,7 @@ namespace ContainerExecutor {
for (int i = 0; i < entries; ++i) {
ASSERT_STREQ(expected[i], ptr[i]);
}
free_values(ptr);
}
TEST_F(TestDockerUtil, test_set_privileged) {
@ -665,7 +675,6 @@ namespace ContainerExecutor {
FAIL();
}
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@ -674,19 +683,22 @@ namespace ContainerExecutor {
ret = set_privileged(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(6, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
}
write_command_file("[docker-command-execution]\n docker-command=run\n user=nobody\n privileged=true\n image=nothadoop/image");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_privileged(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(PRIVILEGED_CONTAINERS_DISABLED, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
free_configuration(&container_cfg);
}
// check default case and when it's turned off
for (int i = 3; i < 6; ++i) {
write_container_executor_cfg(container_executor_cfg_contents[i]);
@ -698,7 +710,6 @@ namespace ContainerExecutor {
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n user=root\n privileged=false", ""));
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@ -708,7 +719,9 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
reset_args(&buff);
free(actual);
free_configuration(&cmd_cfg);
}
write_command_file("[docker-command-execution]\n docker-command=run\n user=root\n privileged=true");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
@ -718,6 +731,9 @@ namespace ContainerExecutor {
ret = set_privileged(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(PRIVILEGED_CONTAINERS_DISABLED, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
free_configuration(&container_cfg);
}
}
@ -752,7 +768,6 @@ namespace ContainerExecutor {
FAIL();
}
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@ -762,16 +777,19 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
reset_args(&buff);
free(actual);
free_configuration(&cmd_cfg);
}
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n cap-add=SETGID");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_capabilities(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
reset_args(&buff);
free_configuration(&container_cfg);
container_executor_cfg_contents = "[docker]\n docker.trusted.registries=hadoop\n";
write_container_executor_cfg(container_executor_cfg_contents);
@ -779,15 +797,16 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_capabilities(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret);
reset_args(&buff);
free_configuration(&cmd_cfg);
free_configuration(&container_cfg);
}
TEST_F(TestDockerUtil, test_set_devices) {
struct configuration container_cfg, cmd_cfg;
struct args buff = ARGS_INITIAL_VALUE;
reset_args(&buff);
int ret = 0;
std::string container_executor_cfg_contents = "[docker]\n"
" docker.trusted.registries=hadoop\n"
@ -821,7 +840,6 @@ namespace ContainerExecutor {
FAIL();
}
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@ -831,67 +849,75 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
reset_args(&buff);
free(actual);
free_configuration(&cmd_cfg);
}
write_command_file("[docker-command-execution]\n docker-command=run\n image=nothadoop/image\n devices=/dev/test-device:/dev/test-device");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device3:/dev/device3");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device1");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/testnvidia:/dev/testnvidia");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/gpu-nvidia-uvm:/dev/gpu-nvidia-uvm");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device1");
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&container_cfg);
container_executor_cfg_contents = "[docker]\n";
write_container_executor_cfg(container_executor_cfg_contents);
@ -899,10 +925,12 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = set_devices(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_DEVICE, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
free_configuration(&container_cfg);
}
@ -951,7 +979,6 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, std::string> >::const_iterator itr;
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@ -961,7 +988,9 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
reset_args(&buff);
free(actual);
free_configuration(&cmd_cfg);
}
std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
@ -978,18 +1007,18 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> >::const_iterator itr2;
for (itr2 = bad_file_cmds_vec.begin(); itr2 != bad_file_cmds_vec.end(); ++itr2) {
reset_args(&buff);
write_command_file(itr2->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_STREQ("", actual);
reset_args(&buff);
free(actual);
free_configuration(&cmd_cfg);
}
// verify that you can't mount any directory in the container-executor.cfg path
@ -997,17 +1026,17 @@ namespace ContainerExecutor {
while (strlen(ce_path) != 0) {
std::string cmd_file_contents = "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=";
cmd_file_contents.append(ce_path).append(":").append("/etc/hadoop");
reset_args(&buff);
write_command_file(cmd_file_contents);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_RW_MOUNT, ret) << " for input " << cmd_file_contents;
char *actual = flatten(&buff);
ASSERT_STREQ("", actual);
reset_args(&buff);
free_configuration(&cmd_cfg);
free(actual);
char *tmp = strrchr(ce_path, '/');
if (tmp != NULL) {
@ -1015,6 +1044,7 @@ namespace ContainerExecutor {
}
}
free(ce_path);
free_configuration(&container_cfg);
// For untrusted image, container add_rw_mounts will pass through
// without mounting or report error code.
@ -1024,12 +1054,13 @@ namespace ContainerExecutor {
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = add_rw_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ("", actual);
reset_args(&buff);
free(actual);
free_configuration(&container_cfg);
}
TEST_F(TestDockerUtil, test_add_ro_mounts) {
@ -1080,7 +1111,6 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, std::string> >::const_iterator itr;
for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) {
reset_args(&buff);
write_command_file(itr->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
@ -1090,7 +1120,9 @@ namespace ContainerExecutor {
char *actual = flatten(&buff);
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), actual);
reset_args(&buff);
free(actual);
free_configuration(&cmd_cfg);
}
std::vector<std::pair<std::string, int> > bad_file_cmds_vec;
@ -1104,19 +1136,20 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> >::const_iterator itr2;
for (itr2 = bad_file_cmds_vec.begin(); itr2 != bad_file_cmds_vec.end(); ++itr2) {
reset_args(&buff);
write_command_file(itr2->first);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
reset_args(&buff);
ret = add_ro_mounts(&cmd_cfg, &container_cfg, &buff);
char *actual = flatten(&buff);
ASSERT_EQ(itr2->second, ret);
ASSERT_STREQ("", actual);
reset_args(&buff);
free(actual);
free_configuration(&cmd_cfg);
}
free_configuration(&container_cfg);
container_executor_cfg_contents = "[docker]\n docker.trusted.registries=hadoop\n";
write_container_executor_cfg(container_executor_cfg_contents);
@ -1125,10 +1158,16 @@ namespace ContainerExecutor {
FAIL();
}
write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home:/home");
reset_args(&buff);
ret = read_config(docker_command_file.c_str(), &cmd_cfg);
if (ret != 0) {
FAIL();
}
ret = add_ro_mounts(&cmd_cfg, &container_cfg, &buff);
ASSERT_EQ(INVALID_DOCKER_RO_MOUNT, ret);
ASSERT_EQ(0, buff.length);
reset_args(&buff);
free_configuration(&cmd_cfg);
free_configuration(&container_cfg);
}
TEST_F(TestDockerUtil, test_docker_run_privileged) {
@ -1310,6 +1349,7 @@ namespace ContainerExecutor {
static_cast<int>(INVALID_DOCKER_NETWORK)));
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
free_configuration(&container_executor_cfg);
}
TEST_F(TestDockerUtil, test_docker_run_entry_point) {
@ -1352,6 +1392,7 @@ namespace ContainerExecutor {
static_cast<int>(INVALID_DOCKER_CONTAINER_NAME)));
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
free_configuration(&container_executor_cfg);
}
TEST_F(TestDockerUtil, test_docker_run_no_privileged) {
@ -1441,6 +1482,7 @@ namespace ContainerExecutor {
static_cast<int>(PRIVILEGED_CONTAINERS_DISABLED)));
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
free_configuration(&container_executor_cfg);
}
}
@ -1471,13 +1513,14 @@ namespace ContainerExecutor {
struct args buffer = ARGS_INITIAL_VALUE;
struct configuration cfg = {0, NULL};
for (itr = input_output_map.begin(); itr != input_output_map.end(); ++itr) {
reset_args(&buffer);
write_command_file(itr->first);
int ret = get_docker_command(docker_command_file.c_str(), &cfg, &buffer);
char *actual = flatten(&buffer);
ASSERT_EQ(0, ret) << "for input " << itr->first;
ASSERT_STREQ(itr->second.c_str(), flatten(&buffer));
}
ASSERT_STREQ(itr->second.c_str(), actual);
reset_args(&buffer);
free(actual);
}
}
TEST_F(TestDockerUtil, test_docker_module_enabled) {
@ -1497,6 +1540,7 @@ namespace ContainerExecutor {
ret = docker_module_enabled(&container_executor_cfg);
ASSERT_EQ(input_out_vec[i].second, ret) << " incorrect output for "
<< input_out_vec[i].first;
free_configuration(&container_executor_cfg);
}
}
@ -1544,6 +1588,7 @@ namespace ContainerExecutor {
static_cast<int>(INVALID_DOCKER_VOLUME_DRIVER)));
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_volume_command);
free_configuration(&container_executor_cfg);
}
TEST_F(TestDockerUtil, test_docker_no_new_privileges) {
@ -1589,6 +1634,7 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
free_configuration(&container_executor_cfg);
}
for (int i = 2; i < 3; ++i) {
@ -1611,6 +1657,7 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
free_configuration(&container_executor_cfg);
}
for (int i = 3; i < 5; ++i) {
@ -1633,6 +1680,7 @@ namespace ContainerExecutor {
std::vector<std::pair<std::string, int> > bad_file_cmd_vec;
run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command);
free_configuration(&container_executor_cfg);
}
}
}