Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7717. Add configuration consistency for module.enabled and docker.privileged-containers.enabled. Contributed by Eric Badger.

This commit is contained in:
Miklos Szegedi 2018-01-17 14:11:14 -08:00
parent 6e42d05829
commit a68e445dc6
6 changed files with 52 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
View File

@ -2,7 +2,7 @@ yarn.nodemanager.linux-container-executor.group=#configured value of yarn.nodema
banned.users=#comma separated list of users who can not run applications
min.user.id=1000#Prevent other super-users
allowed.system.users=##comma separated list of system users who CAN run applications
feature.tc.enabled=0
feature.tc.enabled=false
# The configs below deal with settings for Docker
#[docker]
@ -13,7 +13,7 @@ feature.tc.enabled=0
# docker.allowed.networks=## comma seperated networks that can be used. e.g bridge,host,none
# docker.allowed.ro-mounts=## comma seperated volumes that can be mounted as read-only
# docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs
# docker.privileged-containers.enabled=0
# docker.privileged-containers.enabled=false
# docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers
# The configs below deal with settings for FPGA resource

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
View File

@ -39,6 +39,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <limits.h>
#include <sys/stat.h>
#include <sys/mount.h>
@ -441,6 +442,11 @@ int is_feature_enabled(const char* feature_key, int default_value,
int enabled = default_value;
if (enabled_str != NULL) {
if (strcasecmp(enabled_str, "true") == 0) {
enabled = 1;
} else if (strcasecmp(enabled_str, "false") == 0) {
enabled = 0;
} else {
char *end_ptr = NULL;
enabled = strtol(enabled_str, &end_ptr, 10);
@ -454,6 +460,7 @@ int is_feature_enabled(const char* feature_key, int default_value,
return default_value;
}
}
free(enabled_str);
return enabled;
} else {

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
View File

@ -17,6 +17,7 @@
*/
#include <string.h>
#include <strings.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <ctype.h>
@ -1045,9 +1046,10 @@ static int set_privileged(const struct configuration *command_config, const stru
= get_configuration_value("docker.privileged-containers.enabled", CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf);
int ret = 0;
if (value != NULL && strcmp(value, "true") == 0) {
if (value != NULL && strcasecmp(value, "true") == 0 ) {
if (privileged_container_enabled != NULL) {
if (strcmp(privileged_container_enabled, "1") == 0) {
if (strcmp(privileged_container_enabled, "1") == 0 ||
strcasecmp(privileged_container_enabled, "True") == 0) {
ret = add_to_buffer(out, outlen, "--privileged ");
if (ret != 0) {
ret = BUFFER_TOO_SMALL;

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
View File

@ -449,6 +449,8 @@ void test_is_feature_enabled() {
fprintf(file, "feature.name4.enabled=asdkjfasdkljfklsdjf0\n");
fprintf(file, "feature.name5.enabled=-1\n");
fprintf(file, "feature.name6.enabled=2\n");
fprintf(file, "feature.name7.enabled=true\n");
fprintf(file, "feature.name8.enabled=True\n");
fclose(file);
read_config(filename, &exec_cfg);
cfg = *(get_configuration_section("", &exec_cfg));
@ -465,6 +467,10 @@ void test_is_feature_enabled() {
enabled, &cfg);
validate_feature_enabled_value(disabled, "feature.name6.enabled",
disabled, &cfg);
validate_feature_enabled_value(enabled, "feature.name7.enabled",
disabled, &cfg);
validate_feature_enabled_value(enabled, "feature.name8.enabled",
disabled, &cfg);
free_configuration(&exec_cfg);

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc
View File

@ -495,19 +495,23 @@ namespace ContainerExecutor {
char buff[buff_len];
int ret = 0;
std::string container_executor_cfg_contents[] = {"[docker]\n docker.privileged-containers.enabled=1",
"[docker]\n docker.privileged-containers.enabled=true",
"[docker]\n docker.privileged-containers.enabled=True",
"[docker]\n docker.privileged-containers.enabled=0",
"[docker]\n docker.privileged-containers.enabled=false",
"[docker]\n"};
std::vector<std::pair<std::string, std::string> > file_cmd_vec;
std::vector<std::pair<std::string, std::string> >::const_iterator itr;
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n privileged=true", "--privileged "));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run\n privileged=false", ""));
file_cmd_vec.push_back(std::make_pair<std::string, std::string>(
"[docker-command-execution]\n docker-command=run", ""));
write_container_executor_cfg(container_executor_cfg_contents[0]);
for (int i = 0; i < 3; i++ ) {
write_container_executor_cfg(container_executor_cfg_contents[i]);
ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
std::vector<std::pair<std::string, std::string> >::const_iterator itr;
if (ret != 0) {
FAIL();
}
@ -522,9 +526,11 @@ namespace ContainerExecutor {
ASSERT_EQ(0, ret);
ASSERT_STREQ(itr->second.c_str(), buff);
}
}
// check default case and when it's turned off
for (int i = 1; i < 3; ++i) {
for (int i = 3; i < 5; ++i) {
write_container_executor_cfg(container_executor_cfg_contents[i]);
ret = read_config(container_executor_cfg_file.c_str(), &container_cfg);
if (ret != 0) {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
View File

@ -196,7 +196,7 @@ are allowed. It contains the following properties:
| `docker.allowed.networks` | Comma separated networks that containers are allowed to use. If no network is specified when launching the container, the default Docker network will be used. |
| `docker.allowed.ro-mounts` | Comma separated directories that containers are allowed to mount in read-only mode. By default, no directories are allowed to mounted. |
| `docker.allowed.rw-mounts` | Comma separated directories that containers are allowed to mount in read-write mode. By default, no directories are allowed to mounted. |
| `docker.privileged-containers.enabled` | Set to 1 or 0 to enable or disable launching privileged containers. Default value is 0. |
| `docker.privileged-containers.enabled` | Set to "true" or "false" to enable or disable launching privileged containers. Default value is "false". |
Please note that if you wish to run Docker containers that require access to the YARN local directories, you must add them to the docker.allowed.rw-mounts list.
@ -209,7 +209,7 @@ The following properties are optional:
| `min.user.id` | The minimum UID that is allowed to launch applications. The default is no minimum |
| `banned.users` | A comma-separated list of usernames who should not be allowed to launch applications. The default setting is: yarn, mapred, hdfs, and bin. |
| `allowed.system.users` | A comma-separated list of usernames who should be allowed to launch applications even if their UIDs are below the configured minimum. If a user appears in allowed.system.users and banned.users, the user will be considered banned. |
| `feature.tc.enabled` | Must be 0 or 1. 0 means traffic control commands are disabled. 1 means traffic control commands are allowed. |
| `feature.tc.enabled` | Must be "true" or "false". "false" means traffic control commands are disabled. "true" means traffic control commands are allowed. |
Part of a container-executor.cfg which allows Docker containers to be launched is below: