From ec694145cf9c0ade7606813871ca2a4a371def8e Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Wed, 9 Aug 2017 10:51:29 -0700 Subject: [PATCH] YARN-6033. Add support for sections in container-executor configuration file. (Varun Vasudev via wandga) Change-Id: Ibc6d2a959debe5d8ff2b51504149742449d1f1da --- .../hadoop-yarn-server-nodemanager/pom.xml | 38 + .../src/CMakeLists.txt | 22 + .../container-executor/impl/configuration.c | 718 ++++++++++++------ .../container-executor/impl/configuration.h | 188 +++-- .../impl/container-executor.c | 39 +- .../impl/container-executor.h | 52 +- .../container-executor/impl/get_executable.c | 1 + .../native/container-executor/impl/main.c | 17 +- .../native/container-executor/impl/util.c | 134 ++++ .../native/container-executor/impl/util.h | 115 +++ .../test-configurations/configuration-1.cfg | 31 + .../test-configurations/configuration-2.cfg | 28 + .../test/test-configurations/old-config.cfg | 25 + .../test/test-container-executor.c | 15 +- .../test/test_configuration.cc | 432 +++++++++++ .../container-executor/test/test_main.cc | 29 + .../container-executor/test/test_util.cc | 138 ++++ 17 files changed, 1675 insertions(+), 347 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_configuration.cc create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_main.cc create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 28ee0d96a90..a50a769b0eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -215,6 +215,44 @@ ${project.build.directory}/native-results + + cetest + cmake-test + test + + + cetest + ${project.build.directory}/native/test + ${basedir}/src + ${project.build.directory}/native/test/cetest + + --gtest_filter=-Perf. + --gtest_output=xml:${project.build.directory}/surefire-reports/TEST-cetest.xml + + ${project.build.directory}/surefire-reports + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + make + compile + + run + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt index 5b525360800..100d7ca7e09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt @@ -19,6 +19,9 @@ cmake_minimum_required(VERSION 2.6 FATAL_ERROR) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common) include(HadoopCommon) +# Set gtest path +set(GTEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common/src/main/native/gtest) + # determine if container-executor.conf.dir is an absolute # path in case the OS we're compiling on doesn't have # a hook in get_executable. We'll use this define @@ -80,12 +83,20 @@ endfunction() include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_BINARY_DIR} + ${GTEST_SRC_DIR}/include main/native/container-executor main/native/container-executor/impl ) +# add gtest as system library to suppress gcc warnings +include_directories(SYSTEM ${GTEST_SRC_DIR}/include) + configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) +add_library(gtest ${GTEST_SRC_DIR}/gtest-all.cc) +set_target_properties(gtest PROPERTIES COMPILE_FLAGS "-w") + add_library(container + main/native/container-executor/impl/util.c main/native/container-executor/impl/configuration.c main/native/container-executor/impl/container-executor.c main/native/container-executor/impl/get_executable.c @@ -95,9 +106,11 @@ add_library(container add_executable(container-executor main/native/container-executor/impl/main.c ) + target_link_libraries(container-executor container ) + output_directory(container-executor target/usr/local/bin) add_executable(test-container-executor @@ -107,3 +120,12 @@ target_link_libraries(test-container-executor container ${EXTRA_LIBS} ) output_directory(test-container-executor target/usr/local/bin) + +# unit tests for container executor +add_executable(cetest + main/native/container-executor/impl/util.c + main/native/container-executor/test/test_configuration.cc + main/native/container-executor/test/test_main.cc + main/native/container-executor/test/test_util.cc) +target_link_libraries(cetest gtest) +output_directory(cetest test) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c index a6d7a9ce338..12dbc4c5e4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c @@ -20,35 +20,55 @@ #include #include "configuration.h" -#include "container-executor.h" +#include "util.h" +#define __STDC_FORMAT_MACROS #include #include #include -#include #include #include #include -#include -#include -#include #define MAX_SIZE 10 +static const char COMMENT_BEGIN_CHAR = '#'; +static const char SECTION_LINE_BEGIN_CHAR = '['; +static const char SECTION_LINE_END_CHAR = ']'; + +//clean up method for freeing section +void free_section(struct section *section) { + int i = 0; + for (i = 0; i < section->size; i++) { + if (section->kv_pairs[i]->key != NULL) { + free((void *) section->kv_pairs[i]->key); + } + if (section->kv_pairs[i]->value != NULL) { + free((void *) section->kv_pairs[i]->value); + } + free(section->kv_pairs[i]); + } + if (section->kv_pairs) { + free(section->kv_pairs); + section->kv_pairs = NULL; + } + if (section->name) { + free(section->name); + section->name = NULL; + } + section->size = 0; +} + //clean up method for freeing configuration -void free_configurations(struct configuration *cfg) { +void free_configuration(struct configuration *cfg) { int i = 0; for (i = 0; i < cfg->size; i++) { - if (cfg->confdetails[i]->key != NULL) { - free((void *)cfg->confdetails[i]->key); + if (cfg->sections[i] != NULL) { + free_section(cfg->sections[i]); } - if (cfg->confdetails[i]->value != NULL) { - free((void *)cfg->confdetails[i]->value); - } - free(cfg->confdetails[i]); } - if (cfg->size > 0) { - free(cfg->confdetails); + if (cfg->sections) { + free(cfg->sections); } cfg->size = 0; } @@ -65,13 +85,13 @@ static int is_only_root_writable(const char *file) { } if (file_stat.st_uid != 0) { fprintf(ERRORFILE, "File %s must be owned by root, but is owned by %" PRId64 "\n", - file, (int64_t)file_stat.st_uid); + file, (int64_t) file_stat.st_uid); return 0; } if ((file_stat.st_mode & (S_IWGRP | S_IWOTH)) != 0) { fprintf(ERRORFILE, - "File %s must not be world or group writable, but is %03lo\n", - file, (unsigned long)file_stat.st_mode & (~S_IFMT)); + "File %s must not be world or group writable, but is %03lo\n", + file, (unsigned long) file_stat.st_mode & (~S_IFMT)); return 0; } return 1; @@ -82,9 +102,9 @@ static int is_only_root_writable(const char *file) { * * NOTE: relative path names are resolved relative to the second argument not getwd(3) */ -char *resolve_config_path(const char* file_name, const char *root) { +char *resolve_config_path(const char *file_name, const char *root) { const char *real_fname = NULL; - char buffer[EXECUTOR_PATH_MAX*2 + 1]; + char buffer[EXECUTOR_PATH_MAX * 2 + 1]; if (file_name[0] == '/') { real_fname = file_name; @@ -96,7 +116,7 @@ char *resolve_config_path(const char* file_name, const char *root) { #ifdef HAVE_CANONICALIZE_FILE_NAME char * ret = (real_fname == NULL) ? NULL : canonicalize_file_name(real_fname); #else - char * ret = (real_fname == NULL) ? NULL : realpath(real_fname, NULL); + char *ret = (real_fname == NULL) ? NULL : realpath(real_fname, NULL); #endif #ifdef DEBUG fprintf(stderr,"ret = %s\n", ret); @@ -112,10 +132,19 @@ char *resolve_config_path(const char* file_name, const char *root) { * configuration and potentially cause damage. * returns 0 if permissions are ok */ -int check_configuration_permissions(const char* file_name) { +int check_configuration_permissions(const char *file_name) { + if (!file_name) { + return -1; + } + // copy the input so that we can modify it with dirname - char* dir = strdup(file_name); - char* buffer = dir; + char *dir = strdup(file_name); + if (!dir) { + fprintf(stderr, "Failed to make a copy of filename in %s.\n", __func__); + return -1; + } + + char *buffer = dir; do { if (!is_only_root_writable(dir)) { free(buffer); @@ -128,167 +157,396 @@ int check_configuration_permissions(const char* file_name) { } /** - * Trim whitespace from beginning and end. -*/ -char* trim(char* input) -{ - char *val_begin; - char *val_end; - char *ret; + * Read a line from the the config file and return it without the newline. + * The caller must free the memory allocated. + */ +static char *read_config_line(FILE *conf_file) { + char *line = NULL; + size_t linesize = 100000; + ssize_t size_read = 0; + size_t eol = 0; - if (input == NULL) { - return NULL; + line = (char *) malloc(linesize); + if (line == NULL) { + fprintf(ERRORFILE, "malloc failed while reading configuration file.\n"); + exit(OUT_OF_MEMORY); + } + size_read = getline(&line, &linesize, conf_file); + + //feof returns true only after we read past EOF. + //so a file with no new line, at last can reach this place + //if size_read returns negative check for eof condition + if (size_read == -1) { + free(line); + line = NULL; + if (!feof(conf_file)) { + fprintf(ERRORFILE, "Line read returned -1 without eof\n"); + exit(INVALID_CONFIG_FILE); } - - val_begin = input; - val_end = input + strlen(input); - - while (val_begin < val_end && isspace(*val_begin)) - val_begin++; - while (val_end > val_begin && isspace(*(val_end - 1))) - val_end--; - - ret = (char *) malloc( - sizeof(char) * (val_end - val_begin + 1)); - if (ret == NULL) { - fprintf(ERRORFILE, "Allocation error\n"); - exit(OUT_OF_MEMORY); + } else { + eol = strlen(line) - 1; + if (line[eol] == '\n') { + //trim the ending new line + line[eol] = '\0'; } - - strncpy(ret, val_begin, val_end - val_begin); - ret[val_end - val_begin] = '\0'; - return ret; + } + return line; } -void read_config(const char* file_name, struct configuration *cfg) { - FILE *conf_file; - char *line; - char *equaltok; - char *temp_equaltok; - size_t linesize = 1000; - int size_read = 0; - - if (file_name == NULL) { - fprintf(ERRORFILE, "Null configuration filename passed in\n"); - exit(INVALID_CONFIG_FILE); +/** + * Return if the given line is a comment line. + * + * @param line the line to check + * + * @return 1 if the line is a comment line, 0 otherwise + */ +static int is_comment_line(const char *line) { + if (line != NULL) { + return (line[0] == COMMENT_BEGIN_CHAR); } + return 0; +} - #ifdef DEBUG - fprintf(LOGFILE, "read_config :Conf file name is : %s \n", file_name); - #endif - - //allocate space for ten configuration items. - cfg->confdetails = (struct confentry **) malloc(sizeof(struct confentry *) - * MAX_SIZE); - cfg->size = 0; - conf_file = fopen(file_name, "r"); - if (conf_file == NULL) { - fprintf(ERRORFILE, "Invalid conf file provided : %s \n", file_name); - exit(INVALID_CONFIG_FILE); +/** + * Return if the given line is a section start line. + * + * @param line the line to check + * + * @return 1 if the line is a section start line, 0 otherwise + */ +static int is_section_start_line(const char *line) { + size_t len = 0; + if (line != NULL) { + len = strlen(line) - 1; + return (line[0] == SECTION_LINE_BEGIN_CHAR + && line[len] == SECTION_LINE_END_CHAR); } - while(!feof(conf_file)) { - line = (char *) malloc(linesize); - if(line == NULL) { - fprintf(ERRORFILE, "malloc failed while reading configuration file.\n"); + return 0; +} + +/** + * Return the name of the section from the given section start line. The + * caller must free the memory used. + * + * @param line the line to extract the section name from + * + * @return string with the name of the section, NULL otherwise + */ +static char *get_section_name(const char *line) { + char *name = NULL; + size_t len; + + if (is_section_start_line(line)) { + // length of the name is the line - 2(to account for '[' and ']') + len = strlen(line) - 2; + name = (char *) malloc(len + 1); + if (name == NULL) { + fprintf(ERRORFILE, "malloc failed while reading section name.\n"); exit(OUT_OF_MEMORY); } - size_read = getline(&line,&linesize,conf_file); + strncpy(name, line + sizeof(char), len); + name[len] = '\0'; + } + return name; +} - //feof returns true only after we read past EOF. - //so a file with no new line, at last can reach this place - //if size_read returns negative check for eof condition - if (size_read == -1) { - free(line); - if(!feof(conf_file)){ - exit(INVALID_CONFIG_FILE); - } else { - break; - } +/** + * Read an entry for the section from the line. Function returns 0 if an entry + * was found, non-zero otherwise. Return values less than 0 indicate an error + * with the config file. + * + * @param line the line to read the entry from + * @param section the struct to read the entry into + * + * @return 0 if an entry was found + * <0 for config file errors + * >0 for issues such as empty line + * + */ +static int read_section_entry(const char *line, struct section *section) { + char *equaltok; + char *temp_equaltok; + const char *splitter = "="; + char *buffer; + size_t len = 0; + if (line == NULL || section == NULL) { + fprintf(ERRORFILE, "NULL params passed to read_section_entry"); + return -1; + } + len = strlen(line); + if (len == 0) { + return 1; + } + if ((section->size) % MAX_SIZE == 0) { + section->kv_pairs = (struct kv_pair **) realloc( + section->kv_pairs, + sizeof(struct kv_pair *) * (MAX_SIZE + section->size)); + if (section->kv_pairs == NULL) { + fprintf(ERRORFILE, + "Failed re-allocating memory for configuration items\n"); + exit(OUT_OF_MEMORY); + } + } + + buffer = strdup(line); + if (!buffer) { + fprintf(ERRORFILE, "Failed to allocating memory for line, %s\n", __func__); + exit(OUT_OF_MEMORY); + } + + //tokenize first to get key and list of values. + //if no equals is found ignore this line, can be an empty line also + equaltok = strtok_r(buffer, splitter, &temp_equaltok); + if (equaltok == NULL) { + fprintf(ERRORFILE, "Error with line '%s', no '=' found\n", buffer); + exit(INVALID_CONFIG_FILE); + } + section->kv_pairs[section->size] = (struct kv_pair *) malloc( + sizeof(struct kv_pair)); + if (section->kv_pairs[section->size] == NULL) { + fprintf(ERRORFILE, "Failed allocating memory for single section item\n"); + exit(OUT_OF_MEMORY); + } + memset(section->kv_pairs[section->size], 0, + sizeof(struct kv_pair)); + section->kv_pairs[section->size]->key = trim(equaltok); + + equaltok = strtok_r(NULL, splitter, &temp_equaltok); + if (equaltok == NULL) { + // this can happen because no value was set + // e.g. banned.users=#this is a comment + int has_values = 1; + if (strstr(line, splitter) == NULL) { + fprintf(ERRORFILE, "configuration tokenization failed, error with line %s\n", line); + has_values = 0; + } + + // It is not a valid line, free memory. + free((void *) section->kv_pairs[section->size]->key); + free((void *) section->kv_pairs[section->size]); + section->kv_pairs[section->size] = NULL; + free(buffer); + + // Return -1 when no values + if (!has_values) { + return -1; + } + + // Return 2 for comments + return 2; + } + +#ifdef DEBUG + fprintf(LOGFILE, "read_config : Adding conf value : %s \n", equaltok); +#endif + + section->kv_pairs[section->size]->value = trim(equaltok); + section->size++; + free(buffer); + return 0; +} + +/** + * Remove any trailing comment from the supplied line. Function modifies the + * argument provided. + * + * @param line the line from which to remove the comment + */ +static void trim_comment(char *line) { + char *begin_comment = NULL; + if (line != NULL) { + begin_comment = strchr(line, COMMENT_BEGIN_CHAR); + if (begin_comment != NULL) { + *begin_comment = '\0'; + } + } +} + +/** + * Allocate a section struct and initialize it. The memory must be freed by + * the caller. Function calls exit if any error occurs. + * + * @return pointer to the allocated section struct + * + */ +static struct section *allocate_section() { + struct section *section = (struct section *) malloc(sizeof(struct section)); + if (section == NULL) { + fprintf(ERRORFILE, "malloc failed while allocating section.\n"); + exit(OUT_OF_MEMORY); + } + section->name = NULL; + section->kv_pairs = NULL; + section->size = 0; + return section; +} + +/** + * Populate the given section struct with fields from the config file. + * + * @param conf_file the file to read from + * @param section pointer to the section struct to populate + * + */ +static void populate_section_fields(FILE *conf_file, struct section *section) { + char *line; + long int offset = 0; + while (!feof(conf_file)) { + offset = ftell(conf_file); + line = read_config_line(conf_file); + if (line != NULL) { + if (!is_comment_line(line)) { + trim_comment(line); + if (!is_section_start_line(line)) { + if (section->name != NULL) { + if (read_section_entry(line, section) < 0) { + fprintf(ERRORFILE, "Error parsing line %s", line); + exit(INVALID_CONFIG_FILE); + } + } else { + fprintf(ERRORFILE, "Line '%s' doesn't belong to a section\n", + line); + exit(INVALID_CONFIG_FILE); + } + } else { + if (section->name == NULL) { + section->name = get_section_name(line); + if (strlen(section->name) == 0) { + fprintf(ERRORFILE, "Empty section name"); + exit(INVALID_CONFIG_FILE); + } + } else { + // we've reached the next section + fseek(conf_file, offset, SEEK_SET); + free(line); + return; + } + } + } + free(line); + } + } +} + +/** + * Read the section current section from the conf file. Section start is + * marked by lines of the form '[section-name]' and continue till the next + * section. + */ +static struct section *read_section(FILE *conf_file) { + struct section *section = allocate_section(); + populate_section_fields(conf_file, section); + if (section->name == NULL) { + free_section(section); + section = NULL; + } + return section; +} + +/** + * Merge two sections and free the second one after the merge, if desired. + * @param section1 the first section + * @param section2 the second section + * @param free_second_section free the second section if set + */ +static void merge_sections(struct section *section1, struct section *section2, const int free_second_section) { + int i = 0; + section1->kv_pairs = (struct kv_pair **) realloc( + section1->kv_pairs, + sizeof(struct kv_pair *) * (section1->size + section2->size)); + if (section1->kv_pairs == NULL) { + fprintf(ERRORFILE, + "Failed re-allocating memory for configuration items\n"); + exit(OUT_OF_MEMORY); + } + for (i = 0; i < section2->size; ++i) { + section1->kv_pairs[section1->size + i] = section2->kv_pairs[i]; + } + section1->size += section2->size; + if (free_second_section) { + free(section2->name); + memset(section2, 0, sizeof(*section2)); + free(section2); + } +} + +int read_config(const char *file_path, struct configuration *cfg) { + FILE *conf_file; + + if (file_path == NULL) { + fprintf(ERRORFILE, "Null configuration filename passed in\n"); + return INVALID_CONFIG_FILE; + } + +#ifdef DEBUG + fprintf(LOGFILE, "read_config :Conf file name is : %s \n", file_path); +#endif + + cfg->size = 0; + conf_file = fopen(file_path, "r"); + if (conf_file == NULL) { + fprintf(ERRORFILE, "Invalid conf file provided, unable to open file" + " : %s \n", file_path); + return (INVALID_CONFIG_FILE); + } + + cfg->sections = (struct section **) malloc( + sizeof(struct section *) * MAX_SIZE); + if (!cfg->sections) { + fprintf(ERRORFILE, + "Failed to allocate memory for configuration sections\n"); + exit(OUT_OF_MEMORY); + } + + // populate any entries in the older format(no sections) + cfg->sections[cfg->size] = allocate_section(); + cfg->sections[cfg->size]->name = strdup(""); + populate_section_fields(conf_file, cfg->sections[cfg->size]); + if (cfg->sections[cfg->size]) { + if (cfg->sections[cfg->size]->size) { + cfg->size++; + } else { + free_section(cfg->sections[cfg->size]); + } + } + + // populate entries in the sections format + while (!feof(conf_file)) { + cfg->sections[cfg->size] = NULL; + struct section *new_section = read_section(conf_file); + if (new_section != NULL) { + struct section *existing_section = + get_configuration_section(new_section->name, cfg); + if (existing_section != NULL) { + merge_sections((struct section *) existing_section, new_section, 1); + } else { + cfg->sections[cfg->size] = new_section; + } + } + + // Check if we need to expand memory for sections. + if (cfg->sections[cfg->size]) { + if ((cfg->size + 1) % MAX_SIZE == 0) { + cfg->sections = (struct section **) realloc(cfg->sections, + sizeof(struct sections *) * (MAX_SIZE + cfg->size)); + if (cfg->sections == NULL) { + fprintf(ERRORFILE, + "Failed re-allocating memory for configuration items\n"); + exit(OUT_OF_MEMORY); + } + } + cfg->size++; } - int eol = strlen(line) - 1; - if(line[eol] == '\n') { - //trim the ending new line - line[eol] = '\0'; - } - //comment line - if(line[0] == '#') { - free(line); - continue; - } - //tokenize first to get key and list of values. - //if no equals is found ignore this line, can be an empty line also - equaltok = strtok_r(line, "=", &temp_equaltok); - if(equaltok == NULL) { - free(line); - continue; - } - cfg->confdetails[cfg->size] = (struct confentry *) malloc( - sizeof(struct confentry)); - if(cfg->confdetails[cfg->size] == NULL) { - fprintf(LOGFILE, - "Failed allocating memory for single configuration item\n"); - goto cleanup; - } - - #ifdef DEBUG - fprintf(LOGFILE, "read_config : Adding conf key : %s \n", equaltok); - #endif - - memset(cfg->confdetails[cfg->size], 0, sizeof(struct confentry)); - cfg->confdetails[cfg->size]->key = trim(equaltok); - - equaltok = strtok_r(NULL, "=", &temp_equaltok); - if (equaltok == NULL) { - fprintf(LOGFILE, "configuration tokenization failed \n"); - goto cleanup; - } - //means value is commented so don't store the key - if(equaltok[0] == '#') { - free(line); - free((void *)cfg->confdetails[cfg->size]->key); - free(cfg->confdetails[cfg->size]); - continue; - } - - #ifdef DEBUG - fprintf(LOGFILE, "read_config : Adding conf value : %s \n", equaltok); - #endif - - cfg->confdetails[cfg->size]->value = trim(equaltok); - if((cfg->size + 1) % MAX_SIZE == 0) { - cfg->confdetails = (struct confentry **) realloc(cfg->confdetails, - sizeof(struct confentry **) * (MAX_SIZE + cfg->size)); - if (cfg->confdetails == NULL) { - fprintf(LOGFILE, - "Failed re-allocating memory for configuration items\n"); - goto cleanup; - } - } - if(cfg->confdetails[cfg->size]) { - cfg->size++; - } - - free(line); } - //close the file fclose(conf_file); if (cfg->size == 0) { - fprintf(ERRORFILE, "Invalid configuration provided in %s\n", file_name); - exit(INVALID_CONFIG_FILE); + free_configuration(cfg); + fprintf(ERRORFILE, "Invalid configuration provided in %s\n", file_path); + return INVALID_CONFIG_FILE; } - - //clean up allocated file name - return; - //free spaces alloced. - cleanup: - if (line != NULL) { - free(line); - } - fclose(conf_file); - free_configurations(cfg); - return; + return 0; } /* @@ -297,11 +555,14 @@ void read_config(const char* file_name, struct configuration *cfg) { * array, next time onwards used the populated array. * */ -char * get_value(const char* key, struct configuration *cfg) { +char *get_section_value(const char *key, const struct section *section) { int count; - for (count = 0; count < cfg->size; count++) { - if (strcmp(cfg->confdetails[count]->key, key) == 0) { - return strdup(cfg->confdetails[count]->value); + if (key == NULL || section == NULL) { + return NULL; + } + for (count = 0; count < section->size; count++) { + if (strcmp(section->kv_pairs[count]->key, key) == 0) { + return strdup(section->kv_pairs[count]->value); } } return NULL; @@ -311,61 +572,80 @@ char * get_value(const char* key, struct configuration *cfg) { * Function to return an array of values for a key. * Value delimiter is assumed to be a ','. */ -char ** get_values(const char * key, struct configuration *cfg) { - char *value = get_value(key, cfg); - return extract_values_delim(value, ","); +char **get_section_values(const char *key, const struct section *cfg) { + return get_section_values_delimiter(key, cfg, ","); } /** * Function to return an array of values for a key, using the specified delimiter. */ -char ** get_values_delim(const char * key, struct configuration *cfg, - const char *delim) { - char *value = get_value(key, cfg); - return extract_values_delim(value, delim); +char **get_section_values_delimiter(const char *key, const struct section *cfg, + const char *delim) { + if (key == NULL || cfg == NULL || delim == NULL) { + return NULL; + } + char *value = get_section_value(key, cfg); + char **split_values = split_delimiter(value, delim); + + if (value) { + free(value); + } + + return split_values; } -char ** extract_values_delim(char *value, const char *delim) { - char ** toPass = NULL; - char *tempTok = NULL; - char *tempstr = NULL; - int size = 0; - int toPassSize = MAX_SIZE; - //first allocate any array of 10 - if(value != NULL) { - toPass = (char **) malloc(sizeof(char *) * toPassSize); - tempTok = strtok_r((char *)value, delim, &tempstr); - while (tempTok != NULL) { - toPass[size++] = tempTok; - if(size == toPassSize) { - toPassSize += MAX_SIZE; - toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize)); - } - tempTok = strtok_r(NULL, delim, &tempstr); +char *get_configuration_value(const char *key, const char *section, + const struct configuration *cfg) { + const struct section *section_ptr; + if (key == NULL || section == NULL || cfg == NULL) { + return NULL; + } + section_ptr = get_configuration_section(section, cfg); + if (section_ptr != NULL) { + return get_section_value(key, section_ptr); + } + return NULL; +} + +char **get_configuration_values(const char *key, const char *section, + const struct configuration *cfg) { + const struct section *section_ptr; + if (key == NULL || section == NULL || cfg == NULL) { + return NULL; + } + section_ptr = get_configuration_section(section, cfg); + if (section_ptr != NULL) { + return get_section_values(key, section_ptr); + } + return NULL; +} + +char **get_configuration_values_delimiter(const char *key, const char *section, + const struct configuration *cfg, const char *delim) { + const struct section *section_ptr; + if (key == NULL || section == NULL || cfg == NULL || delim == NULL) { + return NULL; + } + section_ptr = get_configuration_section(section, cfg); + if (section_ptr != NULL) { + return get_section_values_delimiter(key, section_ptr, delim); + } + return NULL; +} + +struct section *get_configuration_section(const char *section, + const struct configuration *cfg) { + int i = 0; + if (cfg == NULL || section == NULL) { + return NULL; + } + for (i = 0; i < cfg->size; ++i) { + if (strcmp(cfg->sections[i]->name, section) == 0) { + return cfg->sections[i]; } } - if (toPass != NULL) { - toPass[size] = NULL; - } - return toPass; -} - -/** - * Extracts array of values from the '%' separated list of values. - */ -char ** extract_values(char *value) { - return extract_values_delim(value, "%"); -} - -// free an entry set of values -void free_values(char** values) { - if (*values != NULL) { - free(*values); - } - if (values != NULL) { - free(values); - } + return NULL; } /** @@ -376,12 +656,12 @@ int get_kv_key(const char *input, char *out, size_t out_len) { if (input == NULL) return -EINVAL; - char *split = strchr(input, '='); + const char *split = strchr(input, '='); if (split == NULL) return -EINVAL; - int key_len = split - input; + unsigned long key_len = split - input; if (out_len < (key_len + 1) || out == NULL) return -ENAMETOOLONG; @@ -400,13 +680,13 @@ int get_kv_value(const char *input, char *out, size_t out_len) { if (input == NULL) return -EINVAL; - char *split = strchr(input, '='); + const char *split = strchr(input, '='); if (split == NULL) return -EINVAL; split++; // advance past '=' to the value - int val_len = (input + strlen(input)) - split; + unsigned long val_len = (input + strlen(input)) - split; if (out_len < (val_len + 1) || out == NULL) return -ENAMETOOLONG; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h index 2d14867a0c5..1ea5561bc7f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h @@ -16,6 +16,9 @@ * limitations under the License. */ +#ifndef __YARN_CONTAINER_EXECUTOR_CONFIG_H__ +#define __YARN_CONTAINER_EXECUTOR_CONFIG_H__ + #ifdef __FreeBSD__ #define _WITH_GETLINE #endif @@ -23,62 +26,160 @@ #include /** Define a platform-independent constant instead of using PATH_MAX */ - #define EXECUTOR_PATH_MAX 4096 -/** - * Ensure that the configuration file and all of the containing directories - * are only writable by root. Otherwise, an attacker can change the - * configuration and potentially cause damage. - * returns 0 if permissions are ok - */ -int check_configuration_permissions(const char* file_name); - -/** - * Return a string with the configuration file path name resolved via realpath(3) - * - * NOTE: relative path names are resolved relative to the second argument not getwd(3) - */ -char *resolve_config_path(const char* file_name, const char *root); - -// Config data structures. -struct confentry { +// Configuration data structures. +struct kv_pair { const char *key; const char *value; }; -struct configuration { +struct section { int size; - struct confentry **confdetails; + char *name; + struct kv_pair **kv_pairs; }; -// read the given configuration file into the specified config struct. -void read_config(const char* config_file, struct configuration *cfg); - -//method exposed to get the configurations -char *get_value(const char* key, struct configuration *cfg); - -//function to return array of values pointing to the key. Values are -//comma seperated strings. -char ** get_values(const char* key, struct configuration *cfg); +struct configuration { + int size; + struct section **sections; +}; /** - * Function to return an array of values for a key, using the specified - delimiter. + * Function to ensure that the configuration file and all of the containing + * directories are only writable by root. Otherwise, an attacker can change + * the configuration and potentially cause damage. + * + * @param file_name name of the config file + * + * @returns 0 if permissions are correct, non-zero on error */ -char ** get_values_delim(const char * key, struct configuration *cfg, +int check_configuration_permissions(const char *file_name); + +/** + * Return a string with the configuration file path name resolved via + * realpath(3). Relative path names are resolved relative to the second + * argument and not getwd(3). It's up to the caller to free the returned + * value. + * + * @param file_name name of the config file + * @param root the path against which relative path names are to be resolved + * + * @returns the resolved configuration file path + */ +char* resolve_config_path(const char *file_name, const char *root); + +/** + * Read the given configuration file into the specified configuration struct. + * It's the responsibility of the caller to call free_configurations to free + * the allocated memory. The function will check to ensure that the + * configuration file has the appropriate owner and permissions. + * + * @param file_path name of the configuration file to be read + * @param cfg the configuration structure to be filled. + * + * @return 0 on success, non-zero if there was an error + */ +int read_config(const char *file_path, struct configuration *cfg); + +/** + * Get the value for a key in the specified section. It's up to the caller to + * free the memory used for storing the return value. + * + * @param key key the name of the key + * @param section the section to be looked up + * + * @return pointer to the value if the key was found, null otherwise + */ +char* get_section_value(const char *key, const struct section *section); + +/** + * Function to get the values for a given key in the specified section. + * The value is split by ",". It's up to the caller to free the memory used + * for storing the return values. + * + * @param key the key to be looked up + * @param section the section to be looked up + * + * @return array of values, null if the key was not found + */ +char** get_section_values(const char *key, const struct section *section); + +/** + * Function to get the values for a given key in the specified section. + * The value is split by the specified delimiter. It's up to the caller to + * free the memory used for storing the return values. + * + * @param key the key to be looked up + * @param section the section to be looked up + * @param delimiter the delimiter to be used to split the value + * + * @return array of values, null if the key was not found + */ +char** get_section_values_delimiter(const char *key, const struct section *section, const char *delim); -// Extracts array of values from the comma separated list of values. -char ** extract_values(char *value); +/** + * Get the value for a key in the specified section in the specified + * configuration. It's up to the caller to free the memory used for storing + * the return value. + * + * @param key key the name of the key + * @param section the name section to be looked up + * @param cfg the configuration to be used + * + * @return pointer to the value if the key was found, null otherwise + */ +char* get_configuration_value(const char *key, const char* section, + const struct configuration *cfg); -char ** extract_values_delim(char *value, const char *delim); +/** + * Function to get the values for a given key in the specified section in the + * specified configuration. The value is split by ",". It's up to the caller to + * free the memory used for storing the return values. + * + * @param key the key to be looked up + * @param section the name of the section to be looked up + * @param cfg the configuration to be looked up + * + * @return array of values, null if the key was not found + */ +char** get_configuration_values(const char *key, const char* section, + const struct configuration *cfg); -// free the memory returned by get_values -void free_values(char** values); +/** + * Function to get the values for a given key in the specified section in the + * specified configuration. The value is split by the specified delimiter. + * It's up to the caller to free the memory used for storing the return values. + * + * @param key the key to be looked up + * @param section the name of the section to be looked up + * @param cfg the section to be looked up + * @param delimiter the delimiter to be used to split the value + * + * @return array of values, null if the key was not found + */ +char** get_configuration_values_delimiter(const char *key, const char* section, + const struct configuration *cfg, const char *delimiter); -//method to free allocated configuration -void free_configurations(struct configuration *cfg); +/** + * Function to retrieve the specified section from the configuration. + * + * @param section the name of the section to retrieve + * @param cfg the configuration structure to use + * + * @return pointer to section struct containing details of the section + * null on error + */ +struct section* get_configuration_section(const char *section, + const struct configuration *cfg); + +/** + * Method to free an allocated config struct. + * + * @param cfg pointer to the structure to free + */ +void free_configuration(struct configuration *cfg); /** * If str is a string of the form key=val, find 'key' @@ -106,11 +207,4 @@ int get_kv_key(const char *input, char *out, size_t out_len); */ int get_kv_value(const char *input, char *out, size_t out_len); -/** - * Trim whitespace from beginning and end. - * - * @param input Input string that needs to be trimmed - * - * @return the trimmed string allocated with malloc. I has to be freed by the caller -*/ -char* trim(char* input); +#endif diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index def628e5902..9f754c4f63f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -19,6 +19,8 @@ #include "configuration.h" #include "container-executor.h" #include "utils/string-utils.h" +#include "util.h" +#include "config.h" #include #include @@ -43,8 +45,6 @@ #include #include -#include "config.h" - #ifndef HAVE_FCHMODAT #include "compat/fchmodat.h" #endif @@ -92,7 +92,8 @@ FILE* ERRORFILE = NULL; static uid_t nm_uid = -1; static gid_t nm_gid = -1; -struct configuration executor_cfg = {.size=0, .confdetails=NULL}; +struct configuration CFG = {.size=0, .sections=NULL}; +struct section executor_cfg = {.size=0, .kv_pairs=NULL}; char *concatenate(char *concat_pattern, char *return_path_name, int numArgs, ...); @@ -103,18 +104,25 @@ void set_nm_uid(uid_t user, gid_t group) { } //function used to load the configurations present in the secure config -void read_executor_config(const char* file_name) { - read_config(file_name, &executor_cfg); +void read_executor_config(const char *file_name) { + const struct section *tmp = NULL; + int ret = read_config(file_name, &CFG); + if (ret == 0) { + tmp = get_configuration_section("", &CFG); + if (tmp != NULL) { + executor_cfg = *tmp; + } + } } //function used to free executor configuration data void free_executor_configurations() { - free_configurations(&executor_cfg); + free_configuration(&CFG); } //Lookup nodemanager group from container executor configuration. char *get_nodemanager_group() { - return get_value(NM_GROUP_KEY, &executor_cfg); + return get_section_value(NM_GROUP_KEY, &executor_cfg); } int check_executor_permissions(char *executable_file) { @@ -431,8 +439,8 @@ int change_user(uid_t user, gid_t group) { } int is_feature_enabled(const char* feature_key, int default_value, - struct configuration *cfg) { - char *enabled_str = get_value(feature_key, cfg); + struct section *cfg) { + char *enabled_str = get_section_value(feature_key, cfg); int enabled = default_value; if (enabled_str != NULL) { @@ -753,7 +761,7 @@ static struct passwd* get_user_info(const char* user) { } int is_whitelisted(const char *user) { - char **whitelist = get_values(ALLOWED_SYSTEM_USERS_KEY, &executor_cfg); + char **whitelist = get_section_values(ALLOWED_SYSTEM_USERS_KEY, &executor_cfg); char **users = whitelist; if (whitelist != NULL) { for(; *users; ++users) { @@ -781,7 +789,7 @@ struct passwd* check_user(const char *user) { fflush(LOGFILE); return NULL; } - char *min_uid_str = get_value(MIN_USERID_KEY, &executor_cfg); + char *min_uid_str = get_section_value(MIN_USERID_KEY, &executor_cfg); int min_uid = DEFAULT_MIN_USERID; if (min_uid_str != NULL) { char *end_ptr = NULL; @@ -808,7 +816,7 @@ struct passwd* check_user(const char *user) { free(user_info); return NULL; } - char **banned_users = get_values(BANNED_USERS_KEY, &executor_cfg); + char **banned_users = get_section_values(BANNED_USERS_KEY, &executor_cfg); banned_users = banned_users == NULL ? (char**) DEFAULT_BANNED_USERS : banned_users; char **banned_user = banned_users; @@ -1194,7 +1202,6 @@ char** tokenize_docker_command(const char *input, int *split_counter) { char *line = (char *)calloc(strlen(input) + 1, sizeof(char)); char **linesplit = (char **) malloc(sizeof(char *)); char *p = NULL; - int c = 0; *split_counter = 0; strncpy(line, input, strlen(input)); @@ -1408,12 +1415,12 @@ char* parse_docker_command_file(const char* command_file) { int run_docker(const char *command_file) { char* docker_command = parse_docker_command_file(command_file); - char* docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg); + char* docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg); docker_binary = check_docker_binary(docker_binary); char* docker_command_with_binary = calloc(sizeof(char), EXECUTOR_PATH_MAX); snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command); - char **args = extract_values_delim(docker_command_with_binary, " "); + char **args = split_delimiter(docker_command_with_binary, " "); int exit_code = -1; if (execvp(docker_binary, args) != 0) { @@ -1574,7 +1581,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id, uid_t prev_uid = geteuid(); char *docker_command = parse_docker_command_file(command_file); - char *docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg); + char *docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg); docker_binary = check_docker_binary(docker_binary); fprintf(LOGFILE, "Creating script paths...\n"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 1dc0491fdfc..ea8b5e38a18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -35,51 +35,6 @@ enum command { LIST_AS_USER = 5 }; -enum errorcodes { - INVALID_ARGUMENT_NUMBER = 1, - //INVALID_USER_NAME 2 - INVALID_COMMAND_PROVIDED = 3, - // SUPER_USER_NOT_ALLOWED_TO_RUN_TASKS (NOT USED) 4 - INVALID_NM_ROOT_DIRS = 5, - SETUID_OPER_FAILED, //6 - UNABLE_TO_EXECUTE_CONTAINER_SCRIPT, //7 - UNABLE_TO_SIGNAL_CONTAINER, //8 - INVALID_CONTAINER_PID, //9 - // ERROR_RESOLVING_FILE_PATH (NOT_USED) 10 - // RELATIVE_PATH_COMPONENTS_IN_FILE_PATH (NOT USED) 11 - // UNABLE_TO_STAT_FILE (NOT USED) 12 - // FILE_NOT_OWNED_BY_ROOT (NOT USED) 13 - // PREPARE_CONTAINER_DIRECTORIES_FAILED (NOT USED) 14 - // INITIALIZE_CONTAINER_FAILED (NOT USED) 15 - // PREPARE_CONTAINER_LOGS_FAILED (NOT USED) 16 - // INVALID_LOG_DIR (NOT USED) 17 - OUT_OF_MEMORY = 18, - // INITIALIZE_DISTCACHEFILE_FAILED (NOT USED) 19 - INITIALIZE_USER_FAILED = 20, - PATH_TO_DELETE_IS_NULL, //21 - INVALID_CONTAINER_EXEC_PERMISSIONS, //22 - // PREPARE_JOB_LOGS_FAILED (NOT USED) 23 - INVALID_CONFIG_FILE = 24, - SETSID_OPER_FAILED = 25, - WRITE_PIDFILE_FAILED = 26, - WRITE_CGROUP_FAILED = 27, - TRAFFIC_CONTROL_EXECUTION_FAILED = 28, - DOCKER_RUN_FAILED = 29, - ERROR_OPENING_DOCKER_FILE = 30, - ERROR_READING_DOCKER_FILE = 31, - FEATURE_DISABLED = 32, - COULD_NOT_CREATE_SCRIPT_COPY = 33, - COULD_NOT_CREATE_CREDENTIALS_FILE = 34, - COULD_NOT_CREATE_WORK_DIRECTORIES = 35, - COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36, - COULD_NOT_CREATE_TMP_DIRECTORIES = 37, - ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38, - ERROR_SANITIZING_DOCKER_COMMAND = 39, - DOCKER_IMAGE_INVALID = 40, - DOCKER_CONTAINER_NAME_INVALID = 41, - ERROR_COMPILING_REGEX = 42 -}; - enum operations { CHECK_SETUP = 1, MOUNT_CGROUPS = 2, @@ -111,11 +66,6 @@ enum operations { extern struct passwd *user_detail; -// the log file for messages -extern FILE *LOGFILE; -// the log file for error messages -extern FILE *ERRORFILE; - // get the executable's filename char* get_executable(char *argv0); @@ -276,7 +226,7 @@ int create_validate_dir(const char* npath, mode_t perm, const char* path, /** Check if a feature is enabled in the specified configuration. */ int is_feature_enabled(const char* feature_key, int default_value, - struct configuration *cfg); + struct section *cfg); /** Check if tc (traffic control) support is enabled in configuration. */ int is_tc_support_enabled(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c index ce46b776fd8..55973a2db57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c @@ -31,6 +31,7 @@ #include "config.h" #include "configuration.h" #include "container-executor.h" +#include "util.h" #include #include diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index fdc0496986d..b2187c9daf0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -19,6 +19,7 @@ #include "config.h" #include "configuration.h" #include "container-executor.h" +#include "util.h" #include #include @@ -420,7 +421,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) cmd_input.resources_key = resources_key; cmd_input.resources_value = resources_value; - cmd_input.resources_values = extract_values(resources_value); + cmd_input.resources_values = split(resources_value); *operation = RUN_AS_USER_LAUNCH_DOCKER_CONTAINER; return 0; } else { @@ -471,7 +472,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) cmd_input.resources_key = resources_key; cmd_input.resources_value = resources_value; - cmd_input.resources_values = extract_values(resources_value); + cmd_input.resources_values = split(resources_value); *operation = RUN_AS_USER_LAUNCH_CONTAINER; return 0; @@ -565,8 +566,8 @@ int main(int argc, char **argv) { exit_code = initialize_app(cmd_input.yarn_user_name, cmd_input.app_id, cmd_input.cred_file, - extract_values(cmd_input.local_dirs), - extract_values(cmd_input.log_dirs), + split(cmd_input.local_dirs), + split(cmd_input.log_dirs), argv + optind); break; case RUN_AS_USER_LAUNCH_DOCKER_CONTAINER: @@ -591,8 +592,8 @@ int main(int argc, char **argv) { cmd_input.script_file, cmd_input.cred_file, cmd_input.pid_file, - extract_values(cmd_input.local_dirs), - extract_values(cmd_input.log_dirs), + split(cmd_input.local_dirs), + split(cmd_input.log_dirs), cmd_input.docker_command_file, cmd_input.resources_key, cmd_input.resources_values); @@ -619,8 +620,8 @@ int main(int argc, char **argv) { cmd_input.script_file, cmd_input.cred_file, cmd_input.pid_file, - extract_values(cmd_input.local_dirs), - extract_values(cmd_input.log_dirs), + split(cmd_input.local_dirs), + split(cmd_input.log_dirs), cmd_input.resources_key, cmd_input.resources_values); free(cmd_input.resources_key); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c new file mode 100644 index 00000000000..8e39ca85760 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "util.h" +#include +#include +#include +#include + +char** split_delimiter(char *value, const char *delim) { + char **return_values = NULL; + char *temp_tok = NULL; + char *tempstr = NULL; + int size = 0; + int per_alloc_size = 10; + int return_values_size = per_alloc_size; + int failed = 0; + + //first allocate any array of 10 + if(value != NULL) { + return_values = (char **) malloc(sizeof(char *) * return_values_size); + if (!return_values) { + fprintf(ERRORFILE, "Allocation error for return_values in %s.\n", + __func__); + failed = 1; + goto cleanup; + } + memset(return_values, 0, sizeof(char *) * return_values_size); + + temp_tok = strtok_r(value, delim, &tempstr); + while (temp_tok != NULL) { + temp_tok = strdup(temp_tok); + if (NULL == temp_tok) { + fprintf(ERRORFILE, "Allocation error in %s.\n", __func__); + failed = 1; + goto cleanup; + } + + return_values[size++] = temp_tok; + + // Make sure returned values has enough space for the trailing NULL. + if (size >= return_values_size - 1) { + return_values_size += per_alloc_size; + return_values = (char **) realloc(return_values,(sizeof(char *) * + return_values_size)); + + // Make sure new added memory are filled with NULL + for (int i = size; i < return_values_size; i++) { + return_values[i] = NULL; + } + } + temp_tok = strtok_r(NULL, delim, &tempstr); + } + } + + // Put trailing NULL to indicate values terminates. + if (return_values != NULL) { + return_values[size] = NULL; + } + +cleanup: + if (failed) { + free_values(return_values); + return NULL; + } + + return return_values; +} + +/** + * Extracts array of values from the '%' separated list of values. + */ +char** split(char *value) { + return split_delimiter(value, "%"); +} + +// free an entry set of values +void free_values(char** values) { + if (values != NULL) { + int idx = 0; + while (values[idx]) { + free(values[idx]); + idx++; + } + free(values); + } +} + +/** + * Trim whitespace from beginning and end. +*/ +char* trim(const char* input) { + const char *val_begin; + const char *val_end; + char *ret; + + if (input == NULL) { + return NULL; + } + + val_begin = input; + val_end = input + strlen(input); + + while (val_begin < val_end && isspace(*val_begin)) + val_begin++; + while (val_end > val_begin && isspace(*(val_end - 1))) + val_end--; + + ret = (char *) malloc( + sizeof(char) * (val_end - val_begin + 1)); + if (ret == NULL) { + fprintf(ERRORFILE, "Allocation error\n"); + exit(OUT_OF_MEMORY); + } + + strncpy(ret, val_begin, val_end - val_begin); + ret[val_end - val_begin] = '\0'; + return ret; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h new file mode 100644 index 00000000000..a8a12a937bb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__ +#define __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__ + +#include + +enum errorcodes { + INVALID_ARGUMENT_NUMBER = 1, + //INVALID_USER_NAME 2 + INVALID_COMMAND_PROVIDED = 3, + // SUPER_USER_NOT_ALLOWED_TO_RUN_TASKS (NOT USED) 4 + INVALID_NM_ROOT_DIRS = 5, + SETUID_OPER_FAILED, //6 + UNABLE_TO_EXECUTE_CONTAINER_SCRIPT, //7 + UNABLE_TO_SIGNAL_CONTAINER, //8 + INVALID_CONTAINER_PID, //9 + // ERROR_RESOLVING_FILE_PATH (NOT_USED) 10 + // RELATIVE_PATH_COMPONENTS_IN_FILE_PATH (NOT USED) 11 + // UNABLE_TO_STAT_FILE (NOT USED) 12 + // FILE_NOT_OWNED_BY_ROOT (NOT USED) 13 + // PREPARE_CONTAINER_DIRECTORIES_FAILED (NOT USED) 14 + // INITIALIZE_CONTAINER_FAILED (NOT USED) 15 + // PREPARE_CONTAINER_LOGS_FAILED (NOT USED) 16 + // INVALID_LOG_DIR (NOT USED) 17 + OUT_OF_MEMORY = 18, + // INITIALIZE_DISTCACHEFILE_FAILED (NOT USED) 19 + INITIALIZE_USER_FAILED = 20, + PATH_TO_DELETE_IS_NULL, //21 + INVALID_CONTAINER_EXEC_PERMISSIONS, //22 + // PREPARE_JOB_LOGS_FAILED (NOT USED) 23 + INVALID_CONFIG_FILE = 24, + SETSID_OPER_FAILED = 25, + WRITE_PIDFILE_FAILED = 26, + WRITE_CGROUP_FAILED = 27, + TRAFFIC_CONTROL_EXECUTION_FAILED = 28, + DOCKER_RUN_FAILED = 29, + ERROR_OPENING_DOCKER_FILE = 30, + ERROR_READING_DOCKER_FILE = 31, + FEATURE_DISABLED = 32, + COULD_NOT_CREATE_SCRIPT_COPY = 33, + COULD_NOT_CREATE_CREDENTIALS_FILE = 34, + COULD_NOT_CREATE_WORK_DIRECTORIES = 35, + COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36, + COULD_NOT_CREATE_TMP_DIRECTORIES = 37, + ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38, + ERROR_SANITIZING_DOCKER_COMMAND = 39, + DOCKER_IMAGE_INVALID = 40, + DOCKER_CONTAINER_NAME_INVALID = 41, + ERROR_COMPILING_REGEX = 42 +}; + + +// the log file for messages +extern FILE *LOGFILE; +// the log file for error messages +extern FILE *ERRORFILE; +/** + * Function to split the given string using '%' as the separator. It's + * up to the caller to free the memory for the returned array. Use the + * free_values function to free the allocated memory. + * + * @param str the string to split + * + * @return an array of strings + */ +char** split(char *str); + +/** + * Function to split the given string using the delimiter specified. It's + * up to the caller to free the memory for the returned array. Use the + * free_values function to free the allocated memory. + * + * @param str the string to split + * @param delimiter the delimiter to use + * + * @return an array of strings + */ +char** split_delimiter(char *value, const char *delimiter); + +/** + * Function to free an array of strings. + * + * @param values the array to free + * + */ +void free_values(char **values); + +/** + * Trim whitespace from beginning and end. The returned string has to be freed + * by the caller. + * + * @param input Input string that needs to be trimmed + * + * @return the trimmed string allocated with malloc +*/ +char* trim(const char *input); + +#endif diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg new file mode 100644 index 00000000000..4d0b90d0cec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +[section-1] +key1=value1 +split-key=val1,val2,val3 +perc-key=perc-val1%perc-val2 +# some comment + +[split-section] +key3=value3 + +[section-2] +key1=value2 + +key2=value2 + +[split-section] +key4=value4 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg new file mode 100644 index 00000000000..aa02db84a44 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test mixed mode config file +# Initial few lines are in the key=value format +# and then the sections start + +key1=value1 +key2=value2 + + +[section-1] +key3=value3 +key1=value4 + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg new file mode 100644 index 00000000000..947a3fae605 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +yarn.nodemanager.linux-container-executor.group=yarn +banned.users=root,testuser1,testuser2#comma separated list of users who can not run applications +min.user.id=1000 +allowed.system.users=nobody,daemon +feature.docker.enabled=1 +feature.tc.enabled=0 +docker.binary=/usr/bin/docker +yarn.local.dirs=/var/run/yarn%/tmp/mydir +test.key=#no value for this key +# test.key2=0 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 320265270c0..3cfefa03e82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -18,6 +18,7 @@ #include "configuration.h" #include "container-executor.h" #include "utils/string-utils.h" +#include "util.h" #include #include @@ -404,7 +405,7 @@ void test_delete_app() { } void validate_feature_enabled_value(int expected_value, const char* key, - int default_value, struct configuration *cfg) { + int default_value, struct section *cfg) { int value = is_feature_enabled(key, default_value, cfg); if (value != expected_value) { @@ -419,7 +420,8 @@ void test_is_feature_enabled() { FILE *file = fopen(filename, "w"); int disabled = 0; int enabled = 1; - struct configuration cfg = {.size=0, .confdetails=NULL}; + struct configuration exec_cfg = {.size=0, .sections=NULL}; + struct section cfg = {.size=0, .kv_pairs=NULL}; if (file == NULL) { printf("FAIL: Could not open configuration file: %s\n", filename); @@ -433,7 +435,8 @@ void test_is_feature_enabled() { fprintf(file, "feature.name5.enabled=-1\n"); fprintf(file, "feature.name6.enabled=2\n"); fclose(file); - read_config(filename, &cfg); + read_config(filename, &exec_cfg); + cfg = *(get_configuration_section("", &exec_cfg)); validate_feature_enabled_value(disabled, "feature.name1.enabled", disabled, &cfg); @@ -449,7 +452,7 @@ void test_is_feature_enabled() { disabled, &cfg); - free_configurations(&cfg); + free_configuration(&exec_cfg); } void test_delete_user() { @@ -1345,8 +1348,8 @@ int main(int argc, char **argv) { read_executor_config(TEST_ROOT "/test.cfg"); - local_dirs = extract_values(strdup(NM_LOCAL_DIRS)); - log_dirs = extract_values(strdup(NM_LOG_DIRS)); + local_dirs = split(strdup(NM_LOCAL_DIRS)); + log_dirs = split(strdup(NM_LOG_DIRS)); create_nm_roots(local_dirs); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_configuration.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_configuration.cc new file mode 100644 index 00000000000..6ee0ab2be1c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_configuration.cc @@ -0,0 +1,432 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +extern "C" { +#include "util.h" +#include "configuration.h" +#include "configuration.c" +} + + +namespace ContainerExecutor { + class TestConfiguration : public ::testing::Test { + protected: + virtual void SetUp() { + new_config_format_file = "test-configurations/configuration-1.cfg"; + old_config_format_file = "test-configurations/old-config.cfg"; + mixed_config_format_file = "test-configurations/configuration-2.cfg"; + loadConfigurations(); + return; + } + + void loadConfigurations() { + int ret = 0; + ret = read_config(new_config_format_file.c_str(), &new_config_format); + ASSERT_EQ(0, ret); + ret = read_config(old_config_format_file.c_str(), &old_config_format); + ASSERT_EQ(0, ret); + ret = read_config(mixed_config_format_file.c_str(), + &mixed_config_format); + ASSERT_EQ(0, ret); + } + + virtual void TearDown() { + free_configuration(&new_config_format); + free_configuration(&old_config_format); + return; + } + + std::string new_config_format_file; + std::string old_config_format_file; + std::string mixed_config_format_file; + struct configuration new_config_format; + struct configuration old_config_format; + struct configuration mixed_config_format; + }; + + + TEST_F(TestConfiguration, test_get_configuration_values_delimiter) { + char **split_values; + split_values = get_configuration_values_delimiter(NULL, "", &old_config_format, "%"); + ASSERT_EQ(NULL, split_values); + split_values = get_configuration_values_delimiter("yarn.local.dirs", NULL, + &old_config_format, "%"); + ASSERT_EQ(NULL, split_values); + split_values = get_configuration_values_delimiter("yarn.local.dirs", "", + NULL, "%"); + ASSERT_EQ(NULL, split_values); + split_values = get_configuration_values_delimiter("yarn.local.dirs", "", + &old_config_format, NULL); + ASSERT_EQ(NULL, split_values); + split_values = get_configuration_values_delimiter("yarn.local.dirs", "abcd", + &old_config_format, "%"); + ASSERT_EQ(NULL, split_values); + split_values = get_configuration_values_delimiter("yarn.local.dirs", "", + &old_config_format, "%"); + ASSERT_STREQ("/var/run/yarn", split_values[0]); + ASSERT_STREQ("/tmp/mydir", split_values[1]); + ASSERT_EQ(NULL, split_values[2]); + free(split_values); + split_values = get_configuration_values_delimiter("allowed.system.users", + "", &old_config_format, "%"); + ASSERT_STREQ("nobody,daemon", split_values[0]); + ASSERT_EQ(NULL, split_values[1]); + free(split_values); + } + + TEST_F(TestConfiguration, test_get_configuration_values) { + char **split_values; + split_values = get_configuration_values(NULL, "", &old_config_format); + ASSERT_EQ(NULL, split_values); + split_values = get_configuration_values("yarn.local.dirs", NULL, &old_config_format); + ASSERT_EQ(NULL, split_values); + split_values = get_configuration_values("yarn.local.dirs", "", NULL); + ASSERT_EQ(NULL, split_values); + split_values = get_configuration_values("yarn.local.dirs", "abcd", &old_config_format); + ASSERT_EQ(NULL, split_values); + split_values = get_configuration_values("yarn.local.dirs", "", &old_config_format); + ASSERT_STREQ("/var/run/yarn%/tmp/mydir", split_values[0]); + ASSERT_EQ(NULL, split_values[1]); + free(split_values); + split_values = get_configuration_values("allowed.system.users", "", + &old_config_format); + ASSERT_STREQ("nobody", split_values[0]); + ASSERT_STREQ("daemon", split_values[1]); + ASSERT_EQ(NULL, split_values[2]); + free(split_values); + } + + TEST_F(TestConfiguration, test_get_configuration_value) { + std::string key_value_array[5][2] = { + {"yarn.nodemanager.linux-container-executor.group", "yarn"}, + {"min.user.id", "1000"}, + {"allowed.system.users", "nobody,daemon"}, + {"feature.docker.enabled", "1"}, + {"yarn.local.dirs", "/var/run/yarn%/tmp/mydir"} + }; + char *value; + value = get_configuration_value(NULL, "", &old_config_format); + ASSERT_EQ(NULL, value); + value = get_configuration_value("yarn.local.dirs", NULL, &old_config_format); + ASSERT_EQ(NULL, value); + value = get_configuration_value("yarn.local.dirs", "", NULL); + ASSERT_EQ(NULL, value); + + for (int i = 0; i < 5; ++i) { + value = get_configuration_value(key_value_array[i][0].c_str(), + "", &old_config_format); + ASSERT_STREQ(key_value_array[i][1].c_str(), value); + free(value); + } + value = get_configuration_value("test.key", "", &old_config_format); + ASSERT_EQ(NULL, value); + value = get_configuration_value("test.key2", "", &old_config_format); + ASSERT_EQ(NULL, value); + value = get_configuration_value("feature.tc.enabled", "abcd", &old_config_format); + ASSERT_EQ(NULL, value); + } + + TEST_F(TestConfiguration, test_no_sections_format) { + const struct section *executor_cfg = get_configuration_section("", &old_config_format); + char *value = NULL; + value = get_section_value("yarn.nodemanager.linux-container-executor.group", executor_cfg); + ASSERT_STREQ("yarn", value); + value = get_section_value("feature.docker.enabled", executor_cfg); + ASSERT_STREQ("1", value); + value = get_section_value("feature.tc.enabled", executor_cfg); + ASSERT_STREQ("0", value); + value = get_section_value("min.user.id", executor_cfg); + ASSERT_STREQ("1000", value); + value = get_section_value("docker.binary", executor_cfg); + ASSERT_STREQ("/usr/bin/docker", value); + char **list = get_section_values("allowed.system.users", executor_cfg); + ASSERT_STREQ("nobody", list[0]); + ASSERT_STREQ("daemon", list[1]); + list = get_section_values("banned.users", executor_cfg); + ASSERT_STREQ("root", list[0]); + ASSERT_STREQ("testuser1", list[1]); + ASSERT_STREQ("testuser2", list[2]); + } + + TEST_F(TestConfiguration, test_get_section_values_delimiter) { + const struct section *section; + char *value; + char **split_values; + section = get_configuration_section("section-1", &new_config_format); + value = get_section_value("key1", section); + ASSERT_STREQ("value1", value); + free(value); + value = get_section_value("key2", section); + ASSERT_EQ(NULL, value); + split_values = get_section_values_delimiter(NULL, section, "%"); + ASSERT_EQ(NULL, split_values); + split_values = get_section_values_delimiter("split-key", NULL, "%"); + ASSERT_EQ(NULL, split_values); + split_values = get_section_values_delimiter("split-key", section, NULL); + ASSERT_EQ(NULL, split_values); + split_values = get_section_values_delimiter("split-key", section, "%"); + ASSERT_FALSE(split_values == NULL); + ASSERT_STREQ("val1,val2,val3", split_values[0]); + ASSERT_TRUE(split_values[1] == NULL); + free_values(split_values); + split_values = get_section_values_delimiter("perc-key", section, "%"); + ASSERT_FALSE(split_values == NULL); + ASSERT_STREQ("perc-val1", split_values[0]); + ASSERT_STREQ("perc-val2", split_values[1]); + ASSERT_TRUE(split_values[2] == NULL); + } + + TEST_F(TestConfiguration, test_get_section_values) { + const struct section *section; + char *value; + char **split_values; + section = get_configuration_section("section-1", &new_config_format); + value = get_section_value(NULL, section); + ASSERT_EQ(NULL, value); + value = get_section_value("key1", NULL); + ASSERT_EQ(NULL, value); + value = get_section_value("key1", section); + ASSERT_STREQ("value1", value); + free(value); + value = get_section_value("key2", section); + ASSERT_EQ(NULL, value); + split_values = get_section_values("split-key", section); + ASSERT_FALSE(split_values == NULL); + ASSERT_STREQ("val1", split_values[0]); + ASSERT_STREQ("val2", split_values[1]); + ASSERT_STREQ("val3", split_values[2]); + ASSERT_TRUE(split_values[3] == NULL); + free_values(split_values); + split_values = get_section_values("perc-key", section); + ASSERT_FALSE(split_values == NULL); + ASSERT_STREQ("perc-val1%perc-val2", split_values[0]); + ASSERT_TRUE(split_values[1] == NULL); + free_values(split_values); + section = get_configuration_section("section-2", &new_config_format); + value = get_section_value("key1", section); + ASSERT_STREQ("value2", value); + free(value); + value = get_section_value("key2", section); + ASSERT_STREQ("value2", value); + free(value); + } + + TEST_F(TestConfiguration, test_split_section) { + const struct section *section; + char *value; + section = get_configuration_section("split-section", &new_config_format); + value = get_section_value(NULL, section); + ASSERT_EQ(NULL, value); + value = get_section_value("key3", NULL); + ASSERT_EQ(NULL, value); + value = get_section_value("key3", section); + ASSERT_STREQ("value3", value); + free(value); + value = get_section_value("key4", section); + ASSERT_STREQ("value4", value); + + } + + TEST_F(TestConfiguration, test_get_configuration_section) { + const struct section *section; + ASSERT_EQ(3, new_config_format.size); + section = get_configuration_section(NULL, &new_config_format); + ASSERT_EQ(NULL, section); + section = get_configuration_section("section-1", NULL); + ASSERT_EQ(NULL, section); + section = get_configuration_section("section-1", &new_config_format); + ASSERT_FALSE(section == NULL); + ASSERT_STREQ("section-1", section->name); + ASSERT_EQ(3, section->size); + ASSERT_FALSE(NULL == section->kv_pairs); + section = get_configuration_section("section-2", &new_config_format); + ASSERT_FALSE(section == NULL); + ASSERT_STREQ("section-2", section->name); + ASSERT_EQ(2, section->size); + ASSERT_FALSE(NULL == section->kv_pairs); + section = get_configuration_section("section-3", &new_config_format); + ASSERT_TRUE(section == NULL); + } + + TEST_F(TestConfiguration, test_read_config) { + struct configuration config; + int ret = 0; + + ret = read_config(NULL, &config); + ASSERT_EQ(INVALID_CONFIG_FILE, ret); + ret = read_config("bad-config-file", &config); + ASSERT_EQ(INVALID_CONFIG_FILE, ret); + ret = read_config(new_config_format_file.c_str(), &config); + ASSERT_EQ(0, ret); + ASSERT_EQ(3, config.size); + ASSERT_STREQ("section-1", config.sections[0]->name); + ASSERT_STREQ("split-section", config.sections[1]->name); + ASSERT_STREQ("section-2", config.sections[2]->name); + free_configuration(&config); + ret = read_config(old_config_format_file.c_str(), &config); + ASSERT_EQ(0, ret); + ASSERT_EQ(1, config.size); + ASSERT_STREQ("", config.sections[0]->name); + free_configuration(&config); + } + + TEST_F(TestConfiguration, test_get_kv_key) { + int ret = 0; + char buff[1024]; + ret = get_kv_key(NULL, buff, 1024); + ASSERT_EQ(-EINVAL, ret); + ret = get_kv_key("key1234", buff, 1024); + ASSERT_EQ(-EINVAL, ret); + ret = get_kv_key("key=abcd", NULL, 1024); + ASSERT_EQ(-ENAMETOOLONG, ret); + ret = get_kv_key("key=abcd", buff, 1); + ASSERT_EQ(-ENAMETOOLONG, ret); + ret = get_kv_key("key=abcd", buff, 1024); + ASSERT_EQ(0, ret); + ASSERT_STREQ("key", buff); + } + + TEST_F(TestConfiguration, test_get_kv_value) { + int ret = 0; + char buff[1024]; + ret = get_kv_value(NULL, buff, 1024); + ASSERT_EQ(-EINVAL, ret); + ret = get_kv_value("key1234", buff, 1024); + ASSERT_EQ(-EINVAL, ret); + ret = get_kv_value("key=abcd", NULL, 1024); + ASSERT_EQ(-ENAMETOOLONG, ret); + ret = get_kv_value("key=abcd", buff, 1); + ASSERT_EQ(-ENAMETOOLONG, ret); + ret = get_kv_value("key=abcd", buff, 1024); + ASSERT_EQ(0, ret); + ASSERT_STREQ("abcd", buff); + } + + TEST_F(TestConfiguration, test_single_section_high_key_count) { + std::string section_name = "section-1"; + std::string sample_file_name = "large-section.cfg"; + std::ofstream sample_file; + sample_file.open(sample_file_name.c_str()); + sample_file << "[" << section_name << "]" << std::endl; + for(int i = 0; i < MAX_SIZE + 2; ++i) { + sample_file << "key" << i << "=" << "value" << i << std::endl; + } + struct configuration cfg; + int ret = read_config(sample_file_name.c_str(), &cfg); + ASSERT_EQ(0, ret); + ASSERT_EQ(1, cfg.size); + const struct section *section1 = get_configuration_section(section_name.c_str(), &cfg); + ASSERT_EQ(MAX_SIZE + 2, section1->size); + ASSERT_STREQ(section_name.c_str(), section1->name); + for(int i = 0; i < MAX_SIZE + 2; ++i) { + std::ostringstream oss; + oss << "key" << i; + const char *value = get_section_value(oss.str().c_str(), section1); + oss.str(""); + oss << "value" << i; + ASSERT_STREQ(oss.str().c_str(), value); + } + remove(sample_file_name.c_str()); + free_configuration(&cfg); + } + + TEST_F(TestConfiguration, test_multiple_sections) { + std::string sample_file_name = "multiple-sections.cfg"; + std::ofstream sample_file; + sample_file.open(sample_file_name.c_str()); + for(int i = 0; i < MAX_SIZE + 2; ++i) { + sample_file << "[section-" << i << "]" << std::endl; + sample_file << "key" << i << "=" << "value" << i << std::endl; + } + struct configuration cfg; + int ret = read_config(sample_file_name.c_str(), &cfg); + ASSERT_EQ(0, ret); + ASSERT_EQ(MAX_SIZE + 2, cfg.size); + for(int i = 0; i < MAX_SIZE + 2; ++i) { + std::ostringstream oss; + oss << "section-" << i; + const struct section *section = get_configuration_section(oss.str().c_str(), &cfg); + ASSERT_EQ(1, section->size); + ASSERT_STREQ(oss.str().c_str(), section->name); + oss.str(""); + oss << "key" << i; + const char *value = get_section_value(oss.str().c_str(), section); + oss.str(""); + oss << "value" << i; + ASSERT_STREQ(oss.str().c_str(), value); + } + remove(sample_file_name.c_str()); + free_configuration(&cfg); + } + + TEST_F(TestConfiguration, test_section_start_line) { + const char *section_start_line = "[abcd]"; + const char *non_section_lines[] = { + "[abcd", "abcd]", "key=value", "#abcd" + }; + int ret = is_section_start_line(section_start_line); + ASSERT_EQ(1, ret); + int length = sizeof(non_section_lines) / sizeof(*non_section_lines); + for( int i = 0; i < length; ++i) { + ret = is_section_start_line(non_section_lines[i]); + ASSERT_EQ(0, ret); + } + ret = is_section_start_line(NULL); + ASSERT_EQ(0, ret); + } + + TEST_F(TestConfiguration, test_comment_line) { + const char *comment_line = "#[abcd]"; + const char *non_comment_lines[] = { + "[abcd", "abcd]", "key=value", "[abcd]" + }; + int ret = is_comment_line(comment_line); + ASSERT_EQ(1, ret); + int length = sizeof(non_comment_lines) / sizeof(*non_comment_lines); + for( int i = 0; i < length; ++i) { + ret = is_comment_line(non_comment_lines[i]); + ASSERT_EQ(0, ret); + } + ret = is_comment_line(NULL); + ASSERT_EQ(0, ret); + } + + TEST_F(TestConfiguration, test_mixed_config_format) { + const struct section *executor_cfg = + get_configuration_section("", &mixed_config_format); + char *value = NULL; + value = get_section_value("key1", executor_cfg); + ASSERT_STREQ("value1", value); + value = get_section_value("key2", executor_cfg); + ASSERT_STREQ("value2", value); + ASSERT_EQ(2, executor_cfg->size); + executor_cfg = get_configuration_section("section-1", + &mixed_config_format); + value = get_section_value("key3", executor_cfg); + ASSERT_STREQ("value3", value); + value = get_section_value("key1", executor_cfg); + ASSERT_STREQ("value4", value); + ASSERT_EQ(2, executor_cfg->size); + ASSERT_EQ(2, mixed_config_format.size); + ASSERT_STREQ("", mixed_config_format.sections[0]->name); + ASSERT_STREQ("section-1", mixed_config_format.sections[1]->name); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_main.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_main.cc new file mode 100644 index 00000000000..d59a3f22a13 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_main.cc @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include
+#include + +FILE* ERRORFILE = stderr; +FILE* LOGFILE = stdout; + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc new file mode 100644 index 00000000000..2ec7b2a09c7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +extern "C" { +#include "util.h" +} + +namespace ContainerExecutor { + + class TestUtil : public ::testing::Test { + protected: + virtual void SetUp() { + } + + virtual void TearDown() { + } + }; + + TEST_F(TestUtil, test_split_delimiter) { + std::string str = "1,2,3,4,5,6,7,8,9,10,11"; + char *split_string = (char *) calloc(str.length() + 1, sizeof(char)); + strncpy(split_string, str.c_str(), str.length()); + char **splits = split_delimiter(split_string, ","); + ASSERT_TRUE(splits != NULL); + int count = 0; + while(splits[count] != NULL) { + ++count; + } + ASSERT_EQ(11, count); + for(int i = 1; i < count; ++i) { + std::ostringstream oss; + oss << i; + ASSERT_STREQ(oss.str().c_str(), splits[i-1]); + } + ASSERT_EQ(NULL, splits[count]); + free_values(splits); + + split_string = (char *) calloc(str.length() + 1, sizeof(char)); + strncpy(split_string, str.c_str(), str.length()); + splits = split_delimiter(split_string, "%"); + ASSERT_TRUE(splits != NULL); + ASSERT_TRUE(splits[1] == NULL); + ASSERT_STREQ(str.c_str(), splits[0]); + free_values(splits); + + splits = split_delimiter(NULL, ","); + ASSERT_EQ(NULL, splits); + return; + } + + TEST_F(TestUtil, test_split) { + std::string str = "1%2%3%4%5%6%7%8%9%10%11"; + char *split_string = (char *) calloc(str.length() + 1, sizeof(char)); + strncpy(split_string, str.c_str(), str.length()); + char **splits = split(split_string); + int count = 0; + while(splits[count] != NULL) { + ++count; + } + ASSERT_EQ(11, count); + for(int i = 1; i < count; ++i) { + std::ostringstream oss; + oss << i; + ASSERT_STREQ(oss.str().c_str(), splits[i-1]); + } + ASSERT_EQ(NULL, splits[count]); + free_values(splits); + + str = "1,2,3,4,5,6,7,8,9,10,11"; + split_string = (char *) calloc(str.length() + 1, sizeof(char)); + strncpy(split_string, str.c_str(), str.length()); + splits = split(split_string); + ASSERT_TRUE(splits != NULL); + ASSERT_TRUE(splits[1] == NULL); + ASSERT_STREQ(str.c_str(), splits[0]); + return; + } + + TEST_F(TestUtil, test_trim) { + char* trimmed = NULL; + + // Check NULL input + ASSERT_EQ(NULL, trim(NULL)); + + // Check empty input + trimmed = trim(""); + ASSERT_STREQ("", trimmed); + free(trimmed); + + // Check single space input + trimmed = trim(" "); + ASSERT_STREQ("", trimmed); + free(trimmed); + + // Check multi space input + trimmed = trim(" "); + ASSERT_STREQ("", trimmed); + free(trimmed); + + // Check both side trim input + trimmed = trim(" foo "); + ASSERT_STREQ("foo", trimmed); + free(trimmed); + + // Check left side trim input + trimmed = trim("foo "); + ASSERT_STREQ("foo", trimmed); + free(trimmed); + + // Check right side trim input + trimmed = trim(" foo"); + ASSERT_STREQ("foo", trimmed); + free(trimmed); + + // Check no trim input + trimmed = trim("foo"); + ASSERT_STREQ("foo", trimmed); + free(trimmed); + } +}