Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6033. Add support for sections in container-executor configuration file. (Varun Vasudev via wandga) 

Change-Id: Ibc6d2a959debe5d8ff2b51504149742449d1f1da
This commit is contained in:
Wangda Tan 2017-08-09 10:51:29 -07:00
parent 63cfcb90ac
commit ec694145cf
17 changed files with 1675 additions and 347 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
View File

@ -215,6 +215,44 @@
<results>${project.build.directory}/native-results</results>
</configuration>
</execution>
<execution>
<id>cetest</id>
<goals><goal>cmake-test</goal></goals>
<phase>test</phase>
<configuration>
<!-- this should match the xml name without the TEST-part down below -->
<testName>cetest</testName>
<workingDirectory>${project.build.directory}/native/test</workingDirectory>
<source>${basedir}/src</source>
<binary>${project.build.directory}/native/test/cetest</binary>
<args>
<arg>--gtest_filter=-Perf.</arg>
<arg>--gtest_output=xml:${project.build.directory}/surefire-reports/TEST-cetest.xml</arg>
</args>
<results>${project.build.directory}/surefire-reports</results>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>make</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<copy todir="${project.build.directory}/native/test/"
overwrite="true">
<fileset dir="${basedir}/src/main/native/container-executor/resources/test" />
</copy>
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
View File

@ -19,6 +19,9 @@ cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common)
include(HadoopCommon)
# Set gtest path
set(GTEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common/src/main/native/gtest)
# determine if container-executor.conf.dir is an absolute
# path in case the OS we're compiling on doesn't have
# a hook in get_executable. We'll use this define
@ -80,12 +83,20 @@ endfunction()
include_directories(
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_BINARY_DIR}
${GTEST_SRC_DIR}/include
main/native/container-executor
main/native/container-executor/impl
)
# add gtest as system library to suppress gcc warnings
include_directories(SYSTEM ${GTEST_SRC_DIR}/include)
configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
add_library(gtest ${GTEST_SRC_DIR}/gtest-all.cc)
set_target_properties(gtest PROPERTIES COMPILE_FLAGS "-w")
add_library(container
main/native/container-executor/impl/util.c
main/native/container-executor/impl/configuration.c
main/native/container-executor/impl/container-executor.c
main/native/container-executor/impl/get_executable.c
@ -95,9 +106,11 @@ add_library(container
add_executable(container-executor
main/native/container-executor/impl/main.c
)
target_link_libraries(container-executor
container
)
output_directory(container-executor target/usr/local/bin)
add_executable(test-container-executor
@ -107,3 +120,12 @@ target_link_libraries(test-container-executor
container ${EXTRA_LIBS}
)
output_directory(test-container-executor target/usr/local/bin)
# unit tests for container executor
add_executable(cetest
main/native/container-executor/impl/util.c
main/native/container-executor/test/test_configuration.cc
main/native/container-executor/test/test_main.cc
main/native/container-executor/test/test_util.cc)
target_link_libraries(cetest gtest)
output_directory(cetest test)

718
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
View File

@ -20,35 +20,55 @@
#include <libgen.h>
#include "configuration.h"
#include "container-executor.h"
#include "util.h"
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include <errno.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <limits.h>
#include <ctype.h>
#define MAX_SIZE 10
static const char COMMENT_BEGIN_CHAR = '#';
static const char SECTION_LINE_BEGIN_CHAR = '[';
static const char SECTION_LINE_END_CHAR = ']';
//clean up method for freeing section
void free_section(struct section *section) {
int i = 0;
for (i = 0; i < section->size; i++) {
if (section->kv_pairs[i]->key != NULL) {
free((void *) section->kv_pairs[i]->key);
}
if (section->kv_pairs[i]->value != NULL) {
free((void *) section->kv_pairs[i]->value);
}
free(section->kv_pairs[i]);
}
if (section->kv_pairs) {
free(section->kv_pairs);
section->kv_pairs = NULL;
}
if (section->name) {
free(section->name);
section->name = NULL;
}
section->size = 0;
}
//clean up method for freeing configuration
void free_configurations(struct configuration *cfg) {
void free_configuration(struct configuration *cfg) {
int i = 0;
for (i = 0; i < cfg->size; i++) {
if (cfg->confdetails[i]->key != NULL) {
free((void *)cfg->confdetails[i]->key);
if (cfg->sections[i] != NULL) {
free_section(cfg->sections[i]);
}
if (cfg->confdetails[i]->value != NULL) {
free((void *)cfg->confdetails[i]->value);
}
free(cfg->confdetails[i]);
}
if (cfg->size > 0) {
free(cfg->confdetails);
if (cfg->sections) {
free(cfg->sections);
}
cfg->size = 0;
}
@ -65,13 +85,13 @@ static int is_only_root_writable(const char *file) {
}
if (file_stat.st_uid != 0) {
fprintf(ERRORFILE, "File %s must be owned by root, but is owned by %" PRId64 "\n",
file, (int64_t)file_stat.st_uid);
file, (int64_t) file_stat.st_uid);
return 0;
}
if ((file_stat.st_mode & (S_IWGRP | S_IWOTH)) != 0) {
fprintf(ERRORFILE,
"File %s must not be world or group writable, but is %03lo\n",
file, (unsigned long)file_stat.st_mode & (~S_IFMT));
"File %s must not be world or group writable, but is %03lo\n",
file, (unsigned long) file_stat.st_mode & (~S_IFMT));
return 0;
}
return 1;
@ -82,9 +102,9 @@ static int is_only_root_writable(const char *file) {
*
* NOTE: relative path names are resolved relative to the second argument not getwd(3)
*/
char *resolve_config_path(const char* file_name, const char *root) {
char *resolve_config_path(const char *file_name, const char *root) {
const char *real_fname = NULL;
char buffer[EXECUTOR_PATH_MAX*2 + 1];
char buffer[EXECUTOR_PATH_MAX * 2 + 1];
if (file_name[0] == '/') {
real_fname = file_name;
@ -96,7 +116,7 @@ char *resolve_config_path(const char* file_name, const char *root) {
#ifdef HAVE_CANONICALIZE_FILE_NAME
char * ret = (real_fname == NULL) ? NULL : canonicalize_file_name(real_fname);
#else
char * ret = (real_fname == NULL) ? NULL : realpath(real_fname, NULL);
char *ret = (real_fname == NULL) ? NULL : realpath(real_fname, NULL);
#endif
#ifdef DEBUG
fprintf(stderr,"ret = %s\n", ret);
@ -112,10 +132,19 @@ char *resolve_config_path(const char* file_name, const char *root) {
* configuration and potentially cause damage.
* returns 0 if permissions are ok
*/
int check_configuration_permissions(const char* file_name) {
int check_configuration_permissions(const char *file_name) {
if (!file_name) {
return -1;
}
// copy the input so that we can modify it with dirname
char* dir = strdup(file_name);
char* buffer = dir;
char *dir = strdup(file_name);
if (!dir) {
fprintf(stderr, "Failed to make a copy of filename in %s.\n", __func__);
return -1;
}
char *buffer = dir;
do {
if (!is_only_root_writable(dir)) {
free(buffer);
@ -128,167 +157,396 @@ int check_configuration_permissions(const char* file_name) {
}
/**
* Trim whitespace from beginning and end.
*/
char* trim(char* input)
{
char *val_begin;
char *val_end;
char *ret;
* Read a line from the the config file and return it without the newline.
* The caller must free the memory allocated.
*/
static char *read_config_line(FILE *conf_file) {
char *line = NULL;
size_t linesize = 100000;
ssize_t size_read = 0;
size_t eol = 0;
if (input == NULL) {
return NULL;
line = (char *) malloc(linesize);
if (line == NULL) {
fprintf(ERRORFILE, "malloc failed while reading configuration file.\n");
exit(OUT_OF_MEMORY);
}
size_read = getline(&line, &linesize, conf_file);
//feof returns true only after we read past EOF.
//so a file with no new line, at last can reach this place
//if size_read returns negative check for eof condition
if (size_read == -1) {
free(line);
line = NULL;
if (!feof(conf_file)) {
fprintf(ERRORFILE, "Line read returned -1 without eof\n");
exit(INVALID_CONFIG_FILE);
}
val_begin = input;
val_end = input + strlen(input);
while (val_begin < val_end && isspace(*val_begin))
val_begin++;
while (val_end > val_begin && isspace(*(val_end - 1)))
val_end--;
ret = (char *) malloc(
sizeof(char) * (val_end - val_begin + 1));
if (ret == NULL) {
fprintf(ERRORFILE, "Allocation error\n");
exit(OUT_OF_MEMORY);
} else {
eol = strlen(line) - 1;
if (line[eol] == '\n') {
//trim the ending new line
line[eol] = '\0';
}
strncpy(ret, val_begin, val_end - val_begin);
ret[val_end - val_begin] = '\0';
return ret;
}
return line;
}
void read_config(const char* file_name, struct configuration *cfg) {
FILE *conf_file;
char *line;
char *equaltok;
char *temp_equaltok;
size_t linesize = 1000;
int size_read = 0;
if (file_name == NULL) {
fprintf(ERRORFILE, "Null configuration filename passed in\n");
exit(INVALID_CONFIG_FILE);
/**
* Return if the given line is a comment line.
*
* @param line the line to check
*
* @return 1 if the line is a comment line, 0 otherwise
*/
static int is_comment_line(const char *line) {
if (line != NULL) {
return (line[0] == COMMENT_BEGIN_CHAR);
}
return 0;
}
#ifdef DEBUG
fprintf(LOGFILE, "read_config :Conf file name is : %s \n", file_name);
#endif
//allocate space for ten configuration items.
cfg->confdetails = (struct confentry **) malloc(sizeof(struct confentry *)
* MAX_SIZE);
cfg->size = 0;
conf_file = fopen(file_name, "r");
if (conf_file == NULL) {
fprintf(ERRORFILE, "Invalid conf file provided : %s \n", file_name);
exit(INVALID_CONFIG_FILE);
/**
* Return if the given line is a section start line.
*
* @param line the line to check
*
* @return 1 if the line is a section start line, 0 otherwise
*/
static int is_section_start_line(const char *line) {
size_t len = 0;
if (line != NULL) {
len = strlen(line) - 1;
return (line[0] == SECTION_LINE_BEGIN_CHAR
&& line[len] == SECTION_LINE_END_CHAR);
}
while(!feof(conf_file)) {
line = (char *) malloc(linesize);
if(line == NULL) {
fprintf(ERRORFILE, "malloc failed while reading configuration file.\n");
return 0;
}
/**
* Return the name of the section from the given section start line. The
* caller must free the memory used.
*
* @param line the line to extract the section name from
*
* @return string with the name of the section, NULL otherwise
*/
static char *get_section_name(const char *line) {
char *name = NULL;
size_t len;
if (is_section_start_line(line)) {
// length of the name is the line - 2(to account for '[' and ']')
len = strlen(line) - 2;
name = (char *) malloc(len + 1);
if (name == NULL) {
fprintf(ERRORFILE, "malloc failed while reading section name.\n");
exit(OUT_OF_MEMORY);
}
size_read = getline(&line,&linesize,conf_file);
strncpy(name, line + sizeof(char), len);
name[len] = '\0';
}
return name;
}
//feof returns true only after we read past EOF.
//so a file with no new line, at last can reach this place
//if size_read returns negative check for eof condition
if (size_read == -1) {
free(line);
if(!feof(conf_file)){
exit(INVALID_CONFIG_FILE);
} else {
break;
}
/**
* Read an entry for the section from the line. Function returns 0 if an entry
* was found, non-zero otherwise. Return values less than 0 indicate an error
* with the config file.
*
* @param line the line to read the entry from
* @param section the struct to read the entry into
*
* @return 0 if an entry was found
* <0 for config file errors
* >0 for issues such as empty line
*
*/
static int read_section_entry(const char *line, struct section *section) {
char *equaltok;
char *temp_equaltok;
const char *splitter = "=";
char *buffer;
size_t len = 0;
if (line == NULL || section == NULL) {
fprintf(ERRORFILE, "NULL params passed to read_section_entry");
return -1;
}
len = strlen(line);
if (len == 0) {
return 1;
}
if ((section->size) % MAX_SIZE == 0) {
section->kv_pairs = (struct kv_pair **) realloc(
section->kv_pairs,
sizeof(struct kv_pair *) * (MAX_SIZE + section->size));
if (section->kv_pairs == NULL) {
fprintf(ERRORFILE,
"Failed re-allocating memory for configuration items\n");
exit(OUT_OF_MEMORY);
}
}
buffer = strdup(line);
if (!buffer) {
fprintf(ERRORFILE, "Failed to allocating memory for line, %s\n", __func__);
exit(OUT_OF_MEMORY);
}
//tokenize first to get key and list of values.
//if no equals is found ignore this line, can be an empty line also
equaltok = strtok_r(buffer, splitter, &temp_equaltok);
if (equaltok == NULL) {
fprintf(ERRORFILE, "Error with line '%s', no '=' found\n", buffer);
exit(INVALID_CONFIG_FILE);
}
section->kv_pairs[section->size] = (struct kv_pair *) malloc(
sizeof(struct kv_pair));
if (section->kv_pairs[section->size] == NULL) {
fprintf(ERRORFILE, "Failed allocating memory for single section item\n");
exit(OUT_OF_MEMORY);
}
memset(section->kv_pairs[section->size], 0,
sizeof(struct kv_pair));
section->kv_pairs[section->size]->key = trim(equaltok);
equaltok = strtok_r(NULL, splitter, &temp_equaltok);
if (equaltok == NULL) {
// this can happen because no value was set
// e.g. banned.users=#this is a comment
int has_values = 1;
if (strstr(line, splitter) == NULL) {
fprintf(ERRORFILE, "configuration tokenization failed, error with line %s\n", line);
has_values = 0;
}
// It is not a valid line, free memory.
free((void *) section->kv_pairs[section->size]->key);
free((void *) section->kv_pairs[section->size]);
section->kv_pairs[section->size] = NULL;
free(buffer);
// Return -1 when no values
if (!has_values) {
return -1;
}
// Return 2 for comments
return 2;
}
#ifdef DEBUG
fprintf(LOGFILE, "read_config : Adding conf value : %s \n", equaltok);
#endif
section->kv_pairs[section->size]->value = trim(equaltok);
section->size++;
free(buffer);
return 0;
}
/**
* Remove any trailing comment from the supplied line. Function modifies the
* argument provided.
*
* @param line the line from which to remove the comment
*/
static void trim_comment(char *line) {
char *begin_comment = NULL;
if (line != NULL) {
begin_comment = strchr(line, COMMENT_BEGIN_CHAR);
if (begin_comment != NULL) {
*begin_comment = '\0';
}
}
}
/**
* Allocate a section struct and initialize it. The memory must be freed by
* the caller. Function calls exit if any error occurs.
*
* @return pointer to the allocated section struct
*
*/
static struct section *allocate_section() {
struct section *section = (struct section *) malloc(sizeof(struct section));
if (section == NULL) {
fprintf(ERRORFILE, "malloc failed while allocating section.\n");
exit(OUT_OF_MEMORY);
}
section->name = NULL;
section->kv_pairs = NULL;
section->size = 0;
return section;
}
/**
* Populate the given section struct with fields from the config file.
*
* @param conf_file the file to read from
* @param section pointer to the section struct to populate
*
*/
static void populate_section_fields(FILE *conf_file, struct section *section) {
char *line;
long int offset = 0;
while (!feof(conf_file)) {
offset = ftell(conf_file);
line = read_config_line(conf_file);
if (line != NULL) {
if (!is_comment_line(line)) {
trim_comment(line);
if (!is_section_start_line(line)) {
if (section->name != NULL) {
if (read_section_entry(line, section) < 0) {
fprintf(ERRORFILE, "Error parsing line %s", line);
exit(INVALID_CONFIG_FILE);
}
} else {
fprintf(ERRORFILE, "Line '%s' doesn't belong to a section\n",
line);
exit(INVALID_CONFIG_FILE);
}
} else {
if (section->name == NULL) {
section->name = get_section_name(line);
if (strlen(section->name) == 0) {
fprintf(ERRORFILE, "Empty section name");
exit(INVALID_CONFIG_FILE);
}
} else {
// we've reached the next section
fseek(conf_file, offset, SEEK_SET);
free(line);
return;
}
}
}
free(line);
}
}
}
/**
* Read the section current section from the conf file. Section start is
* marked by lines of the form '[section-name]' and continue till the next
* section.
*/
static struct section *read_section(FILE *conf_file) {
struct section *section = allocate_section();
populate_section_fields(conf_file, section);
if (section->name == NULL) {
free_section(section);
section = NULL;
}
return section;
}
/**
* Merge two sections and free the second one after the merge, if desired.
* @param section1 the first section
* @param section2 the second section
* @param free_second_section free the second section if set
*/
static void merge_sections(struct section *section1, struct section *section2, const int free_second_section) {
int i = 0;
section1->kv_pairs = (struct kv_pair **) realloc(
section1->kv_pairs,
sizeof(struct kv_pair *) * (section1->size + section2->size));
if (section1->kv_pairs == NULL) {
fprintf(ERRORFILE,
"Failed re-allocating memory for configuration items\n");
exit(OUT_OF_MEMORY);
}
for (i = 0; i < section2->size; ++i) {
section1->kv_pairs[section1->size + i] = section2->kv_pairs[i];
}
section1->size += section2->size;
if (free_second_section) {
free(section2->name);
memset(section2, 0, sizeof(*section2));
free(section2);
}
}
int read_config(const char *file_path, struct configuration *cfg) {
FILE *conf_file;
if (file_path == NULL) {
fprintf(ERRORFILE, "Null configuration filename passed in\n");
return INVALID_CONFIG_FILE;
}
#ifdef DEBUG
fprintf(LOGFILE, "read_config :Conf file name is : %s \n", file_path);
#endif
cfg->size = 0;
conf_file = fopen(file_path, "r");
if (conf_file == NULL) {
fprintf(ERRORFILE, "Invalid conf file provided, unable to open file"
" : %s \n", file_path);
return (INVALID_CONFIG_FILE);
}
cfg->sections = (struct section **) malloc(
sizeof(struct section *) * MAX_SIZE);
if (!cfg->sections) {
fprintf(ERRORFILE,
"Failed to allocate memory for configuration sections\n");
exit(OUT_OF_MEMORY);
}
// populate any entries in the older format(no sections)
cfg->sections[cfg->size] = allocate_section();
cfg->sections[cfg->size]->name = strdup("");
populate_section_fields(conf_file, cfg->sections[cfg->size]);
if (cfg->sections[cfg->size]) {
if (cfg->sections[cfg->size]->size) {
cfg->size++;
} else {
free_section(cfg->sections[cfg->size]);
}
}
// populate entries in the sections format
while (!feof(conf_file)) {
cfg->sections[cfg->size] = NULL;
struct section *new_section = read_section(conf_file);
if (new_section != NULL) {
struct section *existing_section =
get_configuration_section(new_section->name, cfg);
if (existing_section != NULL) {
merge_sections((struct section *) existing_section, new_section, 1);
} else {
cfg->sections[cfg->size] = new_section;
}
}
// Check if we need to expand memory for sections.
if (cfg->sections[cfg->size]) {
if ((cfg->size + 1) % MAX_SIZE == 0) {
cfg->sections = (struct section **) realloc(cfg->sections,
sizeof(struct sections *) * (MAX_SIZE + cfg->size));
if (cfg->sections == NULL) {
fprintf(ERRORFILE,
"Failed re-allocating memory for configuration items\n");
exit(OUT_OF_MEMORY);
}
}
cfg->size++;
}
int eol = strlen(line) - 1;
if(line[eol] == '\n') {
//trim the ending new line
line[eol] = '\0';
}
//comment line
if(line[0] == '#') {
free(line);
continue;
}
//tokenize first to get key and list of values.
//if no equals is found ignore this line, can be an empty line also
equaltok = strtok_r(line, "=", &temp_equaltok);
if(equaltok == NULL) {
free(line);
continue;
}
cfg->confdetails[cfg->size] = (struct confentry *) malloc(
sizeof(struct confentry));
if(cfg->confdetails[cfg->size] == NULL) {
fprintf(LOGFILE,
"Failed allocating memory for single configuration item\n");
goto cleanup;
}
#ifdef DEBUG
fprintf(LOGFILE, "read_config : Adding conf key : %s \n", equaltok);
#endif
memset(cfg->confdetails[cfg->size], 0, sizeof(struct confentry));
cfg->confdetails[cfg->size]->key = trim(equaltok);
equaltok = strtok_r(NULL, "=", &temp_equaltok);
if (equaltok == NULL) {
fprintf(LOGFILE, "configuration tokenization failed \n");
goto cleanup;
}
//means value is commented so don't store the key
if(equaltok[0] == '#') {
free(line);
free((void *)cfg->confdetails[cfg->size]->key);
free(cfg->confdetails[cfg->size]);
continue;
}
#ifdef DEBUG
fprintf(LOGFILE, "read_config : Adding conf value : %s \n", equaltok);
#endif
cfg->confdetails[cfg->size]->value = trim(equaltok);
if((cfg->size + 1) % MAX_SIZE == 0) {
cfg->confdetails = (struct confentry **) realloc(cfg->confdetails,
sizeof(struct confentry **) * (MAX_SIZE + cfg->size));
if (cfg->confdetails == NULL) {
fprintf(LOGFILE,
"Failed re-allocating memory for configuration items\n");
goto cleanup;
}
}
if(cfg->confdetails[cfg->size]) {
cfg->size++;
}
free(line);
}
//close the file
fclose(conf_file);
if (cfg->size == 0) {
fprintf(ERRORFILE, "Invalid configuration provided in %s\n", file_name);
exit(INVALID_CONFIG_FILE);
free_configuration(cfg);
fprintf(ERRORFILE, "Invalid configuration provided in %s\n", file_path);
return INVALID_CONFIG_FILE;
}
//clean up allocated file name
return;
//free spaces alloced.
cleanup:
if (line != NULL) {
free(line);
}
fclose(conf_file);
free_configurations(cfg);
return;
return 0;
}
/*
@ -297,11 +555,14 @@ void read_config(const char* file_name, struct configuration *cfg) {
* array, next time onwards used the populated array.
*
*/
char * get_value(const char* key, struct configuration *cfg) {
char *get_section_value(const char *key, const struct section *section) {
int count;
for (count = 0; count < cfg->size; count++) {
if (strcmp(cfg->confdetails[count]->key, key) == 0) {
return strdup(cfg->confdetails[count]->value);
if (key == NULL || section == NULL) {
return NULL;
}
for (count = 0; count < section->size; count++) {
if (strcmp(section->kv_pairs[count]->key, key) == 0) {
return strdup(section->kv_pairs[count]->value);
}
}
return NULL;
@ -311,61 +572,80 @@ char * get_value(const char* key, struct configuration *cfg) {
* Function to return an array of values for a key.
* Value delimiter is assumed to be a ','.
*/
char ** get_values(const char * key, struct configuration *cfg) {
char *value = get_value(key, cfg);
return extract_values_delim(value, ",");
char **get_section_values(const char *key, const struct section *cfg) {
return get_section_values_delimiter(key, cfg, ",");
}
/**
* Function to return an array of values for a key, using the specified
delimiter.
*/
char ** get_values_delim(const char * key, struct configuration *cfg,
const char *delim) {
char *value = get_value(key, cfg);
return extract_values_delim(value, delim);
char **get_section_values_delimiter(const char *key, const struct section *cfg,
const char *delim) {
if (key == NULL || cfg == NULL || delim == NULL) {
return NULL;
}
char *value = get_section_value(key, cfg);
char **split_values = split_delimiter(value, delim);
if (value) {
free(value);
}
return split_values;
}
char ** extract_values_delim(char *value, const char *delim) {
char ** toPass = NULL;
char *tempTok = NULL;
char *tempstr = NULL;
int size = 0;
int toPassSize = MAX_SIZE;
//first allocate any array of 10
if(value != NULL) {
toPass = (char **) malloc(sizeof(char *) * toPassSize);
tempTok = strtok_r((char *)value, delim, &tempstr);
while (tempTok != NULL) {
toPass[size++] = tempTok;
if(size == toPassSize) {
toPassSize += MAX_SIZE;
toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize));
}
tempTok = strtok_r(NULL, delim, &tempstr);
char *get_configuration_value(const char *key, const char *section,
const struct configuration *cfg) {
const struct section *section_ptr;
if (key == NULL || section == NULL || cfg == NULL) {
return NULL;
}
section_ptr = get_configuration_section(section, cfg);
if (section_ptr != NULL) {
return get_section_value(key, section_ptr);
}
return NULL;
}
char **get_configuration_values(const char *key, const char *section,
const struct configuration *cfg) {
const struct section *section_ptr;
if (key == NULL || section == NULL || cfg == NULL) {
return NULL;
}
section_ptr = get_configuration_section(section, cfg);
if (section_ptr != NULL) {
return get_section_values(key, section_ptr);
}
return NULL;
}
char **get_configuration_values_delimiter(const char *key, const char *section,
const struct configuration *cfg, const char *delim) {
const struct section *section_ptr;
if (key == NULL || section == NULL || cfg == NULL || delim == NULL) {
return NULL;
}
section_ptr = get_configuration_section(section, cfg);
if (section_ptr != NULL) {
return get_section_values_delimiter(key, section_ptr, delim);
}
return NULL;
}
struct section *get_configuration_section(const char *section,
const struct configuration *cfg) {
int i = 0;
if (cfg == NULL || section == NULL) {
return NULL;
}
for (i = 0; i < cfg->size; ++i) {
if (strcmp(cfg->sections[i]->name, section) == 0) {
return cfg->sections[i];
}
}
if (toPass != NULL) {
toPass[size] = NULL;
}
return toPass;
}
/**
* Extracts array of values from the '%' separated list of values.
*/
char ** extract_values(char *value) {
return extract_values_delim(value, "%");
}
// free an entry set of values
void free_values(char** values) {
if (*values != NULL) {
free(*values);
}
if (values != NULL) {
free(values);
}
return NULL;
}
/**
@ -376,12 +656,12 @@ int get_kv_key(const char *input, char *out, size_t out_len) {
if (input == NULL)
return -EINVAL;
char *split = strchr(input, '=');
const char *split = strchr(input, '=');
if (split == NULL)
return -EINVAL;
int key_len = split - input;
unsigned long key_len = split - input;
if (out_len < (key_len + 1) || out == NULL)
return -ENAMETOOLONG;
@ -400,13 +680,13 @@ int get_kv_value(const char *input, char *out, size_t out_len) {
if (input == NULL)
return -EINVAL;
char *split = strchr(input, '=');
const char *split = strchr(input, '=');
if (split == NULL)
return -EINVAL;
split++; // advance past '=' to the value
int val_len = (input + strlen(input)) - split;
unsigned long val_len = (input + strlen(input)) - split;
if (out_len < (val_len + 1) || out == NULL)
return -ENAMETOOLONG;

188
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
View File

@ -16,6 +16,9 @@
* limitations under the License.
*/
#ifndef __YARN_CONTAINER_EXECUTOR_CONFIG_H__
#define __YARN_CONTAINER_EXECUTOR_CONFIG_H__
#ifdef __FreeBSD__
#define _WITH_GETLINE
#endif
@ -23,62 +26,160 @@
#include <stddef.h>
/** Define a platform-independent constant instead of using PATH_MAX */
#define EXECUTOR_PATH_MAX 4096
/**
* Ensure that the configuration file and all of the containing directories
* are only writable by root. Otherwise, an attacker can change the
* configuration and potentially cause damage.
* returns 0 if permissions are ok
*/
int check_configuration_permissions(const char* file_name);
/**
* Return a string with the configuration file path name resolved via realpath(3)
*
* NOTE: relative path names are resolved relative to the second argument not getwd(3)
*/
char *resolve_config_path(const char* file_name, const char *root);
// Config data structures.
struct confentry {
// Configuration data structures.
struct kv_pair {
const char *key;
const char *value;
};
struct configuration {
struct section {
int size;
struct confentry **confdetails;
char *name;
struct kv_pair **kv_pairs;
};
// read the given configuration file into the specified config struct.
void read_config(const char* config_file, struct configuration *cfg);
//method exposed to get the configurations
char *get_value(const char* key, struct configuration *cfg);
//function to return array of values pointing to the key. Values are
//comma seperated strings.
char ** get_values(const char* key, struct configuration *cfg);
struct configuration {
int size;
struct section **sections;
};
/**
* Function to return an array of values for a key, using the specified
delimiter.
* Function to ensure that the configuration file and all of the containing
* directories are only writable by root. Otherwise, an attacker can change
* the configuration and potentially cause damage.
*
* @param file_name name of the config file
*
* @returns 0 if permissions are correct, non-zero on error
*/
char ** get_values_delim(const char * key, struct configuration *cfg,
int check_configuration_permissions(const char *file_name);
/**
* Return a string with the configuration file path name resolved via
* realpath(3). Relative path names are resolved relative to the second
* argument and not getwd(3). It's up to the caller to free the returned
* value.
*
* @param file_name name of the config file
* @param root the path against which relative path names are to be resolved
*
* @returns the resolved configuration file path
*/
char* resolve_config_path(const char *file_name, const char *root);
/**
* Read the given configuration file into the specified configuration struct.
* It's the responsibility of the caller to call free_configurations to free
* the allocated memory. The function will check to ensure that the
* configuration file has the appropriate owner and permissions.
*
* @param file_path name of the configuration file to be read
* @param cfg the configuration structure to be filled.
*
* @return 0 on success, non-zero if there was an error
*/
int read_config(const char *file_path, struct configuration *cfg);
/**
* Get the value for a key in the specified section. It's up to the caller to
* free the memory used for storing the return value.
*
* @param key key the name of the key
* @param section the section to be looked up
*
* @return pointer to the value if the key was found, null otherwise
*/
char* get_section_value(const char *key, const struct section *section);
/**
* Function to get the values for a given key in the specified section.
* The value is split by ",". It's up to the caller to free the memory used
* for storing the return values.
*
* @param key the key to be looked up
* @param section the section to be looked up
*
* @return array of values, null if the key was not found
*/
char** get_section_values(const char *key, const struct section *section);
/**
* Function to get the values for a given key in the specified section.
* The value is split by the specified delimiter. It's up to the caller to
* free the memory used for storing the return values.
*
* @param key the key to be looked up
* @param section the section to be looked up
* @param delimiter the delimiter to be used to split the value
*
* @return array of values, null if the key was not found
*/
char** get_section_values_delimiter(const char *key, const struct section *section,
const char *delim);
// Extracts array of values from the comma separated list of values.
char ** extract_values(char *value);
/**
* Get the value for a key in the specified section in the specified
* configuration. It's up to the caller to free the memory used for storing
* the return value.
*
* @param key key the name of the key
* @param section the name section to be looked up
* @param cfg the configuration to be used
*
* @return pointer to the value if the key was found, null otherwise
*/
char* get_configuration_value(const char *key, const char* section,
const struct configuration *cfg);
char ** extract_values_delim(char *value, const char *delim);
/**
* Function to get the values for a given key in the specified section in the
* specified configuration. The value is split by ",". It's up to the caller to
* free the memory used for storing the return values.
*
* @param key the key to be looked up
* @param section the name of the section to be looked up
* @param cfg the configuration to be looked up
*
* @return array of values, null if the key was not found
*/
char** get_configuration_values(const char *key, const char* section,
const struct configuration *cfg);
// free the memory returned by get_values
void free_values(char** values);
/**
* Function to get the values for a given key in the specified section in the
* specified configuration. The value is split by the specified delimiter.
* It's up to the caller to free the memory used for storing the return values.
*
* @param key the key to be looked up
* @param section the name of the section to be looked up
* @param cfg the section to be looked up
* @param delimiter the delimiter to be used to split the value
*
* @return array of values, null if the key was not found
*/
char** get_configuration_values_delimiter(const char *key, const char* section,
const struct configuration *cfg, const char *delimiter);
//method to free allocated configuration
void free_configurations(struct configuration *cfg);
/**
* Function to retrieve the specified section from the configuration.
*
* @param section the name of the section to retrieve
* @param cfg the configuration structure to use
*
* @return pointer to section struct containing details of the section
* null on error
*/
struct section* get_configuration_section(const char *section,
const struct configuration *cfg);
/**
* Method to free an allocated config struct.
*
* @param cfg pointer to the structure to free
*/
void free_configuration(struct configuration *cfg);
/**
* If str is a string of the form key=val, find 'key'
@ -106,11 +207,4 @@ int get_kv_key(const char *input, char *out, size_t out_len);
*/
int get_kv_value(const char *input, char *out, size_t out_len);
/**
* Trim whitespace from beginning and end.
*
* @param input Input string that needs to be trimmed
*
* @return the trimmed string allocated with malloc. I has to be freed by the caller
*/
char* trim(char* input);
#endif

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
View File

@ -19,6 +19,8 @@
#include "configuration.h"
#include "container-executor.h"
#include "utils/string-utils.h"
#include "util.h"
#include "config.h"
#include <inttypes.h>
#include <libgen.h>
@ -43,8 +45,6 @@
#include <getopt.h>
#include <regex.h>
#include "config.h"
#ifndef HAVE_FCHMODAT
#include "compat/fchmodat.h"
#endif
@ -92,7 +92,8 @@ FILE* ERRORFILE = NULL;
static uid_t nm_uid = -1;
static gid_t nm_gid = -1;
struct configuration executor_cfg = {.size=0, .confdetails=NULL};
struct configuration CFG = {.size=0, .sections=NULL};
struct section executor_cfg = {.size=0, .kv_pairs=NULL};
char *concatenate(char *concat_pattern, char *return_path_name,
int numArgs, ...);
@ -103,18 +104,25 @@ void set_nm_uid(uid_t user, gid_t group) {
}
//function used to load the configurations present in the secure config
void read_executor_config(const char* file_name) {
read_config(file_name, &executor_cfg);
void read_executor_config(const char *file_name) {
const struct section *tmp = NULL;
int ret = read_config(file_name, &CFG);
if (ret == 0) {
tmp = get_configuration_section("", &CFG);
if (tmp != NULL) {
executor_cfg = *tmp;
}
}
}
//function used to free executor configuration data
void free_executor_configurations() {
free_configurations(&executor_cfg);
free_configuration(&CFG);
}
//Lookup nodemanager group from container executor configuration.
char *get_nodemanager_group() {
return get_value(NM_GROUP_KEY, &executor_cfg);
return get_section_value(NM_GROUP_KEY, &executor_cfg);
}
int check_executor_permissions(char *executable_file) {
@ -431,8 +439,8 @@ int change_user(uid_t user, gid_t group) {
}
int is_feature_enabled(const char* feature_key, int default_value,
struct configuration *cfg) {
char *enabled_str = get_value(feature_key, cfg);
struct section *cfg) {
char *enabled_str = get_section_value(feature_key, cfg);
int enabled = default_value;
if (enabled_str != NULL) {
@ -753,7 +761,7 @@ static struct passwd* get_user_info(const char* user) {
}
int is_whitelisted(const char *user) {
char **whitelist = get_values(ALLOWED_SYSTEM_USERS_KEY, &executor_cfg);
char **whitelist = get_section_values(ALLOWED_SYSTEM_USERS_KEY, &executor_cfg);
char **users = whitelist;
if (whitelist != NULL) {
for(; *users; ++users) {
@ -781,7 +789,7 @@ struct passwd* check_user(const char *user) {
fflush(LOGFILE);
return NULL;
}
char *min_uid_str = get_value(MIN_USERID_KEY, &executor_cfg);
char *min_uid_str = get_section_value(MIN_USERID_KEY, &executor_cfg);
int min_uid = DEFAULT_MIN_USERID;
if (min_uid_str != NULL) {
char *end_ptr = NULL;
@ -808,7 +816,7 @@ struct passwd* check_user(const char *user) {
free(user_info);
return NULL;
}
char **banned_users = get_values(BANNED_USERS_KEY, &executor_cfg);
char **banned_users = get_section_values(BANNED_USERS_KEY, &executor_cfg);
banned_users = banned_users == NULL ?
(char**) DEFAULT_BANNED_USERS : banned_users;
char **banned_user = banned_users;
@ -1194,7 +1202,6 @@ char** tokenize_docker_command(const char *input, int *split_counter) {
char *line = (char *)calloc(strlen(input) + 1, sizeof(char));
char **linesplit = (char **) malloc(sizeof(char *));
char *p = NULL;
int c = 0;
*split_counter = 0;
strncpy(line, input, strlen(input));
@ -1408,12 +1415,12 @@ char* parse_docker_command_file(const char* command_file) {
int run_docker(const char *command_file) {
char* docker_command = parse_docker_command_file(command_file);
char* docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg);
char* docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
docker_binary = check_docker_binary(docker_binary);
char* docker_command_with_binary = calloc(sizeof(char), EXECUTOR_PATH_MAX);
snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command);
char **args = extract_values_delim(docker_command_with_binary, " ");
char **args = split_delimiter(docker_command_with_binary, " ");
int exit_code = -1;
if (execvp(docker_binary, args) != 0) {
@ -1574,7 +1581,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
uid_t prev_uid = geteuid();
char *docker_command = parse_docker_command_file(command_file);
char *docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg);
char *docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
docker_binary = check_docker_binary(docker_binary);
fprintf(LOGFILE, "Creating script paths...\n");

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
View File

@ -35,51 +35,6 @@ enum command {
LIST_AS_USER = 5
};
enum errorcodes {
INVALID_ARGUMENT_NUMBER = 1,
//INVALID_USER_NAME 2
INVALID_COMMAND_PROVIDED = 3,
// SUPER_USER_NOT_ALLOWED_TO_RUN_TASKS (NOT USED) 4
INVALID_NM_ROOT_DIRS = 5,
SETUID_OPER_FAILED, //6
UNABLE_TO_EXECUTE_CONTAINER_SCRIPT, //7
UNABLE_TO_SIGNAL_CONTAINER, //8
INVALID_CONTAINER_PID, //9
// ERROR_RESOLVING_FILE_PATH (NOT_USED) 10
// RELATIVE_PATH_COMPONENTS_IN_FILE_PATH (NOT USED) 11
// UNABLE_TO_STAT_FILE (NOT USED) 12
// FILE_NOT_OWNED_BY_ROOT (NOT USED) 13
// PREPARE_CONTAINER_DIRECTORIES_FAILED (NOT USED) 14
// INITIALIZE_CONTAINER_FAILED (NOT USED) 15
// PREPARE_CONTAINER_LOGS_FAILED (NOT USED) 16
// INVALID_LOG_DIR (NOT USED) 17
OUT_OF_MEMORY = 18,
// INITIALIZE_DISTCACHEFILE_FAILED (NOT USED) 19
INITIALIZE_USER_FAILED = 20,
PATH_TO_DELETE_IS_NULL, //21
INVALID_CONTAINER_EXEC_PERMISSIONS, //22
// PREPARE_JOB_LOGS_FAILED (NOT USED) 23
INVALID_CONFIG_FILE = 24,
SETSID_OPER_FAILED = 25,
WRITE_PIDFILE_FAILED = 26,
WRITE_CGROUP_FAILED = 27,
TRAFFIC_CONTROL_EXECUTION_FAILED = 28,
DOCKER_RUN_FAILED = 29,
ERROR_OPENING_DOCKER_FILE = 30,
ERROR_READING_DOCKER_FILE = 31,
FEATURE_DISABLED = 32,
COULD_NOT_CREATE_SCRIPT_COPY = 33,
COULD_NOT_CREATE_CREDENTIALS_FILE = 34,
COULD_NOT_CREATE_WORK_DIRECTORIES = 35,
COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36,
COULD_NOT_CREATE_TMP_DIRECTORIES = 37,
ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38,
ERROR_SANITIZING_DOCKER_COMMAND = 39,
DOCKER_IMAGE_INVALID = 40,
DOCKER_CONTAINER_NAME_INVALID = 41,
ERROR_COMPILING_REGEX = 42
};
enum operations {
CHECK_SETUP = 1,
MOUNT_CGROUPS = 2,
@ -111,11 +66,6 @@ enum operations {
extern struct passwd *user_detail;
// the log file for messages
extern FILE *LOGFILE;
// the log file for error messages
extern FILE *ERRORFILE;
// get the executable's filename
char* get_executable(char *argv0);
@ -276,7 +226,7 @@ int create_validate_dir(const char* npath, mode_t perm, const char* path,
/** Check if a feature is enabled in the specified configuration. */
int is_feature_enabled(const char* feature_key, int default_value,
struct configuration *cfg);
struct section *cfg);
/** Check if tc (traffic control) support is enabled in configuration. */
int is_tc_support_enabled();

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
View File

@ -31,6 +31,7 @@
#include "config.h"
#include "configuration.h"
#include "container-executor.h"
#include "util.h"
#include <errno.h>
#include <stdio.h>

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
View File

@ -19,6 +19,7 @@
#include "config.h"
#include "configuration.h"
#include "container-executor.h"
#include "util.h"
#include <errno.h>
#include <grp.h>
@ -420,7 +421,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
cmd_input.resources_key = resources_key;
cmd_input.resources_value = resources_value;
cmd_input.resources_values = extract_values(resources_value);
cmd_input.resources_values = split(resources_value);
*operation = RUN_AS_USER_LAUNCH_DOCKER_CONTAINER;
return 0;
} else {
@ -471,7 +472,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
cmd_input.resources_key = resources_key;
cmd_input.resources_value = resources_value;
cmd_input.resources_values = extract_values(resources_value);
cmd_input.resources_values = split(resources_value);
*operation = RUN_AS_USER_LAUNCH_CONTAINER;
return 0;
@ -565,8 +566,8 @@ int main(int argc, char **argv) {
exit_code = initialize_app(cmd_input.yarn_user_name,
cmd_input.app_id,
cmd_input.cred_file,
extract_values(cmd_input.local_dirs),
extract_values(cmd_input.log_dirs),
split(cmd_input.local_dirs),
split(cmd_input.log_dirs),
argv + optind);
break;
case RUN_AS_USER_LAUNCH_DOCKER_CONTAINER:
@ -591,8 +592,8 @@ int main(int argc, char **argv) {
cmd_input.script_file,
cmd_input.cred_file,
cmd_input.pid_file,
extract_values(cmd_input.local_dirs),
extract_values(cmd_input.log_dirs),
split(cmd_input.local_dirs),
split(cmd_input.log_dirs),
cmd_input.docker_command_file,
cmd_input.resources_key,
cmd_input.resources_values);
@ -619,8 +620,8 @@ int main(int argc, char **argv) {
cmd_input.script_file,
cmd_input.cred_file,
cmd_input.pid_file,
extract_values(cmd_input.local_dirs),
extract_values(cmd_input.log_dirs),
split(cmd_input.local_dirs),
split(cmd_input.log_dirs),
cmd_input.resources_key,
cmd_input.resources_values);
free(cmd_input.resources_key);

134
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c Normal file
View File

@ -0,0 +1,134 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "util.h"
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
char** split_delimiter(char *value, const char *delim) {
char **return_values = NULL;
char *temp_tok = NULL;
char *tempstr = NULL;
int size = 0;
int per_alloc_size = 10;
int return_values_size = per_alloc_size;
int failed = 0;
//first allocate any array of 10
if(value != NULL) {
return_values = (char **) malloc(sizeof(char *) * return_values_size);
if (!return_values) {
fprintf(ERRORFILE, "Allocation error for return_values in %s.\n",
__func__);
failed = 1;
goto cleanup;
}
memset(return_values, 0, sizeof(char *) * return_values_size);
temp_tok = strtok_r(value, delim, &tempstr);
while (temp_tok != NULL) {
temp_tok = strdup(temp_tok);
if (NULL == temp_tok) {
fprintf(ERRORFILE, "Allocation error in %s.\n", __func__);
failed = 1;
goto cleanup;
}
return_values[size++] = temp_tok;
// Make sure returned values has enough space for the trailing NULL.
if (size >= return_values_size - 1) {
return_values_size += per_alloc_size;
return_values = (char **) realloc(return_values,(sizeof(char *) *
return_values_size));
// Make sure new added memory are filled with NULL
for (int i = size; i < return_values_size; i++) {
return_values[i] = NULL;
}
}
temp_tok = strtok_r(NULL, delim, &tempstr);
}
}
// Put trailing NULL to indicate values terminates.
if (return_values != NULL) {
return_values[size] = NULL;
}
cleanup:
if (failed) {
free_values(return_values);
return NULL;
}
return return_values;
}
/**
* Extracts array of values from the '%' separated list of values.
*/
char** split(char *value) {
return split_delimiter(value, "%");
}
// free an entry set of values
void free_values(char** values) {
if (values != NULL) {
int idx = 0;
while (values[idx]) {
free(values[idx]);
idx++;
}
free(values);
}
}
/**
* Trim whitespace from beginning and end.
*/
char* trim(const char* input) {
const char *val_begin;
const char *val_end;
char *ret;
if (input == NULL) {
return NULL;
}
val_begin = input;
val_end = input + strlen(input);
while (val_begin < val_end && isspace(*val_begin))
val_begin++;
while (val_end > val_begin && isspace(*(val_end - 1)))
val_end--;
ret = (char *) malloc(
sizeof(char) * (val_end - val_begin + 1));
if (ret == NULL) {
fprintf(ERRORFILE, "Allocation error\n");
exit(OUT_OF_MEMORY);
}
strncpy(ret, val_begin, val_end - val_begin);
ret[val_end - val_begin] = '\0';
return ret;
}

115
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h Normal file
View File

@ -0,0 +1,115 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
#define __YARN_POSIX_CONTAINER_EXECUTOR_UTIL_H__
#include <stdio.h>
enum errorcodes {
INVALID_ARGUMENT_NUMBER = 1,
//INVALID_USER_NAME 2
INVALID_COMMAND_PROVIDED = 3,
// SUPER_USER_NOT_ALLOWED_TO_RUN_TASKS (NOT USED) 4
INVALID_NM_ROOT_DIRS = 5,
SETUID_OPER_FAILED, //6
UNABLE_TO_EXECUTE_CONTAINER_SCRIPT, //7
UNABLE_TO_SIGNAL_CONTAINER, //8
INVALID_CONTAINER_PID, //9
// ERROR_RESOLVING_FILE_PATH (NOT_USED) 10
// RELATIVE_PATH_COMPONENTS_IN_FILE_PATH (NOT USED) 11
// UNABLE_TO_STAT_FILE (NOT USED) 12
// FILE_NOT_OWNED_BY_ROOT (NOT USED) 13
// PREPARE_CONTAINER_DIRECTORIES_FAILED (NOT USED) 14
// INITIALIZE_CONTAINER_FAILED (NOT USED) 15
// PREPARE_CONTAINER_LOGS_FAILED (NOT USED) 16
// INVALID_LOG_DIR (NOT USED) 17
OUT_OF_MEMORY = 18,
// INITIALIZE_DISTCACHEFILE_FAILED (NOT USED) 19
INITIALIZE_USER_FAILED = 20,
PATH_TO_DELETE_IS_NULL, //21
INVALID_CONTAINER_EXEC_PERMISSIONS, //22
// PREPARE_JOB_LOGS_FAILED (NOT USED) 23
INVALID_CONFIG_FILE = 24,
SETSID_OPER_FAILED = 25,
WRITE_PIDFILE_FAILED = 26,
WRITE_CGROUP_FAILED = 27,
TRAFFIC_CONTROL_EXECUTION_FAILED = 28,
DOCKER_RUN_FAILED = 29,
ERROR_OPENING_DOCKER_FILE = 30,
ERROR_READING_DOCKER_FILE = 31,
FEATURE_DISABLED = 32,
COULD_NOT_CREATE_SCRIPT_COPY = 33,
COULD_NOT_CREATE_CREDENTIALS_FILE = 34,
COULD_NOT_CREATE_WORK_DIRECTORIES = 35,
COULD_NOT_CREATE_APP_LOG_DIRECTORIES = 36,
COULD_NOT_CREATE_TMP_DIRECTORIES = 37,
ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS = 38,
ERROR_SANITIZING_DOCKER_COMMAND = 39,
DOCKER_IMAGE_INVALID = 40,
DOCKER_CONTAINER_NAME_INVALID = 41,
ERROR_COMPILING_REGEX = 42
};
// the log file for messages
extern FILE *LOGFILE;
// the log file for error messages
extern FILE *ERRORFILE;
/**
* Function to split the given string using '%' as the separator. It's
* up to the caller to free the memory for the returned array. Use the
* free_values function to free the allocated memory.
*
* @param str the string to split
*
* @return an array of strings
*/
char** split(char *str);
/**
* Function to split the given string using the delimiter specified. It's
* up to the caller to free the memory for the returned array. Use the
* free_values function to free the allocated memory.
*
* @param str the string to split
* @param delimiter the delimiter to use
*
* @return an array of strings
*/
char** split_delimiter(char *value, const char *delimiter);
/**
* Function to free an array of strings.
*
* @param values the array to free
*
*/
void free_values(char **values);
/**
* Trim whitespace from beginning and end. The returned string has to be freed
* by the caller.
*
* @param input Input string that needs to be trimmed
*
* @return the trimmed string allocated with malloc
*/
char* trim(const char *input);
#endif

31
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-1.cfg Normal file
View File

@ -0,0 +1,31 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[section-1]
key1=value1
split-key=val1,val2,val3
perc-key=perc-val1%perc-val2
# some comment
[split-section]
key3=value3
[section-2]
key1=value2
key2=value2
[split-section]
key4=value4

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/configuration-2.cfg Normal file
View File

@ -0,0 +1,28 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Test mixed mode config file
# Initial few lines are in the key=value format
# and then the sections start
key1=value1
key2=value2
[section-1]
key3=value3
key1=value4

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/resources/test/test-configurations/old-config.cfg Normal file
View File

@ -0,0 +1,25 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
yarn.nodemanager.linux-container-executor.group=yarn
banned.users=root,testuser1,testuser2#comma separated list of users who can not run applications
min.user.id=1000
allowed.system.users=nobody,daemon
feature.docker.enabled=1
feature.tc.enabled=0
docker.binary=/usr/bin/docker
yarn.local.dirs=/var/run/yarn%/tmp/mydir
test.key=#no value for this key
# test.key2=0

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
View File

@ -18,6 +18,7 @@
#include "configuration.h"
#include "container-executor.h"
#include "utils/string-utils.h"
#include "util.h"
#include <inttypes.h>
#include <errno.h>
@ -404,7 +405,7 @@ void test_delete_app() {
}
void validate_feature_enabled_value(int expected_value, const char* key,
int default_value, struct configuration *cfg) {
int default_value, struct section *cfg) {
int value = is_feature_enabled(key, default_value, cfg);
if (value != expected_value) {
@ -419,7 +420,8 @@ void test_is_feature_enabled() {
FILE *file = fopen(filename, "w");
int disabled = 0;
int enabled = 1;
struct configuration cfg = {.size=0, .confdetails=NULL};
struct configuration exec_cfg = {.size=0, .sections=NULL};
struct section cfg = {.size=0, .kv_pairs=NULL};
if (file == NULL) {
printf("FAIL: Could not open configuration file: %s\n", filename);
@ -433,7 +435,8 @@ void test_is_feature_enabled() {
fprintf(file, "feature.name5.enabled=-1\n");
fprintf(file, "feature.name6.enabled=2\n");
fclose(file);
read_config(filename, &cfg);
read_config(filename, &exec_cfg);
cfg = *(get_configuration_section("", &exec_cfg));
validate_feature_enabled_value(disabled, "feature.name1.enabled",
disabled, &cfg);
@ -449,7 +452,7 @@ void test_is_feature_enabled() {
disabled, &cfg);
free_configurations(&cfg);
free_configuration(&exec_cfg);
}
void test_delete_user() {
@ -1345,8 +1348,8 @@ int main(int argc, char **argv) {
read_executor_config(TEST_ROOT "/test.cfg");
local_dirs = extract_values(strdup(NM_LOCAL_DIRS));
log_dirs = extract_values(strdup(NM_LOG_DIRS));
local_dirs = split(strdup(NM_LOCAL_DIRS));
log_dirs = split(strdup(NM_LOG_DIRS));
create_nm_roots(local_dirs);

432
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_configuration.cc Normal file
View File

@ -0,0 +1,432 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
#include <fstream>
extern "C" {
#include "util.h"
#include "configuration.h"
#include "configuration.c"
}
namespace ContainerExecutor {
class TestConfiguration : public ::testing::Test {
protected:
virtual void SetUp() {
new_config_format_file = "test-configurations/configuration-1.cfg";
old_config_format_file = "test-configurations/old-config.cfg";
mixed_config_format_file = "test-configurations/configuration-2.cfg";
loadConfigurations();
return;
}
void loadConfigurations() {
int ret = 0;
ret = read_config(new_config_format_file.c_str(), &new_config_format);
ASSERT_EQ(0, ret);
ret = read_config(old_config_format_file.c_str(), &old_config_format);
ASSERT_EQ(0, ret);
ret = read_config(mixed_config_format_file.c_str(),
&mixed_config_format);
ASSERT_EQ(0, ret);
}
virtual void TearDown() {
free_configuration(&new_config_format);
free_configuration(&old_config_format);
return;
}
std::string new_config_format_file;
std::string old_config_format_file;
std::string mixed_config_format_file;
struct configuration new_config_format;
struct configuration old_config_format;
struct configuration mixed_config_format;
};
TEST_F(TestConfiguration, test_get_configuration_values_delimiter) {
char **split_values;
split_values = get_configuration_values_delimiter(NULL, "", &old_config_format, "%");
ASSERT_EQ(NULL, split_values);
split_values = get_configuration_values_delimiter("yarn.local.dirs", NULL,
&old_config_format, "%");
ASSERT_EQ(NULL, split_values);
split_values = get_configuration_values_delimiter("yarn.local.dirs", "",
NULL, "%");
ASSERT_EQ(NULL, split_values);
split_values = get_configuration_values_delimiter("yarn.local.dirs", "",
&old_config_format, NULL);
ASSERT_EQ(NULL, split_values);
split_values = get_configuration_values_delimiter("yarn.local.dirs", "abcd",
&old_config_format, "%");
ASSERT_EQ(NULL, split_values);
split_values = get_configuration_values_delimiter("yarn.local.dirs", "",
&old_config_format, "%");
ASSERT_STREQ("/var/run/yarn", split_values[0]);
ASSERT_STREQ("/tmp/mydir", split_values[1]);
ASSERT_EQ(NULL, split_values[2]);
free(split_values);
split_values = get_configuration_values_delimiter("allowed.system.users",
"", &old_config_format, "%");
ASSERT_STREQ("nobody,daemon", split_values[0]);
ASSERT_EQ(NULL, split_values[1]);
free(split_values);
}
TEST_F(TestConfiguration, test_get_configuration_values) {
char **split_values;
split_values = get_configuration_values(NULL, "", &old_config_format);
ASSERT_EQ(NULL, split_values);
split_values = get_configuration_values("yarn.local.dirs", NULL, &old_config_format);
ASSERT_EQ(NULL, split_values);
split_values = get_configuration_values("yarn.local.dirs", "", NULL);
ASSERT_EQ(NULL, split_values);
split_values = get_configuration_values("yarn.local.dirs", "abcd", &old_config_format);
ASSERT_EQ(NULL, split_values);
split_values = get_configuration_values("yarn.local.dirs", "", &old_config_format);
ASSERT_STREQ("/var/run/yarn%/tmp/mydir", split_values[0]);
ASSERT_EQ(NULL, split_values[1]);
free(split_values);
split_values = get_configuration_values("allowed.system.users", "",
&old_config_format);
ASSERT_STREQ("nobody", split_values[0]);
ASSERT_STREQ("daemon", split_values[1]);
ASSERT_EQ(NULL, split_values[2]);
free(split_values);
}
TEST_F(TestConfiguration, test_get_configuration_value) {
std::string key_value_array[5][2] = {
{"yarn.nodemanager.linux-container-executor.group", "yarn"},
{"min.user.id", "1000"},
{"allowed.system.users", "nobody,daemon"},
{"feature.docker.enabled", "1"},
{"yarn.local.dirs", "/var/run/yarn%/tmp/mydir"}
};
char *value;
value = get_configuration_value(NULL, "", &old_config_format);
ASSERT_EQ(NULL, value);
value = get_configuration_value("yarn.local.dirs", NULL, &old_config_format);
ASSERT_EQ(NULL, value);
value = get_configuration_value("yarn.local.dirs", "", NULL);
ASSERT_EQ(NULL, value);
for (int i = 0; i < 5; ++i) {
value = get_configuration_value(key_value_array[i][0].c_str(),
"", &old_config_format);
ASSERT_STREQ(key_value_array[i][1].c_str(), value);
free(value);
}
value = get_configuration_value("test.key", "", &old_config_format);
ASSERT_EQ(NULL, value);
value = get_configuration_value("test.key2", "", &old_config_format);
ASSERT_EQ(NULL, value);
value = get_configuration_value("feature.tc.enabled", "abcd", &old_config_format);
ASSERT_EQ(NULL, value);
}
TEST_F(TestConfiguration, test_no_sections_format) {
const struct section *executor_cfg = get_configuration_section("", &old_config_format);
char *value = NULL;
value = get_section_value("yarn.nodemanager.linux-container-executor.group", executor_cfg);
ASSERT_STREQ("yarn", value);
value = get_section_value("feature.docker.enabled", executor_cfg);
ASSERT_STREQ("1", value);
value = get_section_value("feature.tc.enabled", executor_cfg);
ASSERT_STREQ("0", value);
value = get_section_value("min.user.id", executor_cfg);
ASSERT_STREQ("1000", value);
value = get_section_value("docker.binary", executor_cfg);
ASSERT_STREQ("/usr/bin/docker", value);
char **list = get_section_values("allowed.system.users", executor_cfg);
ASSERT_STREQ("nobody", list[0]);
ASSERT_STREQ("daemon", list[1]);
list = get_section_values("banned.users", executor_cfg);
ASSERT_STREQ("root", list[0]);
ASSERT_STREQ("testuser1", list[1]);
ASSERT_STREQ("testuser2", list[2]);
}
TEST_F(TestConfiguration, test_get_section_values_delimiter) {
const struct section *section;
char *value;
char **split_values;
section = get_configuration_section("section-1", &new_config_format);
value = get_section_value("key1", section);
ASSERT_STREQ("value1", value);
free(value);
value = get_section_value("key2", section);
ASSERT_EQ(NULL, value);
split_values = get_section_values_delimiter(NULL, section, "%");
ASSERT_EQ(NULL, split_values);
split_values = get_section_values_delimiter("split-key", NULL, "%");
ASSERT_EQ(NULL, split_values);
split_values = get_section_values_delimiter("split-key", section, NULL);
ASSERT_EQ(NULL, split_values);
split_values = get_section_values_delimiter("split-key", section, "%");
ASSERT_FALSE(split_values == NULL);
ASSERT_STREQ("val1,val2,val3", split_values[0]);
ASSERT_TRUE(split_values[1] == NULL);
free_values(split_values);
split_values = get_section_values_delimiter("perc-key", section, "%");
ASSERT_FALSE(split_values == NULL);
ASSERT_STREQ("perc-val1", split_values[0]);
ASSERT_STREQ("perc-val2", split_values[1]);
ASSERT_TRUE(split_values[2] == NULL);
}
TEST_F(TestConfiguration, test_get_section_values) {
const struct section *section;
char *value;
char **split_values;
section = get_configuration_section("section-1", &new_config_format);
value = get_section_value(NULL, section);
ASSERT_EQ(NULL, value);
value = get_section_value("key1", NULL);
ASSERT_EQ(NULL, value);
value = get_section_value("key1", section);
ASSERT_STREQ("value1", value);
free(value);
value = get_section_value("key2", section);
ASSERT_EQ(NULL, value);
split_values = get_section_values("split-key", section);
ASSERT_FALSE(split_values == NULL);
ASSERT_STREQ("val1", split_values[0]);
ASSERT_STREQ("val2", split_values[1]);
ASSERT_STREQ("val3", split_values[2]);
ASSERT_TRUE(split_values[3] == NULL);
free_values(split_values);
split_values = get_section_values("perc-key", section);
ASSERT_FALSE(split_values == NULL);
ASSERT_STREQ("perc-val1%perc-val2", split_values[0]);
ASSERT_TRUE(split_values[1] == NULL);
free_values(split_values);
section = get_configuration_section("section-2", &new_config_format);
value = get_section_value("key1", section);
ASSERT_STREQ("value2", value);
free(value);
value = get_section_value("key2", section);
ASSERT_STREQ("value2", value);
free(value);
}
TEST_F(TestConfiguration, test_split_section) {
const struct section *section;
char *value;
section = get_configuration_section("split-section", &new_config_format);
value = get_section_value(NULL, section);
ASSERT_EQ(NULL, value);
value = get_section_value("key3", NULL);
ASSERT_EQ(NULL, value);
value = get_section_value("key3", section);
ASSERT_STREQ("value3", value);
free(value);
value = get_section_value("key4", section);
ASSERT_STREQ("value4", value);
}
TEST_F(TestConfiguration, test_get_configuration_section) {
const struct section *section;
ASSERT_EQ(3, new_config_format.size);
section = get_configuration_section(NULL, &new_config_format);
ASSERT_EQ(NULL, section);
section = get_configuration_section("section-1", NULL);
ASSERT_EQ(NULL, section);
section = get_configuration_section("section-1", &new_config_format);
ASSERT_FALSE(section == NULL);
ASSERT_STREQ("section-1", section->name);
ASSERT_EQ(3, section->size);
ASSERT_FALSE(NULL == section->kv_pairs);
section = get_configuration_section("section-2", &new_config_format);
ASSERT_FALSE(section == NULL);
ASSERT_STREQ("section-2", section->name);
ASSERT_EQ(2, section->size);
ASSERT_FALSE(NULL == section->kv_pairs);
section = get_configuration_section("section-3", &new_config_format);
ASSERT_TRUE(section == NULL);
}
TEST_F(TestConfiguration, test_read_config) {
struct configuration config;
int ret = 0;
ret = read_config(NULL, &config);
ASSERT_EQ(INVALID_CONFIG_FILE, ret);
ret = read_config("bad-config-file", &config);
ASSERT_EQ(INVALID_CONFIG_FILE, ret);
ret = read_config(new_config_format_file.c_str(), &config);
ASSERT_EQ(0, ret);
ASSERT_EQ(3, config.size);
ASSERT_STREQ("section-1", config.sections[0]->name);
ASSERT_STREQ("split-section", config.sections[1]->name);
ASSERT_STREQ("section-2", config.sections[2]->name);
free_configuration(&config);
ret = read_config(old_config_format_file.c_str(), &config);
ASSERT_EQ(0, ret);
ASSERT_EQ(1, config.size);
ASSERT_STREQ("", config.sections[0]->name);
free_configuration(&config);
}
TEST_F(TestConfiguration, test_get_kv_key) {
int ret = 0;
char buff[1024];
ret = get_kv_key(NULL, buff, 1024);
ASSERT_EQ(-EINVAL, ret);
ret = get_kv_key("key1234", buff, 1024);
ASSERT_EQ(-EINVAL, ret);
ret = get_kv_key("key=abcd", NULL, 1024);
ASSERT_EQ(-ENAMETOOLONG, ret);
ret = get_kv_key("key=abcd", buff, 1);
ASSERT_EQ(-ENAMETOOLONG, ret);
ret = get_kv_key("key=abcd", buff, 1024);
ASSERT_EQ(0, ret);
ASSERT_STREQ("key", buff);
}
TEST_F(TestConfiguration, test_get_kv_value) {
int ret = 0;
char buff[1024];
ret = get_kv_value(NULL, buff, 1024);
ASSERT_EQ(-EINVAL, ret);
ret = get_kv_value("key1234", buff, 1024);
ASSERT_EQ(-EINVAL, ret);
ret = get_kv_value("key=abcd", NULL, 1024);
ASSERT_EQ(-ENAMETOOLONG, ret);
ret = get_kv_value("key=abcd", buff, 1);
ASSERT_EQ(-ENAMETOOLONG, ret);
ret = get_kv_value("key=abcd", buff, 1024);
ASSERT_EQ(0, ret);
ASSERT_STREQ("abcd", buff);
}
TEST_F(TestConfiguration, test_single_section_high_key_count) {
std::string section_name = "section-1";
std::string sample_file_name = "large-section.cfg";
std::ofstream sample_file;
sample_file.open(sample_file_name.c_str());
sample_file << "[" << section_name << "]" << std::endl;
for(int i = 0; i < MAX_SIZE + 2; ++i) {
sample_file << "key" << i << "=" << "value" << i << std::endl;
}
struct configuration cfg;
int ret = read_config(sample_file_name.c_str(), &cfg);
ASSERT_EQ(0, ret);
ASSERT_EQ(1, cfg.size);
const struct section *section1 = get_configuration_section(section_name.c_str(), &cfg);
ASSERT_EQ(MAX_SIZE + 2, section1->size);
ASSERT_STREQ(section_name.c_str(), section1->name);
for(int i = 0; i < MAX_SIZE + 2; ++i) {
std::ostringstream oss;
oss << "key" << i;
const char *value = get_section_value(oss.str().c_str(), section1);
oss.str("");
oss << "value" << i;
ASSERT_STREQ(oss.str().c_str(), value);
}
remove(sample_file_name.c_str());
free_configuration(&cfg);
}
TEST_F(TestConfiguration, test_multiple_sections) {
std::string sample_file_name = "multiple-sections.cfg";
std::ofstream sample_file;
sample_file.open(sample_file_name.c_str());
for(int i = 0; i < MAX_SIZE + 2; ++i) {
sample_file << "[section-" << i << "]" << std::endl;
sample_file << "key" << i << "=" << "value" << i << std::endl;
}
struct configuration cfg;
int ret = read_config(sample_file_name.c_str(), &cfg);
ASSERT_EQ(0, ret);
ASSERT_EQ(MAX_SIZE + 2, cfg.size);
for(int i = 0; i < MAX_SIZE + 2; ++i) {
std::ostringstream oss;
oss << "section-" << i;
const struct section *section = get_configuration_section(oss.str().c_str(), &cfg);
ASSERT_EQ(1, section->size);
ASSERT_STREQ(oss.str().c_str(), section->name);
oss.str("");
oss << "key" << i;
const char *value = get_section_value(oss.str().c_str(), section);
oss.str("");
oss << "value" << i;
ASSERT_STREQ(oss.str().c_str(), value);
}
remove(sample_file_name.c_str());
free_configuration(&cfg);
}
TEST_F(TestConfiguration, test_section_start_line) {
const char *section_start_line = "[abcd]";
const char *non_section_lines[] = {
"[abcd", "abcd]", "key=value", "#abcd"
};
int ret = is_section_start_line(section_start_line);
ASSERT_EQ(1, ret);
int length = sizeof(non_section_lines) / sizeof(*non_section_lines);
for( int i = 0; i < length; ++i) {
ret = is_section_start_line(non_section_lines[i]);
ASSERT_EQ(0, ret);
}
ret = is_section_start_line(NULL);
ASSERT_EQ(0, ret);
}
TEST_F(TestConfiguration, test_comment_line) {
const char *comment_line = "#[abcd]";
const char *non_comment_lines[] = {
"[abcd", "abcd]", "key=value", "[abcd]"
};
int ret = is_comment_line(comment_line);
ASSERT_EQ(1, ret);
int length = sizeof(non_comment_lines) / sizeof(*non_comment_lines);
for( int i = 0; i < length; ++i) {
ret = is_comment_line(non_comment_lines[i]);
ASSERT_EQ(0, ret);
}
ret = is_comment_line(NULL);
ASSERT_EQ(0, ret);
}
TEST_F(TestConfiguration, test_mixed_config_format) {
const struct section *executor_cfg =
get_configuration_section("", &mixed_config_format);
char *value = NULL;
value = get_section_value("key1", executor_cfg);
ASSERT_STREQ("value1", value);
value = get_section_value("key2", executor_cfg);
ASSERT_STREQ("value2", value);
ASSERT_EQ(2, executor_cfg->size);
executor_cfg = get_configuration_section("section-1",
&mixed_config_format);
value = get_section_value("key3", executor_cfg);
ASSERT_STREQ("value3", value);
value = get_section_value("key1", executor_cfg);
ASSERT_STREQ("value4", value);
ASSERT_EQ(2, executor_cfg->size);
ASSERT_EQ(2, mixed_config_format.size);
ASSERT_STREQ("", mixed_config_format.sections[0]->name);
ASSERT_STREQ("section-1", mixed_config_format.sections[1]->name);
}
}

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_main.cc Normal file
View File

@ -0,0 +1,29 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
#include <main/native/container-executor/impl/util.h>
#include <cstdio>
FILE* ERRORFILE = stderr;
FILE* LOGFILE = stdout;
int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

138
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_util.cc Normal file
View File

@ -0,0 +1,138 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
#include <sstream>
extern "C" {
#include "util.h"
}
namespace ContainerExecutor {
class TestUtil : public ::testing::Test {
protected:
virtual void SetUp() {
}
virtual void TearDown() {
}
};
TEST_F(TestUtil, test_split_delimiter) {
std::string str = "1,2,3,4,5,6,7,8,9,10,11";
char *split_string = (char *) calloc(str.length() + 1, sizeof(char));
strncpy(split_string, str.c_str(), str.length());
char **splits = split_delimiter(split_string, ",");
ASSERT_TRUE(splits != NULL);
int count = 0;
while(splits[count] != NULL) {
++count;
}
ASSERT_EQ(11, count);
for(int i = 1; i < count; ++i) {
std::ostringstream oss;
oss << i;
ASSERT_STREQ(oss.str().c_str(), splits[i-1]);
}
ASSERT_EQ(NULL, splits[count]);
free_values(splits);
split_string = (char *) calloc(str.length() + 1, sizeof(char));
strncpy(split_string, str.c_str(), str.length());
splits = split_delimiter(split_string, "%");
ASSERT_TRUE(splits != NULL);
ASSERT_TRUE(splits[1] == NULL);
ASSERT_STREQ(str.c_str(), splits[0]);
free_values(splits);
splits = split_delimiter(NULL, ",");
ASSERT_EQ(NULL, splits);
return;
}
TEST_F(TestUtil, test_split) {
std::string str = "1%2%3%4%5%6%7%8%9%10%11";
char *split_string = (char *) calloc(str.length() + 1, sizeof(char));
strncpy(split_string, str.c_str(), str.length());
char **splits = split(split_string);
int count = 0;
while(splits[count] != NULL) {
++count;
}
ASSERT_EQ(11, count);
for(int i = 1; i < count; ++i) {
std::ostringstream oss;
oss << i;
ASSERT_STREQ(oss.str().c_str(), splits[i-1]);
}
ASSERT_EQ(NULL, splits[count]);
free_values(splits);
str = "1,2,3,4,5,6,7,8,9,10,11";
split_string = (char *) calloc(str.length() + 1, sizeof(char));
strncpy(split_string, str.c_str(), str.length());
splits = split(split_string);
ASSERT_TRUE(splits != NULL);
ASSERT_TRUE(splits[1] == NULL);
ASSERT_STREQ(str.c_str(), splits[0]);
return;
}
TEST_F(TestUtil, test_trim) {
char* trimmed = NULL;
// Check NULL input
ASSERT_EQ(NULL, trim(NULL));
// Check empty input
trimmed = trim("");
ASSERT_STREQ("", trimmed);
free(trimmed);
// Check single space input
trimmed = trim(" ");
ASSERT_STREQ("", trimmed);
free(trimmed);
// Check multi space input
trimmed = trim(" ");
ASSERT_STREQ("", trimmed);
free(trimmed);
// Check both side trim input
trimmed = trim(" foo ");
ASSERT_STREQ("foo", trimmed);
free(trimmed);
// Check left side trim input
trimmed = trim("foo ");
ASSERT_STREQ("foo", trimmed);
free(trimmed);
// Check right side trim input
trimmed = trim(" foo");
ASSERT_STREQ("foo", trimmed);
free(trimmed);
// Check no trim input
trimmed = trim("foo");
ASSERT_STREQ("foo", trimmed);
free(trimmed);
}
}