Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5121. fix some container-executor portability issues. Contributed by Allen Wittenauer.

This commit is contained in:
Chris Nauroth 2016-07-30 08:26:00 -07:00
parent ce93595d7a
commit ef501b1a0b
16 changed files with 691 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
LICENSE.txt
View File

@ -414,6 +414,38 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h:
Copyright (c) 2012 The FreeBSD Foundation
All rights reserved.
This software was developed by Pawel Jakub Dawidek under sponsorship from
the FreeBSD Foundation.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH DAMAGE.
=============
The binary distribution of this product bundles binaries of leveldb
(http://code.google.com/p/leveldb/), which is available under the following
license:

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
View File

@ -273,6 +273,18 @@
</configuration>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>src/main/native/container-executor/impl/compat/fstatat.h</exclude>
<exclude>src/main/native/container-executor/impl/compat/openat.h</exclude>
<exclude>src/main/native/container-executor/impl/compat/unlinkat.h</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
View File

@ -24,7 +24,20 @@ string(REPLACE "-D_FILE_OFFSET_BITS=64" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
string(REPLACE "-D_FILE_OFFSET_BITS=64" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
include(CheckFunctionExists)
check_function_exists(canonicalize_file_name HAVE_CANONICALIZE_FILE_NAME)
check_function_exists(fcloseall HAVE_FCLOSEALL)
check_function_exists(fchmodat HAVE_FCHMODAT)
check_function_exists(fdopendir HAVE_FDOPENDIR)
check_function_exists(fstatat HAVE_FSTATAT)
check_function_exists(openat HAVE_OPENAT)
check_function_exists(unlinkat HAVE_UNLINKAT)
if(APPLE)
include_directories( /System/Library/Frameworks )
find_library(COCOA_LIBRARY Cocoa)
mark_as_advanced(COCOA_LIBRARY)
set(EXTRA_LIBS ${COCOA_LIBRARY})
endif(APPLE)
function(output_directory TGT DIR)
set_target_properties(${TGT} PROPERTIES
@ -46,6 +59,7 @@ configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
add_library(container
main/native/container-executor/impl/configuration.c
main/native/container-executor/impl/container-executor.c
main/native/container-executor/impl/get_executable.c
)
add_executable(container-executor
@ -60,6 +74,6 @@ add_executable(test-container-executor
main/native/container-executor/test/test-container-executor.c
)
target_link_libraries(test-container-executor
container
container ${EXTRA_LIBS}
)
output_directory(test-container-executor target/usr/local/bin)

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake
View File

@ -20,6 +20,12 @@
#cmakedefine HADOOP_CONF_DIR "@HADOOP_CONF_DIR@"
#cmakedefine HAVE_FCLOSEALL "@HAVE_FCLOSEALL@"
#cmakedefine HAVE_CANONICALIZE_FILE_NAME @HAVE_CANONICALIZE_FILE_NAME@
#cmakedefine HAVE_FCHMODAT @HAVE_FCHMODAT@
#cmakedefine HAVE_FCLOSEALL @HAVE_FCLOSEALL@
#cmakedefine HAVE_FDOPENDIR @HAVE_FDOPENDIR@
#cmakedefine HAVE_FSTATAT @HAVE_FSTATAT@
#cmakedefine HAVE_OPENAT @HAVE_OPENAT@
#cmakedefine HAVE_UNLINKAT @HAVE_UNLINKAT@
#endif

56
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/fchmodat.h Normal file
View File

@ -0,0 +1,56 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef _FCHMODAT_H_
#define _FCHMODAT_H_
#include <sys/stat.h>
#include <unistd.h>
#define AT_SYMLINK_NOFOLLOW 0x01
static int
fchmodat(int fd, const char *path, mode_t mode, int flag)
{
int cfd, error, ret;
cfd = open(".", O_RDONLY | O_DIRECTORY);
if (cfd == -1)
return (-1);
if (fchdir(fd) == -1) {
error = errno;
(void)close(cfd);
errno = error;
return (-1);
}
if (flag == AT_SYMLINK_NOFOLLOW)
ret = lchmod(path, mode);
else
ret = chmod(path, mode);
error = errno;
(void)fchdir(cfd);
(void)close(cfd);
errno = error;
return (ret);
}
#endif /* !_FCHMODAT_H_ */

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/fdopendir.h Normal file
View File

@ -0,0 +1,52 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef _FDOPENDIR_H_
#define _FDOPENDIR_H_
#include <fcntl.h>
#include <stdarg.h>
#include <unistd.h>
DIR *
fdopendir(int fd)
{
int cfd, error;
DIR *dfd;
cfd = open(".", O_RDONLY | O_DIRECTORY);
if (cfd == -1)
return (NULL);
if (fchdir(fd) == -1) {
error = errno;
(void)close(cfd);
errno = error;
return (NULL);
}
dfd=opendir(".");
error = errno;
(void)fchdir(cfd);
(void)close(cfd);
errno = error;
return (dfd);
}
#endif /* !_FDOPENDIR_H_ */

67
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/fstatat.h Normal file
View File

@ -0,0 +1,67 @@
/*-
* Copyright (c) 2012 The FreeBSD Foundation
* All rights reserved.
*
* This software was developed by Pawel Jakub Dawidek under sponsorship from
* the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _FSTATAT_H_
#define _FSTATAT_H_
#include <sys/stat.h>
#include <unistd.h>
#define AT_SYMLINK_NOFOLLOW 0x01
static int
fstatat(int fd, const char *path, struct stat *buf, int flag)
{
int cfd, error, ret;
cfd = open(".", O_RDONLY | O_DIRECTORY);
if (cfd == -1)
return (-1);
if (fchdir(fd) == -1) {
error = errno;
(void)close(cfd);
errno = error;
return (-1);
}
if (flag == AT_SYMLINK_NOFOLLOW)
ret = lstat(path, buf);
else
ret = stat(path, buf);
error = errno;
(void)fchdir(cfd);
(void)close(cfd);
errno = error;
return (ret);
}
#endif /* !_FSTATAT_H_ */

74
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/openat.h Normal file
View File

@ -0,0 +1,74 @@
/*-
* Copyright (c) 2012 The FreeBSD Foundation
* All rights reserved.
*
* This software was developed by Pawel Jakub Dawidek under sponsorship from
* the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _OPENAT_H_
#define _OPENAT_H_
#include <fcntl.h>
#include <stdarg.h>
#include <unistd.h>
static int
openat(int fd, const char *path, int flags, ...)
{
int cfd, ffd, error;
cfd = open(".", O_RDONLY | O_DIRECTORY);
if (cfd == -1)
return (-1);
if (fchdir(fd) == -1) {
error = errno;
(void)close(cfd);
errno = error;
return (-1);
}
if ((flags & O_CREAT) != 0) {
va_list ap;
int mode;
va_start(ap, flags);
mode = va_arg(ap, int);
va_end(ap);
ffd = open(path, flags, mode);
} else {
ffd = open(path, flags);
}
error = errno;
(void)fchdir(cfd);
(void)close(cfd);
errno = error;
return (ffd);
}
#endif /* !_OPENAT_H_ */

67
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/unlinkat.h Normal file
View File

@ -0,0 +1,67 @@
/*-
* Copyright (c) 2012 The FreeBSD Foundation
* All rights reserved.
*
* This software was developed by Pawel Jakub Dawidek under sponsorship from
* the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _UNLINKAT_H_
#define _UNLINKAT_H_
#include <fcntl.h>
#include <unistd.h>
#define AT_REMOVEDIR 0x01
static int
unlinkat(int fd, const char *path, int flag)
{
int cfd, error, ret;
cfd = open(".", O_RDONLY | O_DIRECTORY);
if (cfd == -1)
return (-1);
if (fchdir(fd) == -1) {
error = errno;
(void)close(cfd);
errno = error;
return (-1);
}
if (flag == AT_REMOVEDIR)
ret = rmdir(path);
else
ret = unlink(path);
error = errno;
(void)fchdir(cfd);
(void)close(cfd);
errno = error;
return (ret);
}
#endif /* !_UNLINKAT_H_ */

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
View File

@ -92,8 +92,13 @@ char *resolve_config_path(const char* file_name, const char *root) {
real_fname = buffer;
}
#ifdef HAVE_CANONICALIZE_FILE_NAME
char * ret = (real_fname == NULL) ? NULL : canonicalize_file_name(real_fname);
#else
char * ret = (real_fname == NULL) ? NULL : realpath(real_fname, NULL);
#endif
#ifdef DEBUG
fprintf(stderr,"ret = %s\n", ret);
fprintf(stderr, "resolve_config_path(file_name=%s,root=%s)=%s\n",
file_name, root ? root : "null", ret ? ret : "null");
#endif

56
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
View File

@ -40,6 +40,28 @@
#include <sys/mount.h>
#include <sys/wait.h>
#include "config.h"
#ifndef HAVE_FCHMODAT
#include "compat/fchmodat.h"
#endif
#ifndef HAVE_FDOPENDIR
#include "compat/fdopendir.h"
#endif
#ifndef HAVE_FSTATAT
#include "compat/fstatat.h"
#endif
#ifndef HAVE_OPENAT
#include "compat/openat.h"
#endif
#ifndef HAVE_UNLINKAT
#include "compat/unlinkat.h"
#endif
static const int DEFAULT_MIN_USERID = 1000;
static const char* DEFAULT_BANNED_USERS[] = {"yarn", "mapred", "hdfs", "bin", 0};
@ -84,31 +106,14 @@ char *get_nodemanager_group() {
return get_value(NM_GROUP_KEY, &executor_cfg);
}
/**
* get the executable filename.
*/
char* get_executable() {
char buffer[EXECUTOR_PATH_MAX];
snprintf(buffer, EXECUTOR_PATH_MAX, "/proc/%" PRId64 "/exe", (int64_t)getpid());
char *filename = malloc(EXECUTOR_PATH_MAX);
ssize_t len = readlink(buffer, filename, EXECUTOR_PATH_MAX);
if (len == -1) {
fprintf(ERRORFILE, "Can't get executable name from %s - %s\n", buffer,
strerror(errno));
exit(-1);
} else if (len >= EXECUTOR_PATH_MAX) {
fprintf(ERRORFILE, "Executable name %.*s is longer than %d characters.\n",
EXECUTOR_PATH_MAX, filename, EXECUTOR_PATH_MAX);
exit(-1);
}
filename[len] = '\0';
return filename;
}
int check_executor_permissions(char *executable_file) {
errno = 0;
#ifdef HAVE_CANONICALIZE_FILE_NAME
char * resolved_path = canonicalize_file_name(executable_file);
#else
char * resolved_path = realpath(executable_file, NULL);
#endif
if (resolved_path == NULL) {
fprintf(ERRORFILE,
"Error resolving the canonical name for the executable : %s!",
@ -181,6 +186,7 @@ static int change_effective_user(uid_t user, gid_t group) {
return 0;
}
#ifdef __linux
/**
* Write the pid of the current process to the cgroup file.
* cgroup_file: Path to cgroup file where pid needs to be written to.
@ -218,6 +224,7 @@ static int write_pid_to_cgroup_as_root(const char* cgroup_file, pid_t pid) {
return 0;
}
#endif
/**
* Write the pid of the current process into the pid file.
@ -1328,6 +1335,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
}
if (pid != 0) {
#ifdef __linux
fprintf(LOGFILE, "Writing to cgroup task files...\n");
// cgroups-based resource enforcement
if (resources_key != NULL && ! strcmp(resources_key, "cgroups")) {
@ -1342,6 +1350,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
}
}
}
#endif
// write pid to pidfile
fprintf(LOGFILE, "Writing pid file...\n");
@ -1496,6 +1505,7 @@ int launch_container_as_user(const char *user, const char *app_id,
goto cleanup;
}
#ifdef __linux
fprintf(LOGFILE, "Writing to cgroup task files...\n");
// cgroups-based resource enforcement
if (resources_key != NULL && ! strcmp(resources_key, "cgroups")) {
@ -1510,6 +1520,7 @@ int launch_container_as_user(const char *user, const char *app_id,
}
}
}
#endif
fprintf(LOGFILE, "Creating local dirs...\n");
exit_code = create_local_dirs(user, app_id, container_id,
@ -1572,9 +1583,6 @@ int signal_container_as_user(const char *user, int pid, int sig) {
fprintf(LOGFILE,
"Error signalling process group %d with signal %d - %s\n",
-pid, sig, strerror(errno));
fprintf(stderr,
"Error signalling process group %d with signal %d - %s\n",
-pid, sig, strerror(errno));
fflush(LOGFILE);
return UNABLE_TO_SIGNAL_CONTAINER;
} else {

127
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c Normal file
View File

@ -0,0 +1,127 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* This code implements OS-specific ways to get the absolute
* filename of the executable. Typically, one would use
* realpath(argv[0]) (or equivalent), however, because this
* code runs as setuid and will be used later on to determine
* relative paths, we want something a big more secure
* since argv[0] is replaceable by malicious code.
*
* NOTE! The value returned will be free()'d later on!
*
*/
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include "config.h"
#include "configuration.h"
#include "container-executor.h"
/*
* A generic function to read a link and return
* the value for use with System V procfs.
* With much thanks to Tom Killian, Roger Faulkner,
* and Ron Gomes, this is pretty generic code.
*
* The various BSDs do not have (reliably)
* have /proc. Custom implementations follow.
*/
char *__get_exec_readproc(char *procfn) {
char *filename;
ssize_t len;
filename = malloc(EXECUTOR_PATH_MAX);
if (!filename) {
fprintf(ERRORFILE,"cannot allocate memory for filename: %s\n",strerror(errno));
exit(-1);
}
len = readlink(procfn, filename, EXECUTOR_PATH_MAX);
if (len == -1) {
fprintf(ERRORFILE,"Can't get executable name from %s - %s\n", procfn,
strerror(errno));
exit(-1);
} else if (len >= EXECUTOR_PATH_MAX) {
fprintf(ERRORFILE,"Executable name %.*s is longer than %d characters.\n",
EXECUTOR_PATH_MAX, filename, EXECUTOR_PATH_MAX);
exit(-1);
}
filename[len] = '\0';
return filename;
}
#ifdef __APPLE__
/*
* Mac OS X doesn't have a procfs, but there is
* libproc which we can use instead. It is available
* in most modern versions of OS X as of this writing (2016).
*/
#include <libproc.h>
char* get_executable() {
char *filename;
pid_t pid;
filename = malloc(PROC_PIDPATHINFO_MAXSIZE);
if (!filename) {
fprintf(ERRORFILE,"cannot allocate memory for filename: %s\n",strerror(errno));
exit(-1);
}
pid = getpid();
if (proc_pidpath(pid,filename,PROC_PIDPATHINFO_MAXSIZE) <= 0) {
fprintf(ERRORFILE,"Can't get executable name from pid %u - %s\n", pid,
strerror(errno));
exit(-1);
}
return filename;
}
#elif defined(__linux)
char* get_executable() {
return __get_exec_readproc("/proc/self/exe");
}
#elif defined(__sun)
/*
* It's tempting to use getexecname(), but there is no guarantee
* we will get a full path and worse, we'd be reliant on getcwd()
* being where our exec is at. Instead, we'll use the /proc
* method, using the "invisible" /proc/self link that only the
* process itself can see. (Anyone that tells you /proc/self
* doesn't exist on Solaris hasn't read the proc(4) man page.)
*/
char* get_executable() {
return __get_exec_readproc("/proc/self/path/a.out");
}
#else
#error Cannot safely determine executable path on this operating system.
#endif

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
View File

@ -96,19 +96,27 @@ in case of validation failures. Also sets up configuration / group information e
This function is to be called in every invocation of container-executor, irrespective
of whether an explicit checksetup operation is requested. */
static void assert_valid_setup(char *current_executable) {
static void assert_valid_setup() {
int ret;
char *executable_file = get_executable();
if (!executable_file) {
fprintf(ERRORFILE,"realpath of executable: %s\n",strerror(errno));
flush_and_close_log_files();
exit(-1);
}
char *orig_conf_file = HADOOP_CONF_DIR "/" CONF_FILENAME;
char *conf_file = resolve_config_path(orig_conf_file, current_executable);
char *conf_file = resolve_config_path(orig_conf_file, executable_file);
if (conf_file == NULL) {
free(executable_file);
fprintf(ERRORFILE, "Configuration file %s not found.\n", orig_conf_file);
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
if (check_configuration_permissions(conf_file) != 0) {
free(executable_file);
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
@ -118,28 +126,42 @@ static void assert_valid_setup(char *current_executable) {
// look up the node manager group in the config file
char *nm_group = get_nodemanager_group();
if (nm_group == NULL) {
free(executable_file);
fprintf(ERRORFILE, "Can't get configured value for %s.\n", NM_GROUP_KEY);
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
struct group *group_info = getgrnam(nm_group);
if (group_info == NULL) {
free(executable_file);
fprintf(ERRORFILE, "Can't get group information for %s - %s.\n", nm_group,
strerror(errno));
flush_and_close_log_files();
exit(INVALID_CONFIG_FILE);
}
set_nm_uid(getuid(), group_info->gr_gid);
// if we are running from a setuid executable, make the real uid root
setuid(0);
// set the real and effective group id to the node manager group
setgid(group_info->gr_gid);
/*
* if we are running from a setuid executable, make the real uid root
* we're going to ignore this result just in case we aren't.
*/
ret=setuid(0);
/*
* set the real and effective group id to the node manager group
* we're going to ignore this result just in case we aren't
*/
ret=setgid(group_info->gr_gid);
/* make the unused var warning to away */
ret++;
if (check_executor_permissions(executable_file) != 0) {
free(executable_file);
fprintf(ERRORFILE, "Invalid permissions on container-executor binary.\n");
flush_and_close_log_files();
exit(INVALID_CONTAINER_EXEC_PERMISSIONS);
}
free(executable_file);
}
@ -407,7 +429,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
int main(int argc, char **argv) {
open_log_files();
assert_valid_setup(argv[0]);
assert_valid_setup();
int operation;
int ret = validate_arguments(argc, argv, &operation);

79
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
View File

@ -29,7 +29,19 @@
#include <sys/stat.h>
#include <sys/wait.h>
#define TEST_ROOT "/tmp/test-container-executor"
#ifdef __APPLE__
#include <CoreFoundation/CFString.h>
#include <CoreFoundation/CFPreferences.h>
#define TMPDIR "/private/tmp"
#define RELTMPDIR "../.."
#else
#define RELTMPDIR ".."
#define TMPDIR "/tmp"
#endif
#define TEST_ROOT TMPDIR "/test-container-executor"
#define DONT_TOUCH_FILE "dont-touch-me"
#define NM_LOCAL_DIRS TEST_ROOT "/local-1%" TEST_ROOT "/local-2%" \
TEST_ROOT "/local-3%" TEST_ROOT "/local-4%" TEST_ROOT "/local-5"
@ -155,8 +167,8 @@ void check_pid_file(const char* pid_file, pid_t mypid) {
}
void test_get_user_directory() {
char *user_dir = get_user_directory("/tmp", "user");
char *expected = "/tmp/usercache/user";
char *user_dir = get_user_directory(TMPDIR, "user");
char *expected = TMPDIR "/usercache/user";
if (strcmp(user_dir, expected) != 0) {
printf("test_get_user_directory expected %s got %s\n", expected, user_dir);
exit(1);
@ -165,8 +177,8 @@ void test_get_user_directory() {
}
void test_get_app_directory() {
char *expected = "/tmp/usercache/user/appcache/app_200906101234_0001";
char *app_dir = (char *) get_app_directory("/tmp", "user",
char *expected = TMPDIR "/usercache/user/appcache/app_200906101234_0001";
char *app_dir = (char *) get_app_directory(TMPDIR, "user",
"app_200906101234_0001");
if (strcmp(app_dir, expected) != 0) {
printf("test_get_app_directory expected %s got %s\n", expected, app_dir);
@ -176,9 +188,9 @@ void test_get_app_directory() {
}
void test_get_container_directory() {
char *container_dir = get_container_work_directory("/tmp", "owen", "app_1",
char *container_dir = get_container_work_directory(TMPDIR, "owen", "app_1",
"container_1");
char *expected = "/tmp/usercache/owen/appcache/app_1/container_1";
char *expected = TMPDIR"/usercache/owen/appcache/app_1/container_1";
if (strcmp(container_dir, expected) != 0) {
printf("Fail get_container_work_directory got %s expected %s\n",
container_dir, expected);
@ -188,9 +200,9 @@ void test_get_container_directory() {
}
void test_get_container_launcher_file() {
char *expected_file = ("/tmp/usercache/user/appcache/app_200906101234_0001"
char *expected_file = (TMPDIR"/usercache/user/appcache/app_200906101234_0001"
"/launch_container.sh");
char *app_dir = get_app_directory("/tmp", "user",
char *app_dir = get_app_directory(TMPDIR, "user",
"app_200906101234_0001");
char *container_file = get_container_launcher_file(app_dir);
if (strcmp(container_file, expected_file) != 0) {
@ -241,9 +253,9 @@ void test_resolve_config_path() {
TEST_ROOT "\n");
exit(1);
}
if (strcmp(resolve_config_path(".." TEST_ROOT, TEST_ROOT), TEST_ROOT) != 0) {
if (strcmp(resolve_config_path(RELTMPDIR TEST_ROOT, TEST_ROOT), TEST_ROOT) != 0) {
printf("FAIL: failed to resolve config_name on a relative path name: "
".." TEST_ROOT " (relative to " TEST_ROOT ")");
RELTMPDIR TEST_ROOT " (relative to " TEST_ROOT ")");
exit(1);
}
}
@ -632,7 +644,7 @@ void test_run_container() {
printf("FAIL: failed to seteuid back to user - %s\n", strerror(errno));
exit(1);
}
if (fprintf(script, "#!/bin/bash\n"
if (fprintf(script, "#!/usr/bin/env bash\n"
"touch foobar\n"
"exit 0") < 0) {
printf("FAIL: fprintf failed - %s\n", strerror(errno));
@ -810,6 +822,7 @@ void test_recursive_unlink_children() {
// 4. super user with a given user and a given yarn user
// # test-container-executor user yarn_user
int main(int argc, char **argv) {
int ret;
LOGFILE = stdout;
ERRORFILE = stderr;
@ -881,10 +894,36 @@ int main(int argc, char **argv) {
test_check_user(0);
#ifdef __APPLE__
printf("OS X: disabling CrashReporter\n");
/*
* disable the "unexpectedly quit" dialog box
* because we know we're going to make our container
* do exactly that.
*/
CFStringRef crashType = CFSTR("DialogType");
CFStringRef crashModeNone = CFSTR("None");
CFStringRef crashAppID = CFSTR("com.apple.CrashReporter");
CFStringRef crashOldMode = CFPreferencesCopyAppValue(CFSTR("DialogType"), CFSTR("com.apple.CrashReporter"));
CFPreferencesSetAppValue(crashType, crashModeNone, crashAppID);
CFPreferencesAppSynchronize(crashAppID);
#endif
// the tests that change user need to be run in a subshell, so that
// when they change user they don't give up our privs
run_test_in_child("test_signal_container_group", test_signal_container_group);
#ifdef __APPLE__
/*
* put the "unexpectedly quit" dialog back
*/
CFPreferencesSetAppValue(crashType, crashOldMode, crashAppID);
CFPreferencesAppSynchronize(crashAppID);
printf("OS X: CrashReporter re-enabled\n");
#endif
// init app and run container can't be run if you aren't testing as root
if (getuid() == 0) {
// these tests do internal forks so that the change_owner and execs
@ -893,7 +932,13 @@ int main(int argc, char **argv) {
test_run_container();
}
seteuid(0);
/*
* try to seteuid(0). if it doesn't work, carry on anyway.
* we're going to capture the return value to get rid of a
* compiler warning.
*/
ret=seteuid(0);
ret++;
// test_delete_user must run as root since that's how we use the delete_as_user
test_delete_user();
free_executor_configurations();
@ -904,11 +949,19 @@ int main(int argc, char **argv) {
}
read_executor_config(TEST_ROOT "/test.cfg");
#ifdef __APPLE__
username = "_uucp";
test_check_user(1);
username = "_networkd";
test_check_user(1);
#else
username = "bin";
test_check_user(1);
username = "sys";
test_check_user(1);
#endif
run("rm -fr " TEST_ROOT);
printf("\nFinished tests\n");