Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-3808. fuse_dfs: postpone libhdfs intialization until after fork. Contributed by Colin Patrick McCabe. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1374107 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Aaron Myers 2012-08-16 23:48:23 +00:00
parent 4923791df8
commit 2fc30022d0
3 changed files with 48 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -464,6 +464,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3194. DataNode block scanner is running too frequently.
(Andy Isaacson via eli)
HDFS-3808. fuse_dfs: postpone libhdfs intialization until after fork.
(Colin Patrick McCabe via atm)
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)

38
hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c
View File

@ -24,6 +24,7 @@
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
int is_protected(const char *path) {
@ -65,15 +66,6 @@ static struct fuse_operations dfs_oper = {
.truncate = dfs_truncate,
};
static void print_env_vars(void)
{
const char *cp = getenv("CLASSPATH");
const char *ld = getenv("LD_LIBRARY_PATH");
fprintf(stderr, "LD_LIBRARY_PATH=%s",ld == NULL ? "NULL" : ld);
fprintf(stderr, "CLASSPATH=%s",cp == NULL ? "NULL" : cp);
}
int main(int argc, char *argv[])
{
int ret;
@ -103,7 +95,7 @@ int main(int argc, char *argv[])
}
{
char buf[1024];
char buf[80];
snprintf(buf, sizeof buf, "-oattr_timeout=%d",options.attribute_timeout);
fuse_opt_add_arg(&args, buf);
@ -114,24 +106,18 @@ int main(int argc, char *argv[])
if (options.nn_uri == NULL) {
print_usage(argv[0]);
exit(0);
}
ret = fuseConnectInit(options.nn_uri, options.nn_port);
if (ret) {
ERROR("FATAL: dfs_init: fuseConnInit failed with error %d!", ret);
print_env_vars();
exit(EXIT_FAILURE);
}
if (options.initchecks == 1) {
ret = fuseConnectTest();
if (ret) {
ERROR("FATAL: dfs_init: fuseConnTest failed with error %d!", ret);
print_env_vars();
exit(EXIT_FAILURE);
}
exit(EXIT_SUCCESS);
}
/* Note: do not call any libhdfs functions until fuse_main has been invoked.
*
* fuse_main will daemonize this process, by calling fork(). This will cause
* any extant threads to be destroyed, which could cause problems if
* libhdfs has started some Java threads.
*
* Most initialization code should go in dfs_init, which is invoked after the
* fork. See HDFS-3808 for details.
*/
ret = fuse_main(args.argc, args.argv, &dfs_oper, NULL);
fuse_opt_free_args(&args);
return ret;

43
hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c
View File

@ -26,11 +26,20 @@
#include <stdlib.h>
#include <string.h>
static void print_env_vars(void)
{
const char *cp = getenv("CLASSPATH");
const char *ld = getenv("LD_LIBRARY_PATH");
ERROR("LD_LIBRARY_PATH=%s",ld == NULL ? "NULL" : ld);
ERROR("CLASSPATH=%s",cp == NULL ? "NULL" : cp);
}
// Hacked up function to basically do:
// protectedpaths = split(options.protected,':');
void init_protectedpaths(dfs_context *dfs) {
static void init_protectedpaths(dfs_context *dfs)
{
char *tmp = options.protected;
// handle degenerate case up front.
@ -39,7 +48,6 @@ void init_protectedpaths(dfs_context *dfs) {
dfs->protectedpaths[0] = NULL;
return;
}
assert(tmp);
if (options.debug) {
print_options();
@ -80,10 +88,10 @@ void init_protectedpaths(dfs_context *dfs) {
static void dfsPrintOptions(FILE *fp, const struct options *o)
{
fprintf(fp, "[ protected=%s, nn_uri=%s, nn_port=%d, "
INFO("Mounting with options: [ protected=%s, nn_uri=%s, nn_port=%d, "
"debug=%d, read_only=%d, initchecks=%d, "
"no_permissions=%d, usetrash=%d, entry_timeout=%d, "
"attribute_timeout=%d, rdbuffer_size=%Zd, direct_io=%d ]",
"attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d ]",
(o->protected ? o->protected : "(NULL)"), o->nn_uri, o->nn_port,
o->debug, o->read_only, o->initchecks,
o->no_permissions, o->usetrash, o->entry_timeout,
@ -92,12 +100,14 @@ static void dfsPrintOptions(FILE *fp, const struct options *o)
void *dfs_init(void)
{
int ret;
//
// Create a private struct of data we will pass to fuse here and which
// will then be accessible on every call.
//
dfs_context *dfs = (dfs_context*)malloc(sizeof(dfs_context));
if (NULL == dfs) {
dfs_context *dfs = calloc(1, sizeof(*dfs));
if (!dfs) {
ERROR("FATAL: could not malloc dfs_context");
exit(1);
}
@ -110,17 +120,30 @@ void *dfs_init(void)
dfs->rdbuffer_size = options.rdbuffer_size;
dfs->direct_io = options.direct_io;
fprintf(stderr, "Mounting with options ");
dfsPrintOptions(stderr, &options);
fprintf(stderr, "\n");
init_protectedpaths(dfs);
assert(dfs->protectedpaths != NULL);
if (dfs->rdbuffer_size <= 0) {
DEBUG("dfs->rdbuffersize <= 0 = %ld", dfs->rdbuffer_size);
DEBUG("dfs->rdbuffersize <= 0 = %zd", dfs->rdbuffer_size);
dfs->rdbuffer_size = 32768;
}
ret = fuseConnectInit(options.nn_uri, options.nn_port);
if (ret) {
ERROR("FATAL: dfs_init: fuseConnectInit failed with error %d!", ret);
print_env_vars();
exit(EXIT_FAILURE);
}
if (options.initchecks == 1) {
ret = fuseConnectTest();
if (ret) {
ERROR("FATAL: dfs_init: fuseConnectTest failed with error %d!", ret);
print_env_vars();
exit(EXIT_FAILURE);
}
}
return (void*)dfs;
}