Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-3610. fuse_dfs: Provide a way to use the default (configured) NN URI. Contributed by Colin Patrick McCabe 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1361566 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-07-14 17:19:26 +00:00
parent d03fb9c0d2
commit 30d0104c0e
20 changed files with 54 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -145,6 +145,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3539. libhdfs code cleanups. (Colin Patrick McCabe via eli)
HDFS-3610. fuse_dfs: Provide a way to use the default (configured) NN URI.
(Colin Patrick McCabe via eli)
OPTIMIZATIONS
HDFS-2982. Startup performance suffers when there are many edit log

8
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_connect.c
View File

@ -172,7 +172,7 @@ done:
* Connect to the NN as the current user/group.
* Returns a fs handle on success, or NULL on failure.
*/
hdfsFS doConnectAsUser(const char *hostname, int port) {
hdfsFS doConnectAsUser(const char *nn_uri, int nn_port) {
struct hdfsBuilder *bld;
uid_t uid = fuse_get_context()->uid;
char *user = getUsername(uid);
@ -202,8 +202,10 @@ hdfsFS doConnectAsUser(const char *hostname, int port) {
goto done;
}
hdfsBuilderSetForceNewInstance(bld);
hdfsBuilderSetNameNode(bld, hostname);
hdfsBuilderSetNameNodePort(bld, port);
hdfsBuilderSetNameNode(bld, nn_uri);
if (nn_port) {
hdfsBuilderSetNameNodePort(bld, nn_port);
}
hdfsBuilderSetUserName(bld, user);
if (hdfsAuthConf == AUTH_CONF_KERBEROS) {
findKerbTicketCachePath(kpath, sizeof(kpath));

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_connect.h
View File

@ -21,7 +21,7 @@
#include "fuse_dfs.h"
hdfsFS doConnectAsUser(const char *hostname, int port);
hdfsFS doConnectAsUser(const char *nn_uri, int nn_port);
int doDisconnect(hdfsFS fs);
int allocFsTable(void);

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_context_handle.h
View File

@ -31,7 +31,7 @@
//
typedef struct dfs_context_struct {
int debug;
char *nn_hostname;
char *nn_uri;
int nn_port;
int read_only;
int usetrash;

4
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.c
View File

@ -101,14 +101,14 @@ int main(int argc, char *argv[])
fuse_opt_add_arg(&args, buf);
}
if (options.server == NULL || options.port == 0) {
if (options.nn_uri == NULL) {
print_usage(argv[0]);
exit(0);
}
// Check connection as root
if (options.initchecks == 1) {
hdfsFS tempFS = hdfsConnectAsUser(options.server, options.port, "root");
hdfsFS tempFS = hdfsConnectAsUser(options.nn_uri, options.nn_port, "root");
if (NULL == tempFS) {
const char *cp = getenv("CLASSPATH");
const char *ld = getenv("LD_LIBRARY_PATH");

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_chmod.c
View File

@ -31,7 +31,7 @@ int dfs_chmod(const char *path, mode_t mode)
assert(dfs);
assert('/' == *path);
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) {
ERROR("Could not connect to HDFS");
ret = -EIO;

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_chown.c
View File

@ -54,7 +54,7 @@ int dfs_chown(const char *path, uid_t uid, gid_t gid)
goto cleanup;
}
userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) {
ERROR("Could not connect to HDFS");
ret = -EIO;

4
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_getattr.c
View File

@ -31,9 +31,9 @@ int dfs_getattr(const char *path, struct stat *st)
assert(path);
assert(st);
hdfsFS fs = doConnectAsUser(dfs->nn_hostname,dfs->nn_port);
hdfsFS fs = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (NULL == fs) {
ERROR("Could not connect to %s:%d", dfs->nn_hostname, dfs->nn_port);
ERROR("Could not connect to %s:%d", dfs->nn_uri, dfs->nn_port);
return -EIO;
}

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_mkdir.c
View File

@ -41,7 +41,7 @@ int dfs_mkdir(const char *path, mode_t mode)
return -EACCES;
}
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) {
ERROR("Could not connect");
return -EIO;

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_open.c
View File

@ -45,7 +45,7 @@ int dfs_open(const char *path, struct fuse_file_info *fi)
return -EIO;
}
fh->fs = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
fh->fs = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (fh->fs == NULL) {
ERROR("Could not connect to dfs");
return -EIO;

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_readdir.c
View File

@ -31,7 +31,7 @@ int dfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
assert(path);
assert(buf);
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) {
ERROR("Could not connect");
return -EIO;

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_rename.c
View File

@ -46,7 +46,7 @@ int dfs_rename(const char *from, const char *to)
return -EACCES;
}
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) {
ERROR("Could not connect");
return -EIO;

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_rmdir.c
View File

@ -43,7 +43,7 @@ int dfs_rmdir(const char *path)
return -EACCES;
}
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) {
ERROR("Could not connect");
return -EIO;

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_statfs.c
View File

@ -33,7 +33,7 @@ int dfs_statfs(const char *path, struct statvfs *st)
memset(st,0,sizeof(struct statvfs));
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) {
ERROR("Could not connect");
return -EIO;

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_truncate.c
View File

@ -45,7 +45,7 @@ int dfs_truncate(const char *path, off_t size)
return ret;
}
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) {
ERROR("Could not connect");
ret = -EIO;

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_unlink.c
View File

@ -43,7 +43,7 @@ int dfs_unlink(const char *path)
return -EACCES;
}
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) {
ERROR("Could not connect");
return -EIO;

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_utimens.c
View File

@ -33,7 +33,7 @@ int dfs_utimens(const char *path, const struct timespec ts[2])
time_t aTime = ts[0].tv_sec;
time_t mTime = ts[1].tv_sec;
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) {
ERROR("Could not connect");
return -EIO;

6
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_init.c
View File

@ -92,15 +92,15 @@ void *dfs_init(void) {
// initialize the context
dfs->debug = options.debug;
dfs->nn_hostname = options.server;
dfs->nn_port = options.port;
dfs->nn_uri = options.nn_uri;
dfs->nn_port = options.nn_port;
dfs->read_only = options.read_only;
dfs->usetrash = options.usetrash;
dfs->protectedpaths = NULL;
dfs->rdbuffer_size = options.rdbuffer_size;
dfs->direct_io = options.direct_io;
INFO("Mounting %s:%d", dfs->nn_hostname, dfs->nn_port);
INFO("Mounting. nn_uri=%s, nn_port=%d", dfs->nn_uri, dfs->nn_port);
init_protectedpaths(dfs);
assert(dfs->protectedpaths != NULL);

34
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_options.c
View File

@ -23,6 +23,9 @@
#include <getopt.h>
#include <stdlib.h>
#define OLD_HDFS_URI_LOCATION "dfs://"
#define NEW_HDFS_URI_LOCATION "hdfs://"
void print_options() {
printf("options:\n"
"\tprotected=%s\n"
@ -35,7 +38,7 @@ void print_options() {
"\tattribute_timeout=%d\n"
"\tprivate=%d\n"
"\trdbuffer_size=%d (KBs)\n",
options.protected, options.server, options.port, options.debug,
options.protected, options.nn_uri, options.nn_port, options.debug,
options.read_only, options.usetrash, options.entry_timeout,
options.attribute_timeout, options.private,
(int)options.rdbuffer_size / 1024);
@ -78,11 +81,11 @@ enum
struct fuse_opt dfs_opts[] =
{
DFSFS_OPT_KEY("server=%s", server, 0),
DFSFS_OPT_KEY("server=%s", nn_uri, 0),
DFSFS_OPT_KEY("entry_timeout=%d", entry_timeout, 0),
DFSFS_OPT_KEY("attribute_timeout=%d", attribute_timeout, 0),
DFSFS_OPT_KEY("protected=%s", protected, 0),
DFSFS_OPT_KEY("port=%d", port, 0),
DFSFS_OPT_KEY("port=%d", nn_port, 0),
DFSFS_OPT_KEY("rdbuffer=%d", rdbuffer_size,0),
FUSE_OPT_KEY("private", KEY_PRIVATE),
@ -105,6 +108,7 @@ struct fuse_opt dfs_opts[] =
int dfs_options(void *data, const char *arg, int key, struct fuse_args *outargs)
{
(void) data;
int nn_uri_len;
switch (key) {
case FUSE_OPT_KEY_OPT:
@ -150,11 +154,8 @@ int dfs_options(void *data, const char *arg, int key, struct fuse_args *outargs
#endif
break;
default: {
// try and see if the arg is a URI for DFS
int tmp_port;
char tmp_server[1024];
if (!sscanf(arg,"dfs://%1024[a-zA-Z0-9_.-]:%d",tmp_server,&tmp_port)) {
// try and see if the arg is a URI
if (!strstr(arg, "://")) {
if (strcmp(arg,"ro") == 0) {
options.read_only = 1;
} else if (strcmp(arg,"rw") == 0) {
@ -165,8 +166,21 @@ int dfs_options(void *data, const char *arg, int key, struct fuse_args *outargs
return 0;
}
} else {
options.port = tmp_port;
options.server = strdup(tmp_server);
if (options.nn_uri) {
INFO("Ignoring option %s because '-server' was already "
"specified!", arg);
return 1;
}
if (strstr(arg, OLD_HDFS_URI_LOCATION) == arg) {
// For historical reasons, we let people refer to hdfs:// as dfs://
nn_uri_len = strlen(NEW_HDFS_URI_LOCATION) +
strlen(arg + strlen(OLD_HDFS_URI_LOCATION)) + 1;
options.nn_uri = malloc(nn_uri_len);
snprintf(options.nn_uri, nn_uri_len, "%s%s", NEW_HDFS_URI_LOCATION,
arg + strlen(OLD_HDFS_URI_LOCATION));
} else {
options.nn_uri = strdup(arg);
}
}
}
}

4
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_options.h
View File

@ -22,8 +22,8 @@
/** options for fuse_opt.h */
struct options {
char* protected;
char* server;
int port;
char* nn_uri;
int nn_port;
int debug;
int read_only;
int initchecks;