HDFS-3610. fuse_dfs: Provide a way to use the default (configured) NN URI. Contributed by Colin Patrick McCabe

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1361566 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-07-14 17:19:26 +00:00
parent d03fb9c0d2
commit 30d0104c0e
20 changed files with 54 additions and 35 deletions

View File

@ -145,6 +145,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3539. libhdfs code cleanups. (Colin Patrick McCabe via eli) HDFS-3539. libhdfs code cleanups. (Colin Patrick McCabe via eli)
HDFS-3610. fuse_dfs: Provide a way to use the default (configured) NN URI.
(Colin Patrick McCabe via eli)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-2982. Startup performance suffers when there are many edit log HDFS-2982. Startup performance suffers when there are many edit log

View File

@ -172,7 +172,7 @@ done:
* Connect to the NN as the current user/group. * Connect to the NN as the current user/group.
* Returns a fs handle on success, or NULL on failure. * Returns a fs handle on success, or NULL on failure.
*/ */
hdfsFS doConnectAsUser(const char *hostname, int port) { hdfsFS doConnectAsUser(const char *nn_uri, int nn_port) {
struct hdfsBuilder *bld; struct hdfsBuilder *bld;
uid_t uid = fuse_get_context()->uid; uid_t uid = fuse_get_context()->uid;
char *user = getUsername(uid); char *user = getUsername(uid);
@ -202,8 +202,10 @@ hdfsFS doConnectAsUser(const char *hostname, int port) {
goto done; goto done;
} }
hdfsBuilderSetForceNewInstance(bld); hdfsBuilderSetForceNewInstance(bld);
hdfsBuilderSetNameNode(bld, hostname); hdfsBuilderSetNameNode(bld, nn_uri);
hdfsBuilderSetNameNodePort(bld, port); if (nn_port) {
hdfsBuilderSetNameNodePort(bld, nn_port);
}
hdfsBuilderSetUserName(bld, user); hdfsBuilderSetUserName(bld, user);
if (hdfsAuthConf == AUTH_CONF_KERBEROS) { if (hdfsAuthConf == AUTH_CONF_KERBEROS) {
findKerbTicketCachePath(kpath, sizeof(kpath)); findKerbTicketCachePath(kpath, sizeof(kpath));

View File

@ -21,7 +21,7 @@
#include "fuse_dfs.h" #include "fuse_dfs.h"
hdfsFS doConnectAsUser(const char *hostname, int port); hdfsFS doConnectAsUser(const char *nn_uri, int nn_port);
int doDisconnect(hdfsFS fs); int doDisconnect(hdfsFS fs);
int allocFsTable(void); int allocFsTable(void);

View File

@ -31,7 +31,7 @@
// //
typedef struct dfs_context_struct { typedef struct dfs_context_struct {
int debug; int debug;
char *nn_hostname; char *nn_uri;
int nn_port; int nn_port;
int read_only; int read_only;
int usetrash; int usetrash;

View File

@ -101,14 +101,14 @@ int main(int argc, char *argv[])
fuse_opt_add_arg(&args, buf); fuse_opt_add_arg(&args, buf);
} }
if (options.server == NULL || options.port == 0) { if (options.nn_uri == NULL) {
print_usage(argv[0]); print_usage(argv[0]);
exit(0); exit(0);
} }
// Check connection as root // Check connection as root
if (options.initchecks == 1) { if (options.initchecks == 1) {
hdfsFS tempFS = hdfsConnectAsUser(options.server, options.port, "root"); hdfsFS tempFS = hdfsConnectAsUser(options.nn_uri, options.nn_port, "root");
if (NULL == tempFS) { if (NULL == tempFS) {
const char *cp = getenv("CLASSPATH"); const char *cp = getenv("CLASSPATH");
const char *ld = getenv("LD_LIBRARY_PATH"); const char *ld = getenv("LD_LIBRARY_PATH");

View File

@ -31,7 +31,7 @@ int dfs_chmod(const char *path, mode_t mode)
assert(dfs); assert(dfs);
assert('/' == *path); assert('/' == *path);
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port); hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) { if (userFS == NULL) {
ERROR("Could not connect to HDFS"); ERROR("Could not connect to HDFS");
ret = -EIO; ret = -EIO;

View File

@ -54,7 +54,7 @@ int dfs_chown(const char *path, uid_t uid, gid_t gid)
goto cleanup; goto cleanup;
} }
userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port); userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) { if (userFS == NULL) {
ERROR("Could not connect to HDFS"); ERROR("Could not connect to HDFS");
ret = -EIO; ret = -EIO;

View File

@ -31,9 +31,9 @@ int dfs_getattr(const char *path, struct stat *st)
assert(path); assert(path);
assert(st); assert(st);
hdfsFS fs = doConnectAsUser(dfs->nn_hostname,dfs->nn_port); hdfsFS fs = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (NULL == fs) { if (NULL == fs) {
ERROR("Could not connect to %s:%d", dfs->nn_hostname, dfs->nn_port); ERROR("Could not connect to %s:%d", dfs->nn_uri, dfs->nn_port);
return -EIO; return -EIO;
} }

View File

@ -41,7 +41,7 @@ int dfs_mkdir(const char *path, mode_t mode)
return -EACCES; return -EACCES;
} }
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port); hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) { if (userFS == NULL) {
ERROR("Could not connect"); ERROR("Could not connect");
return -EIO; return -EIO;

View File

@ -45,7 +45,7 @@ int dfs_open(const char *path, struct fuse_file_info *fi)
return -EIO; return -EIO;
} }
fh->fs = doConnectAsUser(dfs->nn_hostname, dfs->nn_port); fh->fs = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (fh->fs == NULL) { if (fh->fs == NULL) {
ERROR("Could not connect to dfs"); ERROR("Could not connect to dfs");
return -EIO; return -EIO;

View File

@ -31,7 +31,7 @@ int dfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
assert(path); assert(path);
assert(buf); assert(buf);
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port); hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) { if (userFS == NULL) {
ERROR("Could not connect"); ERROR("Could not connect");
return -EIO; return -EIO;

View File

@ -46,7 +46,7 @@ int dfs_rename(const char *from, const char *to)
return -EACCES; return -EACCES;
} }
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port); hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) { if (userFS == NULL) {
ERROR("Could not connect"); ERROR("Could not connect");
return -EIO; return -EIO;

View File

@ -43,7 +43,7 @@ int dfs_rmdir(const char *path)
return -EACCES; return -EACCES;
} }
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port); hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) { if (userFS == NULL) {
ERROR("Could not connect"); ERROR("Could not connect");
return -EIO; return -EIO;

View File

@ -33,7 +33,7 @@ int dfs_statfs(const char *path, struct statvfs *st)
memset(st,0,sizeof(struct statvfs)); memset(st,0,sizeof(struct statvfs));
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port); hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) { if (userFS == NULL) {
ERROR("Could not connect"); ERROR("Could not connect");
return -EIO; return -EIO;

View File

@ -45,7 +45,7 @@ int dfs_truncate(const char *path, off_t size)
return ret; return ret;
} }
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port); hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) { if (userFS == NULL) {
ERROR("Could not connect"); ERROR("Could not connect");
ret = -EIO; ret = -EIO;

View File

@ -43,7 +43,7 @@ int dfs_unlink(const char *path)
return -EACCES; return -EACCES;
} }
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port); hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) { if (userFS == NULL) {
ERROR("Could not connect"); ERROR("Could not connect");
return -EIO; return -EIO;

View File

@ -33,7 +33,7 @@ int dfs_utimens(const char *path, const struct timespec ts[2])
time_t aTime = ts[0].tv_sec; time_t aTime = ts[0].tv_sec;
time_t mTime = ts[1].tv_sec; time_t mTime = ts[1].tv_sec;
hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port); hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
if (userFS == NULL) { if (userFS == NULL) {
ERROR("Could not connect"); ERROR("Could not connect");
return -EIO; return -EIO;

View File

@ -92,15 +92,15 @@ void *dfs_init(void) {
// initialize the context // initialize the context
dfs->debug = options.debug; dfs->debug = options.debug;
dfs->nn_hostname = options.server; dfs->nn_uri = options.nn_uri;
dfs->nn_port = options.port; dfs->nn_port = options.nn_port;
dfs->read_only = options.read_only; dfs->read_only = options.read_only;
dfs->usetrash = options.usetrash; dfs->usetrash = options.usetrash;
dfs->protectedpaths = NULL; dfs->protectedpaths = NULL;
dfs->rdbuffer_size = options.rdbuffer_size; dfs->rdbuffer_size = options.rdbuffer_size;
dfs->direct_io = options.direct_io; dfs->direct_io = options.direct_io;
INFO("Mounting %s:%d", dfs->nn_hostname, dfs->nn_port); INFO("Mounting. nn_uri=%s, nn_port=%d", dfs->nn_uri, dfs->nn_port);
init_protectedpaths(dfs); init_protectedpaths(dfs);
assert(dfs->protectedpaths != NULL); assert(dfs->protectedpaths != NULL);

View File

@ -23,6 +23,9 @@
#include <getopt.h> #include <getopt.h>
#include <stdlib.h> #include <stdlib.h>
#define OLD_HDFS_URI_LOCATION "dfs://"
#define NEW_HDFS_URI_LOCATION "hdfs://"
void print_options() { void print_options() {
printf("options:\n" printf("options:\n"
"\tprotected=%s\n" "\tprotected=%s\n"
@ -35,7 +38,7 @@ void print_options() {
"\tattribute_timeout=%d\n" "\tattribute_timeout=%d\n"
"\tprivate=%d\n" "\tprivate=%d\n"
"\trdbuffer_size=%d (KBs)\n", "\trdbuffer_size=%d (KBs)\n",
options.protected, options.server, options.port, options.debug, options.protected, options.nn_uri, options.nn_port, options.debug,
options.read_only, options.usetrash, options.entry_timeout, options.read_only, options.usetrash, options.entry_timeout,
options.attribute_timeout, options.private, options.attribute_timeout, options.private,
(int)options.rdbuffer_size / 1024); (int)options.rdbuffer_size / 1024);
@ -78,11 +81,11 @@ enum
struct fuse_opt dfs_opts[] = struct fuse_opt dfs_opts[] =
{ {
DFSFS_OPT_KEY("server=%s", server, 0), DFSFS_OPT_KEY("server=%s", nn_uri, 0),
DFSFS_OPT_KEY("entry_timeout=%d", entry_timeout, 0), DFSFS_OPT_KEY("entry_timeout=%d", entry_timeout, 0),
DFSFS_OPT_KEY("attribute_timeout=%d", attribute_timeout, 0), DFSFS_OPT_KEY("attribute_timeout=%d", attribute_timeout, 0),
DFSFS_OPT_KEY("protected=%s", protected, 0), DFSFS_OPT_KEY("protected=%s", protected, 0),
DFSFS_OPT_KEY("port=%d", port, 0), DFSFS_OPT_KEY("port=%d", nn_port, 0),
DFSFS_OPT_KEY("rdbuffer=%d", rdbuffer_size,0), DFSFS_OPT_KEY("rdbuffer=%d", rdbuffer_size,0),
FUSE_OPT_KEY("private", KEY_PRIVATE), FUSE_OPT_KEY("private", KEY_PRIVATE),
@ -105,6 +108,7 @@ struct fuse_opt dfs_opts[] =
int dfs_options(void *data, const char *arg, int key, struct fuse_args *outargs) int dfs_options(void *data, const char *arg, int key, struct fuse_args *outargs)
{ {
(void) data; (void) data;
int nn_uri_len;
switch (key) { switch (key) {
case FUSE_OPT_KEY_OPT: case FUSE_OPT_KEY_OPT:
@ -150,11 +154,8 @@ int dfs_options(void *data, const char *arg, int key, struct fuse_args *outargs
#endif #endif
break; break;
default: { default: {
// try and see if the arg is a URI for DFS // try and see if the arg is a URI
int tmp_port; if (!strstr(arg, "://")) {
char tmp_server[1024];
if (!sscanf(arg,"dfs://%1024[a-zA-Z0-9_.-]:%d",tmp_server,&tmp_port)) {
if (strcmp(arg,"ro") == 0) { if (strcmp(arg,"ro") == 0) {
options.read_only = 1; options.read_only = 1;
} else if (strcmp(arg,"rw") == 0) { } else if (strcmp(arg,"rw") == 0) {
@ -165,8 +166,21 @@ int dfs_options(void *data, const char *arg, int key, struct fuse_args *outargs
return 0; return 0;
} }
} else { } else {
options.port = tmp_port; if (options.nn_uri) {
options.server = strdup(tmp_server); INFO("Ignoring option %s because '-server' was already "
"specified!", arg);
return 1;
}
if (strstr(arg, OLD_HDFS_URI_LOCATION) == arg) {
// For historical reasons, we let people refer to hdfs:// as dfs://
nn_uri_len = strlen(NEW_HDFS_URI_LOCATION) +
strlen(arg + strlen(OLD_HDFS_URI_LOCATION)) + 1;
options.nn_uri = malloc(nn_uri_len);
snprintf(options.nn_uri, nn_uri_len, "%s%s", NEW_HDFS_URI_LOCATION,
arg + strlen(OLD_HDFS_URI_LOCATION));
} else {
options.nn_uri = strdup(arg);
}
} }
} }
} }

View File

@ -22,8 +22,8 @@
/** options for fuse_opt.h */ /** options for fuse_opt.h */
struct options { struct options {
char* protected; char* protected;
char* server; char* nn_uri;
int port; int nn_port;
int debug; int debug;
int read_only; int read_only;
int initchecks; int initchecks;