HDFS-3609. libhdfs: don't force the URI to look like hdfs://hostname:port. Contributed by Colin Patrick McCabe

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1361455 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -330,6 +330,9 @@ Release 2.0.1-alpha - UNRELEASED
     HDFS-3492. fix some misuses of InputStream#skip.
     (Colin Patrick McCabe via eli)
 
+    HDFS-3609. libhdfs: don't force the URI to look like hdfs://hostname:port.
+    (Colin Patrick McCabe via eli)
+
   BREAKDOWN OF HDFS-3042 SUBTASKS
 
     HDFS-2185. HDFS portion of ZK-based FailoverController (todd)

hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.c

@@ -406,6 +406,57 @@ hdfsFS hdfsConnectAsUserNewInstance(const char* host, tPort port,
     return hdfsBuilderConnect(bld);
 }
 
+
+/**
+ * Calculate the effective URI to use, given a builder configuration.
+ *
+ * If there is not already a URI scheme, we prepend 'hdfs://'.
+ *
+ * If there is not already a port specified, and a port was given to the
+ * builder, we suffix that port.  If there is a port specified but also one in
+ * the URI, that is an error.
+ *
+ * @param bld       The hdfs builder object
+ * @param uri       (out param) dynamically allocated string representing the
+ *                  effective URI
+ *
+ * @return          0 on success; error code otherwise
+ */
+static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri)
+{
+    const char *scheme;
+    char suffix[64];
+    const char *lastColon;
+    char *u;
+    size_t uriLen;
+
+    if (!bld->nn)
+        return EINVAL;
+    scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://";
+    if (bld->port == 0) {
+        suffix[0] = '\0';
+    } else {
+        lastColon = rindex(bld->nn, ':');
+        if (lastColon && (strspn(lastColon + 1, "0123456789") ==
+                          strlen(lastColon + 1))) {
+            fprintf(stderr, "port %d was given, but URI '%s' already "
+                "contains a port!\n", bld->port, bld->nn);
+            return EINVAL;
+        }
+        snprintf(suffix, sizeof(suffix), ":%d", bld->port);
+    }
+
+    uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix);
+    u = malloc((uriLen + 1) * (sizeof(char)));
+    if (!u) {
+        fprintf(stderr, "calcEffectiveURI: out of memory");
+        return ENOMEM;
+    }
+    snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix);
+    *uri = u;
+    return 0;
+}
+
 hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
 {
     JNIEnv *env = 0;
@@ -414,7 +465,6 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
     jstring jURIString = NULL, jUserString = NULL;
     jvalue  jVal;
     jthrowable jExc = NULL;
-    size_t cURILen;
     char *cURI = 0;
     int ret = 0;
 
@@ -472,14 +522,9 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
             jURI = jVal.l;
         } else {
             // fs = FileSystem::get(URI, conf, ugi);
-            cURILen = strlen(bld->nn) + 16;
+            ret = calcEffectiveURI(bld, &cURI);
-            cURI = malloc(cURILen);
+            if (ret)
-            if (!cURI) {
-                fprintf(stderr, "failed to allocate memory for HDFS URI\n");
-                ret = ENOMEM;			
                 goto done;
-            }
-            snprintf(cURI, cURILen, "hdfs://%s:%d", bld->nn, (int)(bld->port));
             jURIString = (*env)->NewStringUTF(env, cURI);
             if (invokeMethod(env, &jVal, &jExc, STATIC, NULL, JAVA_NET_URI,
                              "create", "(Ljava/lang/String;)Ljava/net/URI;",

hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.h

@@ -175,11 +175,21 @@ extern  "C" {
      *
      * @param bld  The HDFS builder
      * @param nn   The NameNode to use.
+     *
      *             If the string given is 'default', the default NameNode
      *             configuration will be used (from the XML configuration files)
+     *
      *             If NULL is given, a LocalFileSystem will be created.
-     *             Otherwise, the string will be interpreted as a hostname or IP
+     *
-     *             address.
+     *             If the string starts with a protocol type such as file:// or
+     *             hdfs://, this protocol type will be used.  If not, the
+     *             hdfs:// protocol type will be used.
+     *
+     *             You may specify a NameNode port in the usual way by 
+     *             passing a string of the format hdfs://<hostname>:<port>.
+     *             Alternately, you may set the port with
+     *             hdfsBuilderSetNameNodePort.  However, you must not pass the
+     *             port in two different ways.
      */
     void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn);