diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9f48c7686b2..9c1be4a9c67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -653,6 +653,9 @@ Release 2.7.0 - UNRELEASED HDFS-7561. TestFetchImage should write fetched-image-dir under target. (Liang Xie via shv) + HDFS-7589. Break the dependency between libnative_mini_dfs and libhdfs. + (Zhanwei Wang via cnauroth) + Release 2.6.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt index 227be45da5e..8da49c01163 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt @@ -172,9 +172,15 @@ target_link_libraries(test_libhdfs_write add_library(native_mini_dfs main/native/libhdfs/native_mini_dfs.c + main/native/libhdfs/common/htable.c + main/native/libhdfs/exception.c + main/native/libhdfs/jni_helper.c + ${OS_DIR}/mutexes.c + ${OS_DIR}/thread_local_storage.c ) target_link_libraries(native_mini_dfs - hdfs + ${JAVA_JVM_LIBRARY} + ${OS_LINK_LIBRARIES} ) add_executable(test_native_mini_dfs diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c index 2c42fa5f204..b37ebcce0b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c @@ -17,8 +17,6 @@ */ #include "exception.h" -#include "hdfs.h" -#include "hdfs_test.h" #include "jni_helper.h" #include "native_mini_dfs.h" #include "platform.h" @@ -32,6 +30,10 @@ #include #include +#ifndef EINTERNAL +#define EINTERNAL 255 +#endif + #define MINIDFS_CLUSTER_BUILDER "org/apache/hadoop/hdfs/MiniDFSCluster$Builder" #define MINIDFS_CLUSTER "org/apache/hadoop/hdfs/MiniDFSCluster" #define HADOOP_CONF "org/apache/hadoop/conf/Configuration" @@ -52,6 +54,25 @@ struct NativeMiniDfsCluster { char domainSocketPath[PATH_MAX]; }; +static int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + static jthrowable nmdConfigureShortCircuit(JNIEnv *env, struct NativeMiniDfsCluster *cl, jobject cobj) { @@ -345,29 +366,10 @@ error_dlr_nn: return ret; } -int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl, - struct hdfsBuilder *bld) -{ - int ret; - tPort port; - - hdfsBuilderSetNameNode(bld, "localhost"); - port = (tPort)nmdGetNameNodePort(cl); - if (port < 0) { - fprintf(stderr, "nmdGetNameNodePort failed with error %d\n", -port); - return EIO; - } - hdfsBuilderSetNameNodePort(bld, port); +const char *hdfsGetDomainSocketPath(const struct NativeMiniDfsCluster *cl) { if (cl->domainSocketPath[0]) { - ret = hdfsBuilderConfSetStr(bld, "dfs.client.read.shortcircuit", "true"); - if (ret) { - return ret; - } - ret = hdfsBuilderConfSetStr(bld, "dfs.domain.socket.path", - cl->domainSocketPath); - if (ret) { - return ret; - } + return cl->domainSocketPath; } - return 0; + + return NULL; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h index 41d69c2966a..ce8b1cfdab0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h @@ -21,6 +21,10 @@ #include /* for jboolean */ +#ifdef __cplusplus +extern "C" { +#endif + struct hdfsBuilder; struct NativeMiniDfsCluster; @@ -110,13 +114,16 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl, int *port, const char **hostName); /** - * Configure the HDFS builder appropriately to connect to this cluster. + * Get domain socket path set for this cluster. * - * @param bld The hdfs builder + * @param cl The cluster * - * @return the port, or a negative error code + * @return A const string of domain socket path, or NULL if not set. */ -int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl, - struct hdfsBuilder *bld); +const char *hdfsGetDomainSocketPath(const struct NativeMiniDfsCluster *cl); + +#ifdef __cplusplus +} +#endif #endif diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c index 3774417c837..92941cfb355 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c @@ -201,6 +201,37 @@ static int createZeroCopyTestFile(hdfsFS fs, char *testFileName, return 0; } +static int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl, + struct hdfsBuilder *bld) { + int ret; + tPort port; + const char *domainSocket; + + hdfsBuilderSetNameNode(bld, "localhost"); + port = (tPort) nmdGetNameNodePort(cl); + if (port < 0) { + fprintf(stderr, "nmdGetNameNodePort failed with error %d\n", -port); + return EIO; + } + hdfsBuilderSetNameNodePort(bld, port); + + domainSocket = hdfsGetDomainSocketPath(cl); + + if (domainSocket) { + ret = hdfsBuilderConfSetStr(bld, "dfs.client.read.shortcircuit", "true"); + if (ret) { + return ret; + } + ret = hdfsBuilderConfSetStr(bld, "dfs.domain.socket.path", + domainSocket); + if (ret) { + return ret; + } + } + return 0; +} + + /** * Test that we can write a file with libhdfs and then read it back */