From 62f557775b90f053a92114f4dc657b8255ace374 Mon Sep 17 00:00:00 2001 From: Luke Lu Date: Thu, 18 Jul 2013 22:44:43 +0000 Subject: [PATCH] HADOOP-9164. Print paths of loaded native libraries in NativeLibraryChecker. (Binglin Chang via llu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1504706 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../apache/hadoop/io/compress/Lz4Codec.java | 4 ++ .../hadoop/io/compress/SnappyCodec.java | 4 ++ .../io/compress/bzip2/Bzip2Compressor.java | 2 + .../io/compress/bzip2/Bzip2Factory.java | 8 ++++ .../hadoop/io/compress/lz4/Lz4Compressor.java | 2 + .../io/compress/snappy/SnappyCompressor.java | 2 + .../io/compress/zlib/ZlibCompressor.java | 2 + .../hadoop/io/compress/zlib/ZlibFactory.java | 6 ++- .../apache/hadoop/util/NativeCodeLoader.java | 2 + .../hadoop/util/NativeLibraryChecker.java | 37 +++++++++++++++---- .../io/compress/bzip2/Bzip2Compressor.c | 15 ++++++++ .../hadoop/io/compress/lz4/Lz4Compressor.c | 6 +++ .../io/compress/snappy/SnappyCompressor.c | 15 ++++++++ .../hadoop/io/compress/zlib/ZlibCompressor.c | 15 ++++++++ .../org/apache/hadoop/util/NativeCodeLoader.c | 13 ++++++- .../hadoop/util/TestNativeCodeLoader.java | 11 ++++++ 17 files changed, 138 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 26f297ad3fc..8135cb10758 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -102,6 +102,9 @@ Release 2.1.0-beta - 2013-07-02 IMPROVEMENTS + HADOOP-9164. Print paths of loaded native libraries in + NativeLibraryChecker. (Binglin Chang via llu) + HADOOP-9253. Capture ulimit info in the logs at service start time. (Arpit Gupta via suresh) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java index 4613ebff40a..e963a95b99f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java @@ -69,6 +69,10 @@ public static boolean isNativeCodeLoaded() { return NativeCodeLoader.isNativeCodeLoaded(); } + public static String getLibraryName() { + return Lz4Compressor.getLibraryName(); + } + /** * Create a {@link CompressionOutputStream} that will write to the given * {@link OutputStream}. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java index 8dde9f6c3a5..cc72f39f688 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java @@ -79,6 +79,10 @@ public static boolean isNativeCodeLoaded() { SnappyDecompressor.isNativeCodeLoaded(); } + public static String getLibraryName() { + return SnappyCompressor.getLibraryName(); + } + /** * Create a {@link CompressionOutputStream} that will write to the given * {@link OutputStream}. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java index 6f502476131..0f333bb36ce 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java @@ -298,4 +298,6 @@ private void checkStream() { private native static long getBytesRead(long strm); private native static long getBytesWritten(long strm); private native static void end(long strm); + + public native static String getLibraryName(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java index 80dc4e93bad..c82ac9939f7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java @@ -78,6 +78,14 @@ public static boolean isNativeBzip2Loaded(Configuration conf) { return nativeBzip2Loaded; } + public static String getLibraryName(Configuration conf) { + if (isNativeBzip2Loaded(conf)) { + return Bzip2Compressor.getLibraryName(); + } else { + return bzip2LibraryName; + } + } + /** * Return the appropriate type of the bzip2 compressor. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java index 63a3afb7d42..5fefcb76db4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java @@ -296,4 +296,6 @@ public synchronized void end() { private native static void initIDs(); private native int compressBytesDirect(); + + public native static String getLibraryName(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java index c37b97ef830..376ea0685b1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java @@ -298,4 +298,6 @@ public synchronized void end() { private native static void initIDs(); private native int compressBytesDirect(); + + public native static String getLibraryName(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java index c0d0d699a55..ce7f68dcc78 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java @@ -444,4 +444,6 @@ private native static void setDictionary(long strm, byte[] b, int off, private native static long getBytesWritten(long strm); private native static void reset(long strm); private native static void end(long strm); + + public native static String getLibraryName(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java index 000518d1459..9a0853a901f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java @@ -65,7 +65,11 @@ public static boolean isNativeZlibLoaded(Configuration conf) { CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT); } - + + public static String getLibraryName() { + return ZlibCompressor.getLibraryName(); + } + /** * Return the appropriate type of the zlib compressor. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java index 4fe81da1019..5667d98b3e7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java @@ -79,6 +79,8 @@ public static boolean isNativeCodeLoaded() { */ public static native boolean buildSupportsSnappy(); + public static native String getLibraryName(); + /** * Return if native hadoop libraries, if present, can be used for this job. * @param conf configuration diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java index f4cf2f53314..84117e2002e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java @@ -20,7 +20,9 @@ import org.apache.hadoop.util.NativeCodeLoader; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.compress.Lz4Codec; import org.apache.hadoop.io.compress.SnappyCodec; +import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; import org.apache.hadoop.io.compress.zlib.ZlibFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -35,7 +37,7 @@ public static void main(String[] args) { String usage = "NativeLibraryChecker [-a|-h]\n" + " -a use -a to check all libraries are available\n" + " by default just check hadoop library is available\n" - + " exit with error code if check failed\n" + + " exit with error code 1 if check failed\n" + " -h print this message\n"; if (args.length > 1 || (args.length == 1 && @@ -51,23 +53,44 @@ public static void main(String[] args) { } checkAll = true; } + Configuration conf = new Configuration(); boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded(); boolean zlibLoaded = false; boolean snappyLoaded = false; // lz4 is linked within libhadoop boolean lz4Loaded = nativeHadoopLoaded; + boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf); + String hadoopLibraryName = ""; + String zlibLibraryName = ""; + String snappyLibraryName = ""; + String lz4LibraryName = ""; + String bzip2LibraryName = ""; if (nativeHadoopLoaded) { - zlibLoaded = ZlibFactory.isNativeZlibLoaded(new Configuration()); + hadoopLibraryName = NativeCodeLoader.getLibraryName(); + zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf); + if (zlibLoaded) { + zlibLibraryName = ZlibFactory.getLibraryName(); + } snappyLoaded = NativeCodeLoader.buildSupportsSnappy() && SnappyCodec.isNativeCodeLoaded(); + if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) { + snappyLibraryName = SnappyCodec.getLibraryName(); + } + if (lz4Loaded) { + lz4LibraryName = Lz4Codec.getLibraryName(); + } + if (bzip2Loaded) { + bzip2LibraryName = Bzip2Factory.getLibraryName(conf); + } } System.out.println("Native library checking:"); - System.out.printf("hadoop: %b\n", nativeHadoopLoaded); - System.out.printf("zlib: %b\n", zlibLoaded); - System.out.printf("snappy: %b\n", snappyLoaded); - System.out.printf("lz4: %b\n", lz4Loaded); + System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName); + System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName); + System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName); + System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName); + System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName); if ((!nativeHadoopLoaded) || - (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded))) { + (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) { // return 1 to indicated check failed ExitUtil.terminate(1); } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c index 8d0b005ab85..d4cd6dfbd24 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c @@ -239,6 +239,21 @@ Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_end( } } +JNIEXPORT jstring JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_getLibraryName(JNIEnv *env, jclass class) { +#ifdef UNIX + if (dlsym_BZ2_bzCompress) { + Dl_info dl_info; + if(dladdr( + dlsym_BZ2_bzCompress, + &dl_info)) { + return (*env)->NewStringUTF(env, dl_info.dli_fname); + } + } +#endif + return (*env)->NewStringUTF(env, HADOOP_BZIP2_LIBRARY); +} + /** * vim: sw=2: ts=2: et: */ diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c index 5b737b52d23..1064d3bd296 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c @@ -103,3 +103,9 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp return (jint)compressed_direct_buf_len; } +JNIEXPORT jstring JNICALL +Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_getLibraryName( + JNIEnv *env, jclass class + ) { + return (*env)->NewStringUTF(env, "revision:43"); +} diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c index f10732f3ab2..247c3dad625 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c @@ -117,4 +117,19 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso return (jint)buf_len; } +JNIEXPORT jstring JNICALL +Java_org_apache_hadoop_io_compress_snappy_SnappyCompressor_getLibraryName(JNIEnv *env, jclass class) { +#ifdef UNIX + if (dlsym_snappy_compress) { + Dl_info dl_info; + if(dladdr( + dlsym_snappy_compress, + &dl_info)) { + return (*env)->NewStringUTF(env, dl_info.dli_fname); + } + } +#endif + return (*env)->NewStringUTF(env, HADOOP_SNAPPY_LIBRARY); +} + #endif //define HADOOP_SNAPPY_LIBRARY diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c index 7298892c1c3..98305c2e710 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c @@ -367,6 +367,21 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_end( } } +JNIEXPORT jstring JNICALL +Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_getLibraryName(JNIEnv *env, jclass class) { +#ifdef UNIX + if (dlsym_deflateInit2_) { + Dl_info dl_info; + if(dladdr( + dlsym_deflateInit2_, + &dl_info)) { + return (*env)->NewStringUTF(env, dl_info.dli_fname); + } + } +#endif + return (*env)->NewStringUTF(env, HADOOP_ZLIB_LIBRARY); +} + /** * vim: sw=2: ts=2: et: */ diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c index 738129e24bf..88d9240c99f 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c @@ -19,6 +19,7 @@ #include "org_apache_hadoop.h" #ifdef UNIX +#include #include "config.h" #endif // UNIX @@ -32,4 +33,14 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup #else return JNI_FALSE; #endif -} \ No newline at end of file +} + +JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName + (JNIEnv *env, jclass clazz) +{ + Dl_info dl_info; + int ret = dladdr( + Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName, + &dl_info); + return (*env)->NewStringUTF(env, ret==0 ? "Unavailable" : dl_info.dli_fname); +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java index e50ae614749..9efaca95c89 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java @@ -22,6 +22,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.compress.Lz4Codec; +import org.apache.hadoop.io.compress.SnappyCodec; +import org.apache.hadoop.io.compress.zlib.ZlibFactory; import org.apache.hadoop.util.NativeCodeLoader; public class TestNativeCodeLoader { @@ -44,6 +47,14 @@ public void testNativeCodeLoaded() { fail("TestNativeCodeLoader: libhadoop.so testing was required, but " + "libhadoop.so was not loaded."); } + assertFalse(NativeCodeLoader.getLibraryName().isEmpty()); + // library names are depended on platform and build envs + // so just check names are available + assertFalse(ZlibFactory.getLibraryName().isEmpty()); + if (NativeCodeLoader.buildSupportsSnappy()) { + assertFalse(SnappyCodec.getLibraryName().isEmpty()); + } + assertFalse(Lz4Codec.getLibraryName().isEmpty()); LOG.info("TestNativeCodeLoader: libhadoop.so is loaded."); } }