HADOOP-9164. Print paths of loaded native libraries in NativeLibraryChecker. (Binglin Chang via llu)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1504706 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Luke Lu
parent
bddf1659f8
commit
62f557775b
|
|
@ -102,6 +102,9 @@ Release 2.1.0-beta - 2013-07-02
|
|||
|
||||
IMPROVEMENTS
|
||||
|
||||
HADOOP-9164. Print paths of loaded native libraries in
|
||||
NativeLibraryChecker. (Binglin Chang via llu)
|
||||
|
||||
HADOOP-9253. Capture ulimit info in the logs at service start time.
|
||||
(Arpit Gupta via suresh)
|
||||
|
||||
|
|
|
|||
|
|
@ -69,6 +69,10 @@ public class Lz4Codec implements Configurable, CompressionCodec {
|
|||
return NativeCodeLoader.isNativeCodeLoaded();
|
||||
}
|
||||
|
||||
public static String getLibraryName() {
|
||||
return Lz4Compressor.getLibraryName();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a {@link CompressionOutputStream} that will write to the given
|
||||
* {@link OutputStream}.
|
||||
|
|
|
|||
|
|
@ -79,6 +79,10 @@ public class SnappyCodec implements Configurable, CompressionCodec {
|
|||
SnappyDecompressor.isNativeCodeLoaded();
|
||||
}
|
||||
|
||||
public static String getLibraryName() {
|
||||
return SnappyCompressor.getLibraryName();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a {@link CompressionOutputStream} that will write to the given
|
||||
* {@link OutputStream}.
|
||||
|
|
|
|||
|
|
@ -298,4 +298,6 @@ public class Bzip2Compressor implements Compressor {
|
|||
private native static long getBytesRead(long strm);
|
||||
private native static long getBytesWritten(long strm);
|
||||
private native static void end(long strm);
|
||||
|
||||
public native static String getLibraryName();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,14 @@ public class Bzip2Factory {
|
|||
return nativeBzip2Loaded;
|
||||
}
|
||||
|
||||
public static String getLibraryName(Configuration conf) {
|
||||
if (isNativeBzip2Loaded(conf)) {
|
||||
return Bzip2Compressor.getLibraryName();
|
||||
} else {
|
||||
return bzip2LibraryName;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the appropriate type of the bzip2 compressor.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -296,4 +296,6 @@ public class Lz4Compressor implements Compressor {
|
|||
private native static void initIDs();
|
||||
|
||||
private native int compressBytesDirect();
|
||||
|
||||
public native static String getLibraryName();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -298,4 +298,6 @@ public class SnappyCompressor implements Compressor {
|
|||
private native static void initIDs();
|
||||
|
||||
private native int compressBytesDirect();
|
||||
|
||||
public native static String getLibraryName();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -444,4 +444,6 @@ public class ZlibCompressor implements Compressor {
|
|||
private native static long getBytesWritten(long strm);
|
||||
private native static void reset(long strm);
|
||||
private native static void end(long strm);
|
||||
|
||||
public native static String getLibraryName();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,7 +65,11 @@ public class ZlibFactory {
|
|||
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
|
||||
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT);
|
||||
}
|
||||
|
||||
|
||||
public static String getLibraryName() {
|
||||
return ZlibCompressor.getLibraryName();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the appropriate type of the zlib compressor.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -79,6 +79,8 @@ public class NativeCodeLoader {
|
|||
*/
|
||||
public static native boolean buildSupportsSnappy();
|
||||
|
||||
public static native String getLibraryName();
|
||||
|
||||
/**
|
||||
* Return if native hadoop libraries, if present, can be used for this job.
|
||||
* @param conf configuration
|
||||
|
|
|
|||
|
|
@ -20,7 +20,9 @@ package org.apache.hadoop.util;
|
|||
|
||||
import org.apache.hadoop.util.NativeCodeLoader;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.io.compress.Lz4Codec;
|
||||
import org.apache.hadoop.io.compress.SnappyCodec;
|
||||
import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
|
||||
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
|
|
@ -35,7 +37,7 @@ public class NativeLibraryChecker {
|
|||
String usage = "NativeLibraryChecker [-a|-h]\n"
|
||||
+ " -a use -a to check all libraries are available\n"
|
||||
+ " by default just check hadoop library is available\n"
|
||||
+ " exit with error code if check failed\n"
|
||||
+ " exit with error code 1 if check failed\n"
|
||||
+ " -h print this message\n";
|
||||
if (args.length > 1 ||
|
||||
(args.length == 1 &&
|
||||
|
|
@ -51,23 +53,44 @@ public class NativeLibraryChecker {
|
|||
}
|
||||
checkAll = true;
|
||||
}
|
||||
Configuration conf = new Configuration();
|
||||
boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded();
|
||||
boolean zlibLoaded = false;
|
||||
boolean snappyLoaded = false;
|
||||
// lz4 is linked within libhadoop
|
||||
boolean lz4Loaded = nativeHadoopLoaded;
|
||||
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
|
||||
String hadoopLibraryName = "";
|
||||
String zlibLibraryName = "";
|
||||
String snappyLibraryName = "";
|
||||
String lz4LibraryName = "";
|
||||
String bzip2LibraryName = "";
|
||||
if (nativeHadoopLoaded) {
|
||||
zlibLoaded = ZlibFactory.isNativeZlibLoaded(new Configuration());
|
||||
hadoopLibraryName = NativeCodeLoader.getLibraryName();
|
||||
zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf);
|
||||
if (zlibLoaded) {
|
||||
zlibLibraryName = ZlibFactory.getLibraryName();
|
||||
}
|
||||
snappyLoaded = NativeCodeLoader.buildSupportsSnappy() &&
|
||||
SnappyCodec.isNativeCodeLoaded();
|
||||
if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
|
||||
snappyLibraryName = SnappyCodec.getLibraryName();
|
||||
}
|
||||
if (lz4Loaded) {
|
||||
lz4LibraryName = Lz4Codec.getLibraryName();
|
||||
}
|
||||
if (bzip2Loaded) {
|
||||
bzip2LibraryName = Bzip2Factory.getLibraryName(conf);
|
||||
}
|
||||
}
|
||||
System.out.println("Native library checking:");
|
||||
System.out.printf("hadoop: %b\n", nativeHadoopLoaded);
|
||||
System.out.printf("zlib: %b\n", zlibLoaded);
|
||||
System.out.printf("snappy: %b\n", snappyLoaded);
|
||||
System.out.printf("lz4: %b\n", lz4Loaded);
|
||||
System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
|
||||
System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
|
||||
System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
|
||||
System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
|
||||
System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
|
||||
if ((!nativeHadoopLoaded) ||
|
||||
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded))) {
|
||||
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
|
||||
// return 1 to indicated check failed
|
||||
ExitUtil.terminate(1);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -239,6 +239,21 @@ Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_end(
|
|||
}
|
||||
}
|
||||
|
||||
JNIEXPORT jstring JNICALL
|
||||
Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_getLibraryName(JNIEnv *env, jclass class) {
|
||||
#ifdef UNIX
|
||||
if (dlsym_BZ2_bzCompress) {
|
||||
Dl_info dl_info;
|
||||
if(dladdr(
|
||||
dlsym_BZ2_bzCompress,
|
||||
&dl_info)) {
|
||||
return (*env)->NewStringUTF(env, dl_info.dli_fname);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return (*env)->NewStringUTF(env, HADOOP_BZIP2_LIBRARY);
|
||||
}
|
||||
|
||||
/**
|
||||
* vim: sw=2: ts=2: et:
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -103,3 +103,9 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp
|
|||
return (jint)compressed_direct_buf_len;
|
||||
}
|
||||
|
||||
JNIEXPORT jstring JNICALL
|
||||
Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_getLibraryName(
|
||||
JNIEnv *env, jclass class
|
||||
) {
|
||||
return (*env)->NewStringUTF(env, "revision:43");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -117,4 +117,19 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso
|
|||
return (jint)buf_len;
|
||||
}
|
||||
|
||||
JNIEXPORT jstring JNICALL
|
||||
Java_org_apache_hadoop_io_compress_snappy_SnappyCompressor_getLibraryName(JNIEnv *env, jclass class) {
|
||||
#ifdef UNIX
|
||||
if (dlsym_snappy_compress) {
|
||||
Dl_info dl_info;
|
||||
if(dladdr(
|
||||
dlsym_snappy_compress,
|
||||
&dl_info)) {
|
||||
return (*env)->NewStringUTF(env, dl_info.dli_fname);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return (*env)->NewStringUTF(env, HADOOP_SNAPPY_LIBRARY);
|
||||
}
|
||||
|
||||
#endif //define HADOOP_SNAPPY_LIBRARY
|
||||
|
|
|
|||
|
|
@ -367,6 +367,21 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_end(
|
|||
}
|
||||
}
|
||||
|
||||
JNIEXPORT jstring JNICALL
|
||||
Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_getLibraryName(JNIEnv *env, jclass class) {
|
||||
#ifdef UNIX
|
||||
if (dlsym_deflateInit2_) {
|
||||
Dl_info dl_info;
|
||||
if(dladdr(
|
||||
dlsym_deflateInit2_,
|
||||
&dl_info)) {
|
||||
return (*env)->NewStringUTF(env, dl_info.dli_fname);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return (*env)->NewStringUTF(env, HADOOP_ZLIB_LIBRARY);
|
||||
}
|
||||
|
||||
/**
|
||||
* vim: sw=2: ts=2: et:
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
#include "org_apache_hadoop.h"
|
||||
|
||||
#ifdef UNIX
|
||||
#include <dlfcn.h>
|
||||
#include "config.h"
|
||||
#endif // UNIX
|
||||
|
||||
|
|
@ -32,4 +33,14 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup
|
|||
#else
|
||||
return JNI_FALSE;
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName
|
||||
(JNIEnv *env, jclass clazz)
|
||||
{
|
||||
Dl_info dl_info;
|
||||
int ret = dladdr(
|
||||
Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName,
|
||||
&dl_info);
|
||||
return (*env)->NewStringUTF(env, ret==0 ? "Unavailable" : dl_info.dli_fname);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ import static org.junit.Assert.*;
|
|||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.io.compress.Lz4Codec;
|
||||
import org.apache.hadoop.io.compress.SnappyCodec;
|
||||
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
|
||||
import org.apache.hadoop.util.NativeCodeLoader;
|
||||
|
||||
public class TestNativeCodeLoader {
|
||||
|
|
@ -44,6 +47,14 @@ public class TestNativeCodeLoader {
|
|||
fail("TestNativeCodeLoader: libhadoop.so testing was required, but " +
|
||||
"libhadoop.so was not loaded.");
|
||||
}
|
||||
assertFalse(NativeCodeLoader.getLibraryName().isEmpty());
|
||||
// library names are depended on platform and build envs
|
||||
// so just check names are available
|
||||
assertFalse(ZlibFactory.getLibraryName().isEmpty());
|
||||
if (NativeCodeLoader.buildSupportsSnappy()) {
|
||||
assertFalse(SnappyCodec.getLibraryName().isEmpty());
|
||||
}
|
||||
assertFalse(Lz4Codec.getLibraryName().isEmpty());
|
||||
LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue