diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 22a17ed8172..3b10f00c0c7 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -310,6 +310,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-8616. ViewFS configuration requires a trailing slash. (Sandy Ryza via atm) + HADOOP-8756. Fix SEGV when libsnappy is in java.library.path but + not LD_LIBRARY_PATH. (Colin Patrick McCabe via eli) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index 69c52a986a7..bff64d95a4d 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -123,6 +123,7 @@ add_dual_library(hadoop ${D}/security/JniBasedUnixGroupsMapping.c ${D}/security/JniBasedUnixGroupsNetgroupMapping.c ${D}/security/getGroup.c + ${D}/util/NativeCodeLoader.c ${D}/util/NativeCrc32.c ${D}/util/bulk_crc32.c ) diff --git a/hadoop-common-project/hadoop-common/src/config.h.cmake b/hadoop-common-project/hadoop-common/src/config.h.cmake index 9098b68b87e..7423de73a82 100644 --- a/hadoop-common-project/hadoop-common/src/config.h.cmake +++ b/hadoop-common-project/hadoop-common/src/config.h.cmake @@ -2,7 +2,6 @@ #define CONFIG_H #cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@" -#cmakedefine HADOOP_RUNAS_HOME "@HADOOP_RUNAS_HOME@" #cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@" #cmakedefine HAVE_SYNC_FILE_RANGE #cmakedefine HAVE_POSIX_FADVISE diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java index 8a9506448de..8dde9f6c3a5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java @@ -24,7 +24,6 @@ import java.io.OutputStream; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.compress.snappy.LoadSnappy; import org.apache.hadoop.io.compress.snappy.SnappyCompressor; import org.apache.hadoop.io.compress.snappy.SnappyDecompressor; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -34,11 +33,6 @@ import org.apache.hadoop.util.NativeCodeLoader; * This class creates snappy compressors/decompressors. */ public class SnappyCodec implements Configurable, CompressionCodec { - - static { - LoadSnappy.isLoaded(); - } - Configuration conf; /** @@ -63,11 +57,26 @@ public class SnappyCodec implements Configurable, CompressionCodec { /** * Are the native snappy libraries loaded & initialized? - * - * @return true if loaded & initialized, otherwise false */ + public static void checkNativeCodeLoaded() { + if (!NativeCodeLoader.buildSupportsSnappy()) { + throw new RuntimeException("native snappy library not available: " + + "this version of libhadoop was built without " + + "snappy support."); + } + if (!SnappyCompressor.isNativeCodeLoaded()) { + throw new RuntimeException("native snappy library not available: " + + "SnappyCompressor has not been loaded."); + } + if (!SnappyDecompressor.isNativeCodeLoaded()) { + throw new RuntimeException("native snappy library not available: " + + "SnappyDecompressor has not been loaded."); + } + } + public static boolean isNativeCodeLoaded() { - return LoadSnappy.isLoaded() && NativeCodeLoader.isNativeCodeLoaded(); + return SnappyCompressor.isNativeCodeLoaded() && + SnappyDecompressor.isNativeCodeLoaded(); } /** @@ -97,9 +106,7 @@ public class SnappyCodec implements Configurable, CompressionCodec { public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } + checkNativeCodeLoaded(); int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); @@ -117,10 +124,7 @@ public class SnappyCodec implements Configurable, CompressionCodec { */ @Override public Class getCompressorType() { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } - + checkNativeCodeLoaded(); return SnappyCompressor.class; } @@ -131,9 +135,7 @@ public class SnappyCodec implements Configurable, CompressionCodec { */ @Override public Compressor createCompressor() { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } + checkNativeCodeLoaded(); int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); @@ -167,10 +169,7 @@ public class SnappyCodec implements Configurable, CompressionCodec { public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } - + checkNativeCodeLoaded(); return new BlockDecompressorStream(in, decompressor, conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT)); @@ -183,10 +182,7 @@ public class SnappyCodec implements Configurable, CompressionCodec { */ @Override public Class getDecompressorType() { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } - + checkNativeCodeLoaded(); return SnappyDecompressor.class; } @@ -197,9 +193,7 @@ public class SnappyCodec implements Configurable, CompressionCodec { */ @Override public Decompressor createDecompressor() { - if (!isNativeCodeLoaded()) { - throw new RuntimeException("native snappy library not available"); - } + checkNativeCodeLoaded(); int bufferSize = conf.getInt( CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY, CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/LoadSnappy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/LoadSnappy.java deleted file mode 100644 index 05dc984afac..00000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/LoadSnappy.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.io.compress.snappy; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.util.NativeCodeLoader; - -/** - * Determines if Snappy native library is available and loads it if available. - */ -public class LoadSnappy { - private static final Log LOG = LogFactory.getLog(LoadSnappy.class.getName()); - - private static boolean AVAILABLE = false; - private static boolean LOADED = false; - - static { - try { - System.loadLibrary("snappy"); - LOG.warn("Snappy native library is available"); - AVAILABLE = true; - } catch (UnsatisfiedLinkError ex) { - //NOP - } - boolean hadoopNativeAvailable = NativeCodeLoader.isNativeCodeLoaded(); - LOADED = AVAILABLE && hadoopNativeAvailable; - if (LOADED) { - LOG.info("Snappy native library loaded"); - } else { - LOG.warn("Snappy native library not loaded"); - } - } - - /** - * Returns if Snappy native library is loaded. - * - * @return true if Snappy native library is loaded, - * false if not. - */ - public static boolean isAvailable() { - return AVAILABLE; - } - - /** - * Returns if Snappy native library is loaded. - * - * @return true if Snappy native library is loaded, - * false if not. - */ - public static boolean isLoaded() { - return LOADED; - } - -} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java index ba778e0e0bd..c37b97ef830 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.Compressor; +import org.apache.hadoop.util.NativeCodeLoader; /** * A {@link Compressor} based on the snappy compression algorithm. @@ -51,22 +52,24 @@ public class SnappyCompressor implements Compressor { private long bytesRead = 0L; private long bytesWritten = 0L; - + private static boolean nativeSnappyLoaded = false; + static { - if (LoadSnappy.isLoaded()) { - // Initialize the native library + if (NativeCodeLoader.isNativeCodeLoaded() && + NativeCodeLoader.buildSupportsSnappy()) { try { initIDs(); + nativeSnappyLoaded = true; } catch (Throwable t) { - // Ignore failure to load/initialize snappy - LOG.warn(t.toString()); + LOG.error("failed to load SnappyCompressor", t); } - } else { - LOG.error("Cannot load " + SnappyCompressor.class.getName() + - " without snappy library!"); } } - + + public static boolean isNativeCodeLoaded() { + return nativeSnappyLoaded; + } + /** * Creates a new compressor. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java index 4620092f084..b29014e37d8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java @@ -25,6 +25,7 @@ import java.nio.ByteBuffer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.compress.Decompressor; +import org.apache.hadoop.util.NativeCodeLoader; /** * A {@link Decompressor} based on the snappy compression algorithm. @@ -47,21 +48,24 @@ public class SnappyDecompressor implements Decompressor { private int userBufOff = 0, userBufLen = 0; private boolean finished; + private static boolean nativeSnappyLoaded = false; + static { - if (LoadSnappy.isLoaded()) { - // Initialize the native library + if (NativeCodeLoader.isNativeCodeLoaded() && + NativeCodeLoader.buildSupportsSnappy()) { try { initIDs(); + nativeSnappyLoaded = true; } catch (Throwable t) { - // Ignore failure to load/initialize snappy - LOG.warn(t.toString()); + LOG.error("failed to load SnappyDecompressor", t); } - } else { - LOG.error("Cannot load " + SnappyDecompressor.class.getName() + - " without snappy library!"); } } - + + public static boolean isNativeCodeLoaded() { + return nativeSnappyLoaded; + } + /** * Creates a new compressor. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java index dc0c88e29a7..4fe81da1019 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java @@ -74,6 +74,11 @@ public class NativeCodeLoader { return nativeCodeLoaded; } + /** + * Returns true only if this build was compiled with support for snappy. + */ + public static native boolean buildSupportsSnappy(); + /** * Return if native hadoop libraries, if present, can be used for this job. * @param conf configuration diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c new file mode 100644 index 00000000000..4edb1516302 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" + +#include + +JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSupportsSnappy + (JNIEnv *env, jclass clazz) +{ +#ifdef HADOOP_SNAPPY_LIBRARY + return JNI_TRUE; +#else + return JNI_FALSE; +#endif +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java index 5f5fc265864..30a414b8bf2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java @@ -54,7 +54,6 @@ import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.SequenceFile.CompressionType; -import org.apache.hadoop.io.compress.snappy.LoadSnappy; import org.apache.hadoop.io.compress.zlib.BuiltInGzipDecompressor; import org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater; import org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater; @@ -103,14 +102,9 @@ public class TestCodec { @Test public void testSnappyCodec() throws IOException { - if (LoadSnappy.isAvailable()) { - if (LoadSnappy.isLoaded()) { - codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.SnappyCodec"); - codecTest(conf, seed, count, "org.apache.hadoop.io.compress.SnappyCodec"); - } - else { - Assert.fail("Snappy native available but Hadoop native not"); - } + if (SnappyCodec.isNativeCodeLoaded()) { + codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.SnappyCodec"); + codecTest(conf, seed, count, "org.apache.hadoop.io.compress.SnappyCodec"); } }