HBASE-27872 xerial's snappy-java requires GLIBC >= 2.32 (#5245)
We need to add a native library load check with a helpful error message if xerial snappy fails to initialize due to a too old glibc or similar reason, and disable the unit test if the native library fails to load. Signed-off-by: Duo Zhang <zhangduo@apache.org> Signed-off-by: Viraj Jasani <vjasani@apache.org>
This commit is contained in:
parent
6a5a710e31
commit
f347867440
|
@ -31,6 +31,8 @@ import org.apache.hadoop.io.compress.CompressionOutputStream;
|
||||||
import org.apache.hadoop.io.compress.Compressor;
|
import org.apache.hadoop.io.compress.Compressor;
|
||||||
import org.apache.hadoop.io.compress.Decompressor;
|
import org.apache.hadoop.io.compress.Decompressor;
|
||||||
import org.apache.yetus.audience.InterfaceAudience;
|
import org.apache.yetus.audience.InterfaceAudience;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
import org.xerial.snappy.Snappy;
|
import org.xerial.snappy.Snappy;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -43,10 +45,31 @@ public class SnappyCodec implements Configurable, CompressionCodec {
|
||||||
|
|
||||||
public static final String SNAPPY_BUFFER_SIZE_KEY = "hbase.io.compress.snappy.buffersize";
|
public static final String SNAPPY_BUFFER_SIZE_KEY = "hbase.io.compress.snappy.buffersize";
|
||||||
|
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(SnappyCodec.class);
|
||||||
private Configuration conf;
|
private Configuration conf;
|
||||||
private int bufferSize;
|
private int bufferSize;
|
||||||
|
private static boolean loaded = false;
|
||||||
|
private static Throwable loadError;
|
||||||
|
|
||||||
|
static {
|
||||||
|
try {
|
||||||
|
Snappy.getNativeLibraryVersion();
|
||||||
|
loaded = true;
|
||||||
|
} catch (Throwable t) {
|
||||||
|
loadError = t;
|
||||||
|
LOG.error("The Snappy native libraries could not be loaded", t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Return true if the native shared libraries were loaded; false otherwise. */
|
||||||
|
public static boolean isLoaded() {
|
||||||
|
return loaded;
|
||||||
|
}
|
||||||
|
|
||||||
public SnappyCodec() {
|
public SnappyCodec() {
|
||||||
|
if (!isLoaded()) {
|
||||||
|
throw new RuntimeException("Snappy codec could not be loaded", loadError);
|
||||||
|
}
|
||||||
conf = new Configuration();
|
conf = new Configuration();
|
||||||
bufferSize = getBufferSize(conf);
|
bufferSize = getBufferSize(conf);
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,6 +17,8 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hbase.io.compress.xerial;
|
package org.apache.hadoop.hbase.io.compress.xerial;
|
||||||
|
|
||||||
|
import static org.junit.Assume.assumeTrue;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||||
|
@ -41,6 +43,7 @@ public class TestHFileCompressionSnappy extends HFileTestBase {
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void setUpBeforeClass() throws Exception {
|
public static void setUpBeforeClass() throws Exception {
|
||||||
|
assumeTrue(SnappyCodec.isLoaded());
|
||||||
conf = TEST_UTIL.getConfiguration();
|
conf = TEST_UTIL.getConfiguration();
|
||||||
conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName());
|
conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName());
|
||||||
Compression.Algorithm.SNAPPY.reload(conf);
|
Compression.Algorithm.SNAPPY.reload(conf);
|
||||||
|
|
|
@ -17,9 +17,12 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hbase.io.compress.xerial;
|
package org.apache.hadoop.hbase.io.compress.xerial;
|
||||||
|
|
||||||
|
import static org.junit.Assume.assumeTrue;
|
||||||
|
|
||||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||||
import org.apache.hadoop.hbase.io.compress.CompressionTestBase;
|
import org.apache.hadoop.hbase.io.compress.CompressionTestBase;
|
||||||
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
import org.junit.ClassRule;
|
import org.junit.ClassRule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.experimental.categories.Category;
|
import org.junit.experimental.categories.Category;
|
||||||
|
@ -31,6 +34,11 @@ public class TestSnappyCodec extends CompressionTestBase {
|
||||||
public static final HBaseClassTestRule CLASS_RULE =
|
public static final HBaseClassTestRule CLASS_RULE =
|
||||||
HBaseClassTestRule.forClass(TestSnappyCodec.class);
|
HBaseClassTestRule.forClass(TestSnappyCodec.class);
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void setupClass() throws Exception {
|
||||||
|
assumeTrue(SnappyCodec.isLoaded());
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSnappyCodecSmall() throws Exception {
|
public void testSnappyCodecSmall() throws Exception {
|
||||||
codecSmallTest(new SnappyCodec());
|
codecSmallTest(new SnappyCodec());
|
||||||
|
|
|
@ -17,6 +17,8 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hbase.io.compress.xerial;
|
package org.apache.hadoop.hbase.io.compress.xerial;
|
||||||
|
|
||||||
|
import static org.junit.Assume.assumeTrue;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
|
@ -46,6 +48,7 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase {
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void setUpBeforeClass() throws Exception {
|
public static void setUpBeforeClass() throws Exception {
|
||||||
|
assumeTrue(SnappyCodec.isLoaded());
|
||||||
Configuration conf = TEST_UTIL.getConfiguration();
|
Configuration conf = TEST_UTIL.getConfiguration();
|
||||||
conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName());
|
conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName());
|
||||||
Compression.Algorithm.SNAPPY.reload(conf);
|
Compression.Algorithm.SNAPPY.reload(conf);
|
||||||
|
|
Loading…
Reference in New Issue