From f3478674401164545065e4a039ac1d70a6c8bc2e Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Thu, 25 May 2023 17:05:57 -0700 Subject: [PATCH] HBASE-27872 xerial's snappy-java requires GLIBC >= 2.32 (#5245) We need to add a native library load check with a helpful error message if xerial snappy fails to initialize due to a too old glibc or similar reason, and disable the unit test if the native library fails to load. Signed-off-by: Duo Zhang Signed-off-by: Viraj Jasani --- .../hbase/io/compress/xerial/SnappyCodec.java | 23 +++++++++++++++++++ .../xerial/TestHFileCompressionSnappy.java | 3 +++ .../io/compress/xerial/TestSnappyCodec.java | 8 +++++++ .../xerial/TestWALCompressionSnappy.java | 3 +++ 4 files changed, 37 insertions(+) diff --git a/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java b/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java index b6806dcbeef..917676be2fe 100644 --- a/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java +++ b/hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java @@ -31,6 +31,8 @@ import org.apache.hadoop.io.compress.CompressionOutputStream; import org.apache.hadoop.io.compress.Compressor; import org.apache.hadoop.io.compress.Decompressor; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xerial.snappy.Snappy; /** @@ -43,10 +45,31 @@ public class SnappyCodec implements Configurable, CompressionCodec { public static final String SNAPPY_BUFFER_SIZE_KEY = "hbase.io.compress.snappy.buffersize"; + private static final Logger LOG = LoggerFactory.getLogger(SnappyCodec.class); private Configuration conf; private int bufferSize; + private static boolean loaded = false; + private static Throwable loadError; + + static { + try { + Snappy.getNativeLibraryVersion(); + loaded = true; + } catch (Throwable t) { + loadError = t; + LOG.error("The Snappy native libraries could not be loaded", t); + } + } + + /** Return true if the native shared libraries were loaded; false otherwise. */ + public static boolean isLoaded() { + return loaded; + } public SnappyCodec() { + if (!isLoaded()) { + throw new RuntimeException("Snappy codec could not be loaded", loadError); + } conf = new Configuration(); bufferSize = getBufferSize(conf); } diff --git a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java index 42b0735dd26..53c7c819a28 100644 --- a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java +++ b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.io.compress.xerial; +import static org.junit.Assume.assumeTrue; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -41,6 +43,7 @@ public class TestHFileCompressionSnappy extends HFileTestBase { @BeforeClass public static void setUpBeforeClass() throws Exception { + assumeTrue(SnappyCodec.isLoaded()); conf = TEST_UTIL.getConfiguration(); conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName()); Compression.Algorithm.SNAPPY.reload(conf); diff --git a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java index f62c3595673..8c011f9f689 100644 --- a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java +++ b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java @@ -17,9 +17,12 @@ */ package org.apache.hadoop.hbase.io.compress.xerial; +import static org.junit.Assume.assumeTrue; + import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.io.compress.CompressionTestBase; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -31,6 +34,11 @@ public class TestSnappyCodec extends CompressionTestBase { public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestSnappyCodec.class); + @BeforeClass + public static void setupClass() throws Exception { + assumeTrue(SnappyCodec.isLoaded()); + } + @Test public void testSnappyCodecSmall() throws Exception { codecSmallTest(new SnappyCodec()); diff --git a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java index dfbb63d0f6c..7a2bbfe7b87 100644 --- a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java +++ b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.io.compress.xerial; +import static org.junit.Assume.assumeTrue; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; @@ -46,6 +48,7 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase { @BeforeClass public static void setUpBeforeClass() throws Exception { + assumeTrue(SnappyCodec.isLoaded()); Configuration conf = TEST_UTIL.getConfiguration(); conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName()); Compression.Algorithm.SNAPPY.reload(conf);