Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-27872 xerial's snappy-java requires GLIBC >= 2.32 (#5245) 

We need to add a native library load check with a helpful error message
if xerial snappy fails to initialize due to a too old glibc or similar
reason, and disable the unit test if the native library fails to load.

Signed-off-by: Duo Zhang <zhangduo@apache.org>
Signed-off-by: Viraj Jasani <vjasani@apache.org>
This commit is contained in:
Andrew Purtell 2023-05-25 17:05:57 -07:00
parent b6d063550a
commit aff6cdbfe0
4 changed files with 37 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
hbase-compression/hbase-compression-snappy/src/main/java/org/apache/hadoop/hbase/io/compress/xerial/SnappyCodec.java
View File

@ -31,6 +31,8 @@ import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.io.compress.Compressor;
import org.apache.hadoop.io.compress.Decompressor;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xerial.snappy.Snappy;
/**
@ -43,10 +45,31 @@ public class SnappyCodec implements Configurable, CompressionCodec {
public static final String SNAPPY_BUFFER_SIZE_KEY = "hbase.io.compress.snappy.buffersize";
private static final Logger LOG = LoggerFactory.getLogger(SnappyCodec.class);
private Configuration conf;
private int bufferSize;
private static boolean loaded = false;
private static Throwable loadError;
static {
try {
Snappy.getNativeLibraryVersion();
loaded = true;
} catch (Throwable t) {
loadError = t;
LOG.error("The Snappy native libraries could not be loaded", t);
}
}
/** Return true if the native shared libraries were loaded; false otherwise. */
public static boolean isLoaded() {
return loaded;
}
public SnappyCodec() {
if (!isLoaded()) {
throw new RuntimeException("Snappy codec could not be loaded", loadError);
}
conf = new Configuration();
bufferSize = getBufferSize(conf);
}

3
hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestHFileCompressionSnappy.java
View File

@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.io.compress.xerial;
import static org.junit.Assume.assumeTrue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
@ -41,6 +43,7 @@ public class TestHFileCompressionSnappy extends HFileTestBase {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
assumeTrue(SnappyCodec.isLoaded());
conf = TEST_UTIL.getConfiguration();
conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName());
Compression.Algorithm.SNAPPY.reload(conf);

8
hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestSnappyCodec.java
View File

@ -17,9 +17,12 @@
*/
package org.apache.hadoop.hbase.io.compress.xerial;
import static org.junit.Assume.assumeTrue;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.io.compress.CompressionTestBase;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@ -31,6 +34,11 @@ public class TestSnappyCodec extends CompressionTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestSnappyCodec.class);
@BeforeClass
public static void setupClass() throws Exception {
assumeTrue(SnappyCodec.isLoaded());
}
@Test
public void testSnappyCodecSmall() throws Exception {
codecSmallTest(new SnappyCodec());

3
hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java
View File

@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.io.compress.xerial;
import static org.junit.Assume.assumeTrue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
@ -46,6 +48,7 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
assumeTrue(SnappyCodec.isLoaded());
Configuration conf = TEST_UTIL.getConfiguration();
conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName());
Compression.Algorithm.SNAPPY.reload(conf);