HBASE-27872 xerial's snappy-java requires GLIBC >= 2.32 (#5245)

We need to add a native library load check with a helpful error message
if xerial snappy fails to initialize due to a too old glibc or similar
reason, and disable the unit test if the native library fails to load.

Signed-off-by: Duo Zhang <zhangduo@apache.org>
Signed-off-by: Viraj Jasani <vjasani@apache.org>
This commit is contained in:
Andrew Purtell 2023-05-25 17:05:57 -07:00 committed by GitHub
parent 6a5a710e31
commit f347867440
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 37 additions and 0 deletions

View File

@ -31,6 +31,8 @@ import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.io.compress.Compressor; import org.apache.hadoop.io.compress.Compressor;
import org.apache.hadoop.io.compress.Decompressor; import org.apache.hadoop.io.compress.Decompressor;
import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xerial.snappy.Snappy; import org.xerial.snappy.Snappy;
/** /**
@ -43,10 +45,31 @@ public class SnappyCodec implements Configurable, CompressionCodec {
public static final String SNAPPY_BUFFER_SIZE_KEY = "hbase.io.compress.snappy.buffersize"; public static final String SNAPPY_BUFFER_SIZE_KEY = "hbase.io.compress.snappy.buffersize";
private static final Logger LOG = LoggerFactory.getLogger(SnappyCodec.class);
private Configuration conf; private Configuration conf;
private int bufferSize; private int bufferSize;
private static boolean loaded = false;
private static Throwable loadError;
static {
try {
Snappy.getNativeLibraryVersion();
loaded = true;
} catch (Throwable t) {
loadError = t;
LOG.error("The Snappy native libraries could not be loaded", t);
}
}
/** Return true if the native shared libraries were loaded; false otherwise. */
public static boolean isLoaded() {
return loaded;
}
public SnappyCodec() { public SnappyCodec() {
if (!isLoaded()) {
throw new RuntimeException("Snappy codec could not be loaded", loadError);
}
conf = new Configuration(); conf = new Configuration();
bufferSize = getBufferSize(conf); bufferSize = getBufferSize(conf);
} }

View File

@ -17,6 +17,8 @@
*/ */
package org.apache.hadoop.hbase.io.compress.xerial; package org.apache.hadoop.hbase.io.compress.xerial;
import static org.junit.Assume.assumeTrue;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseClassTestRule;
@ -41,6 +43,7 @@ public class TestHFileCompressionSnappy extends HFileTestBase {
@BeforeClass @BeforeClass
public static void setUpBeforeClass() throws Exception { public static void setUpBeforeClass() throws Exception {
assumeTrue(SnappyCodec.isLoaded());
conf = TEST_UTIL.getConfiguration(); conf = TEST_UTIL.getConfiguration();
conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName()); conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName());
Compression.Algorithm.SNAPPY.reload(conf); Compression.Algorithm.SNAPPY.reload(conf);

View File

@ -17,9 +17,12 @@
*/ */
package org.apache.hadoop.hbase.io.compress.xerial; package org.apache.hadoop.hbase.io.compress.xerial;
import static org.junit.Assume.assumeTrue;
import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.io.compress.CompressionTestBase; import org.apache.hadoop.hbase.io.compress.CompressionTestBase;
import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.junit.BeforeClass;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.Test; import org.junit.Test;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
@ -31,6 +34,11 @@ public class TestSnappyCodec extends CompressionTestBase {
public static final HBaseClassTestRule CLASS_RULE = public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestSnappyCodec.class); HBaseClassTestRule.forClass(TestSnappyCodec.class);
@BeforeClass
public static void setupClass() throws Exception {
assumeTrue(SnappyCodec.isLoaded());
}
@Test @Test
public void testSnappyCodecSmall() throws Exception { public void testSnappyCodecSmall() throws Exception {
codecSmallTest(new SnappyCodec()); codecSmallTest(new SnappyCodec());

View File

@ -17,6 +17,8 @@
*/ */
package org.apache.hadoop.hbase.io.compress.xerial; package org.apache.hadoop.hbase.io.compress.xerial;
import static org.junit.Assume.assumeTrue;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
@ -46,6 +48,7 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase {
@BeforeClass @BeforeClass
public static void setUpBeforeClass() throws Exception { public static void setUpBeforeClass() throws Exception {
assumeTrue(SnappyCodec.isLoaded());
Configuration conf = TEST_UTIL.getConfiguration(); Configuration conf = TEST_UTIL.getConfiguration();
conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName()); conf.set(Compression.SNAPPY_CODEC_CLASS_KEY, SnappyCodec.class.getCanonicalName());
Compression.Algorithm.SNAPPY.reload(conf); Compression.Algorithm.SNAPPY.reload(conf);