Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-25940 Update Compression/TestCompressionTest: LZ4, SNAPPY, LZO (#3334) 

Undo asserts that LZ4 and SNAPPY fails if their native libs are NOT
loaded; as of hadoop 3.3.1, LZ4 and SNAPPY can work w/o native libs.

Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
Michael Stack 2021-05-29 17:02:52 +01:00 committed by GitHub
parent 9a2027bf71
commit f119a865cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hbase-common/src/main/java/org/apache/hadoop/hbase/io/compress/Compression.java
View File

@ -102,6 +102,8 @@ public final class Compression {
justification="We are not serializing so doesn't apply (not sure why transient though)") justification="We are not serializing so doesn't apply (not sure why transient though)")
@InterfaceAudience.Public @InterfaceAudience.Public
public static enum Algorithm { public static enum Algorithm {
// LZO is GPL and requires extra install to setup. See
// https://stackoverflow.com/questions/23441142/class-com-hadoop-compression-lzo-lzocodec-not-found-for-spark-on-cdh-5
LZO("lzo") { LZO("lzo") {
// Use base type to avoid compile-time dependencies. // Use base type to avoid compile-time dependencies.
private volatile transient CompressionCodec lzoCodec; private volatile transient CompressionCodec lzoCodec;

18
hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
View File

@ -76,7 +76,11 @@ public class TestCompressionTest {
assertTrue(CompressionTest.testCompression("GZ")); assertTrue(CompressionTest.testCompression("GZ"));
if (NativeCodeLoader.isNativeCodeLoaded()) { if (NativeCodeLoader.isNativeCodeLoaded()) {
nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec"); // LZO is GPL so not included in hadoop install. You need to do an extra install to pick
// up the needed support. This article is good on the steps needed to add LZO support:
// https://stackoverflow.com/questions/23441142/class-com-hadoop-compression-lzo-lzocodec-not-found-for-spark-on-cdh-5
// Its unlikely at test time that the extras are installed so this test is useless.
// nativeCodecTest("LZO", "lzo2", "com.hadoop.compression.lzo.LzoCodec");
nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec"); nativeCodecTest("LZ4", null, "org.apache.hadoop.io.compress.Lz4Codec");
nativeCodecTest("SNAPPY", "snappy", "org.apache.hadoop.io.compress.SnappyCodec"); nativeCodecTest("SNAPPY", "snappy", "org.apache.hadoop.io.compress.SnappyCodec");
nativeCodecTest("BZIP2", "bzip2", "org.apache.hadoop.io.compress.BZip2Codec"); nativeCodecTest("BZIP2", "bzip2", "org.apache.hadoop.io.compress.BZip2Codec");
@ -84,9 +88,15 @@ public class TestCompressionTest {
} else { } else {
// Hadoop nativelib is not available // Hadoop nativelib is not available
LOG.debug("Native code not loaded"); LOG.debug("Native code not loaded");
assertFalse(CompressionTest.testCompression("LZO")); // This check is useless as it fails with
assertFalse(CompressionTest.testCompression("LZ4")); // ...DoNotRetryIOException: Compression algorithm 'lzo' previously failed test.
assertFalse(CompressionTest.testCompression("SNAPPY")); // assertFalse("LZO", CompressionTest.testCompression("LZO"));
// LZ4 requires that the native lib be present before 3.3.1. After 3.3.1, hadoop uses
// lz4-java which will do java version of lz4 as last resort -- so the below fails before
// 3.3.1 but passes at 3.3.1+... so commenting it out. See HADOOP-17292.
// assertFalse("LZ4", CompressionTest.testCompression("LZ4"));
// Same thing happens for snappy. See HADOOP-17125
// assertFalse(CompressionTest.testCompression("SNAPPY"));
assertFalse(CompressionTest.testCompression("BZIP2")); assertFalse(CompressionTest.testCompression("BZIP2"));
assertFalse(CompressionTest.testCompression("ZSTD")); assertFalse(CompressionTest.testCompression("ZSTD"));
} }