Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Snappy decompression support (#5864) 

Support decompression of files using Google's Snappy algorithm.

This only supports files compressed with the Snappy framing format
described here: https://github.com/google/snappy/blob/master/framing_format.txt
This commit is contained in:
Joseph Glanville 2018-06-14 16:55:42 +07:00 committed by Slim Bouguerra
parent e0eb7048f6
commit 1032387d78
2 changed files with 19 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java
View File

@ -29,6 +29,7 @@ import com.google.common.io.Files;
import io.druid.java.util.common.io.NativeIO;
import io.druid.java.util.common.logger.Logger;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import java.io.BufferedInputStream;
@ -56,6 +57,7 @@ public class CompressionUtils
private static final String GZ_SUFFIX = ".gz";
private static final String XZ_SUFFIX = ".xz";
private static final String ZIP_SUFFIX = ".zip";
private static final String SNAPPY_SUFFIX = ".sz";
/**
* Zip the contents of directory into the file indicated by outputZipFile. Sub directories are skipped
@ -560,6 +562,8 @@ public class CompressionUtils
return new BZip2CompressorInputStream(in, true);
} else if (fileName.endsWith(XZ_SUFFIX)) {
return new XZCompressorInputStream(in, true);
} else if (fileName.endsWith(SNAPPY_SUFFIX)) {
return new FramedSnappyCompressorInputStream(in);
} else if (fileName.endsWith(ZIP_SUFFIX)) {
// This reads the first file in the archive.
final ZipInputStream zipIn = new ZipInputStream(in, StandardCharsets.UTF_8);

15
java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java
View File

@ -26,6 +26,7 @@ import com.google.common.io.ByteSource;
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorOutputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
import org.junit.Assert;
import org.junit.Before;
@ -275,6 +276,20 @@ public class CompressionUtilsTest
}
}
@Test
public void testDecompressSnappy() throws IOException
{
final File tmpDir = temporaryFolder.newFolder("testDecompressSnappy");
final File snappyFile = new File(tmpDir, testFile.getName() + ".sz");
Assert.assertFalse(snappyFile.exists());
try (final OutputStream out = new FramedSnappyCompressorOutputStream(new FileOutputStream(snappyFile))) {
ByteStreams.copy(new FileInputStream(testFile), out);
}
try (final InputStream inputStream = CompressionUtils.decompress(new FileInputStream(snappyFile), snappyFile.getName())) {
assertGoodDataStream(inputStream);
}
}
@Test
public void testDecompressZip() throws IOException
{