diff --git a/java-util/pom.xml b/java-util/pom.xml index 696e579a73d..569aa4fa8d3 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -93,6 +93,10 @@ org.tukaani xz + + com.github.luben + zstd-jni + com.jayway.jsonpath json-path diff --git a/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java b/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java index 99c6367b095..77ef23a358c 100644 --- a/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java @@ -31,6 +31,7 @@ import io.druid.java.util.common.logger.Logger; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream; import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; @@ -58,6 +59,7 @@ public class CompressionUtils private static final String XZ_SUFFIX = ".xz"; private static final String ZIP_SUFFIX = ".zip"; private static final String SNAPPY_SUFFIX = ".sz"; + private static final String ZSTD_SUFFIX = ".zst"; /** * Zip the contents of directory into the file indicated by outputZipFile. Sub directories are skipped @@ -567,6 +569,8 @@ public class CompressionUtils return new XZCompressorInputStream(in, true); } else if (fileName.endsWith(SNAPPY_SUFFIX)) { return new FramedSnappyCompressorInputStream(in); + } else if (fileName.endsWith(ZSTD_SUFFIX)) { + return new ZstdCompressorInputStream(in); } else if (fileName.endsWith(ZIP_SUFFIX)) { // This reads the first file in the archive. final ZipInputStream zipIn = new ZipInputStream(in, StandardCharsets.UTF_8); diff --git a/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java b/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java index 26d3d7914b6..2e02c770ea8 100644 --- a/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java @@ -28,6 +28,7 @@ import com.google.common.io.Files; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorOutputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; +import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -290,6 +291,20 @@ public class CompressionUtilsTest } } + @Test + public void testDecompressZstd() throws IOException + { + final File tmpDir = temporaryFolder.newFolder("testDecompressZstd"); + final File zstdFile = new File(tmpDir, testFile.getName() + ".zst"); + Assert.assertFalse(zstdFile.exists()); + try (final OutputStream out = new ZstdCompressorOutputStream(new FileOutputStream(zstdFile))) { + ByteStreams.copy(new FileInputStream(testFile), out); + } + try (final InputStream inputStream = CompressionUtils.decompress(new FileInputStream(zstdFile), zstdFile.getName())) { + assertGoodDataStream(inputStream); + } + } + @Test public void testDecompressZip() throws IOException { diff --git a/pom.xml b/pom.xml index a7bc4faac46..9774ac8b33e 100644 --- a/pom.xml +++ b/pom.xml @@ -340,6 +340,11 @@ xz 1.8 + + com.github.luben + zstd-jni + 1.3.3-1 + com.fasterxml.jackson.core jackson-annotations