Zstandard decompression support (#6224)

java-util/pom.xml

@@ -93,6 +93,10 @@
             <groupId>org.tukaani</groupId>
             <artifactId>xz</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.jayway.jsonpath</groupId>
             <artifactId>json-path</artifactId>

java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java

@@ -31,6 +31,7 @@ import io.druid.java.util.common.logger.Logger;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
 import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
+import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
 
 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
@@ -58,6 +59,7 @@ public class CompressionUtils
   private static final String XZ_SUFFIX = ".xz";
   private static final String ZIP_SUFFIX = ".zip";
   private static final String SNAPPY_SUFFIX = ".sz";
+  private static final String ZSTD_SUFFIX = ".zst";
 
   /**
    * Zip the contents of directory into the file indicated by outputZipFile. Sub directories are skipped
@@ -567,6 +569,8 @@ public class CompressionUtils
       return new XZCompressorInputStream(in, true);
     } else if (fileName.endsWith(SNAPPY_SUFFIX)) {
       return new FramedSnappyCompressorInputStream(in);
+    } else if (fileName.endsWith(ZSTD_SUFFIX)) {
+      return new ZstdCompressorInputStream(in);
     } else if (fileName.endsWith(ZIP_SUFFIX)) {
       // This reads the first file in the archive.
       final ZipInputStream zipIn = new ZipInputStream(in, StandardCharsets.UTF_8);

java-util/src/test/java/io/druid/java/util/common/CompressionUtilsTest.java

@@ -28,6 +28,7 @@ import com.google.common.io.Files;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
 import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorOutputStream;
 import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
+import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
@@ -290,6 +291,20 @@ public class CompressionUtilsTest
     }
   }
 
+  @Test
+  public void testDecompressZstd() throws IOException
+  {
+    final File tmpDir = temporaryFolder.newFolder("testDecompressZstd");
+    final File zstdFile = new File(tmpDir, testFile.getName() + ".zst");
+    Assert.assertFalse(zstdFile.exists());
+    try (final OutputStream out = new ZstdCompressorOutputStream(new FileOutputStream(zstdFile))) {
+      ByteStreams.copy(new FileInputStream(testFile), out);
+    }
+    try (final InputStream inputStream = CompressionUtils.decompress(new FileInputStream(zstdFile), zstdFile.getName())) {
+      assertGoodDataStream(inputStream);
+    }
+  }
+
   @Test
   public void testDecompressZip() throws IOException
   {

pom.xml

@@ -340,6 +340,11 @@
                 <artifactId>xz</artifactId>
                 <version>1.8</version>
             </dependency>
+            <dependency>
+                <groupId>com.github.luben</groupId>
+                <artifactId>zstd-jni</artifactId>
+                <version>1.3.3-1</version>
+            </dependency>
             <dependency>
                 <groupId>com.fasterxml.jackson.core</groupId>
                 <artifactId>jackson-annotations</artifactId>