Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-15822. zstd compressor can fail with a small output buffer. Contributed by Jason Lowe. 

(cherry picked from commit 8f97d6f2cd)
This commit is contained in:
Akira Ajisaka 2018-10-24 13:49:28 +09:00
parent 7097755925
commit 63396beab4
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
5 changed files with 15 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dev-support/docker/Dockerfile
View File

@ -63,6 +63,7 @@ RUN apt-get -q update \
libsnappy-dev \ libsnappy-dev \
libssl-dev \ libssl-dev \
libtool \ libtool \
libzstd1-dev \
locales \ locales \
make \ make \
pinentry-curses \ pinentry-curses \

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
View File

@ -262,8 +262,8 @@ public class ZStandardDecompressor implements Decompressor {
int originalPosition = dst.position(); int originalPosition = dst.position();
int n = inflateBytesDirect( int n = inflateBytesDirect(
src, src.position(), src.remaining(), dst, dst.position(), src, src.position(), src.limit(), dst, dst.position(),
dst.remaining() dst.limit()
); );
dst.position(originalPosition + n); dst.position(originalPosition + n);
if (bytesInCompressedBuffer > 0) { if (bytesInCompressedBuffer > 0) {

5
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
View File

@ -195,11 +195,14 @@ JNIEXPORT jint Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_defla
ZSTD_inBuffer input = { uncompressed_bytes, uncompressed_direct_buf_len, uncompressed_direct_buf_off }; ZSTD_inBuffer input = { uncompressed_bytes, uncompressed_direct_buf_len, uncompressed_direct_buf_off };
ZSTD_outBuffer output = { compressed_bytes, compressed_direct_buf_len, 0 }; ZSTD_outBuffer output = { compressed_bytes, compressed_direct_buf_len, 0 };
size_t size = dlsym_ZSTD_compressStream(stream, &output, &input); size_t size;
if (uncompressed_direct_buf_len != 0) {
size = dlsym_ZSTD_compressStream(stream, &output, &input);
if (dlsym_ZSTD_isError(size)) { if (dlsym_ZSTD_isError(size)) {
THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size)); THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size));
return (jint) 0; return (jint) 0;
} }
}
if (finish && input.pos == input.size) { if (finish && input.pos == input.size) {
// end the stream, flush and write the frame epilogue // end the stream, flush and write the frame epilogue
size = dlsym_ZSTD_endStream(stream, &output); size = dlsym_ZSTD_endStream(stream, &output);

1
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
View File

@ -178,6 +178,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompre
return (jint) 0; return (jint) 0;
} }
uncompressed_bytes = ((char*) uncompressed_bytes) + uncompressed_direct_buf_off; uncompressed_bytes = ((char*) uncompressed_bytes) + uncompressed_direct_buf_off;
uncompressed_direct_buf_len -= uncompressed_direct_buf_off;
ZSTD_inBuffer input = { compressed_bytes, compressed_direct_buf_len, compressed_direct_buf_off }; ZSTD_inBuffer input = { compressed_bytes, compressed_direct_buf_len, compressed_direct_buf_off };
ZSTD_outBuffer output = { uncompressed_bytes, uncompressed_direct_buf_len, 0 }; ZSTD_outBuffer output = { uncompressed_bytes, uncompressed_direct_buf_len, 0 };

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
View File

@ -414,14 +414,12 @@ public class TestZStandardCompressorDecompressor {
outBuf.clear(); outBuf.clear();
while (!decompressor.finished()) { while (!decompressor.finished()) {
decompressor.decompress(inBuf, outBuf); decompressor.decompress(inBuf, outBuf);
if (outBuf.remaining() == 0) {
outBuf.flip(); outBuf.flip();
while (outBuf.remaining() > 0) { while (outBuf.remaining() > 0) {
assertEquals(expected.get(), outBuf.get()); assertEquals(expected.get(), outBuf.get());
} }
outBuf.clear(); outBuf.clear();
} }
}
outBuf.flip(); outBuf.flip();
while (outBuf.remaining() > 0) { while (outBuf.remaining() > 0) {
assertEquals(expected.get(), outBuf.get()); assertEquals(expected.get(), outBuf.get());