Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-15822. zstd compressor can fail with a small output buffer. Contributed by Jason Lowe.

This commit is contained in:
Akira Ajisaka 2018-10-24 13:49:28 +09:00
parent a0c0b793c3
commit 8f97d6f2cd
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
5 changed files with 15 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dev-support/docker/Dockerfile
View File

@ -63,6 +63,7 @@ RUN apt-get -q update \
libsnappy-dev \
libssl-dev \
libtool \
libzstd1-dev \
locales \
make \
pinentry-curses \

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
View File

@ -262,8 +262,8 @@ int inflateDirect(ByteBuffer src, ByteBuffer dst) throws IOException {
int originalPosition = dst.position();
int n = inflateBytesDirect(
src, src.position(), src.remaining(), dst, dst.position(),
dst.remaining()
src, src.position(), src.limit(), dst, dst.position(),
dst.limit()
);
dst.position(originalPosition + n);
if (bytesInCompressedBuffer > 0) {

11
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
View File

@ -195,10 +195,13 @@ JNIEXPORT jint Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_defla
ZSTD_inBuffer input = { uncompressed_bytes, uncompressed_direct_buf_len, uncompressed_direct_buf_off };
ZSTD_outBuffer output = { compressed_bytes, compressed_direct_buf_len, 0 };
size_t size = dlsym_ZSTD_compressStream(stream, &output, &input);
if (dlsym_ZSTD_isError(size)) {
THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size));
return (jint) 0;
size_t size;
if (uncompressed_direct_buf_len != 0) {
size = dlsym_ZSTD_compressStream(stream, &output, &input);
if (dlsym_ZSTD_isError(size)) {
THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size));
return (jint) 0;
}
}
if (finish && input.pos == input.size) {
// end the stream, flush and write the frame epilogue

1
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
View File

@ -178,6 +178,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompre
return (jint) 0;
}
uncompressed_bytes = ((char*) uncompressed_bytes) + uncompressed_direct_buf_off;
uncompressed_direct_buf_len -= uncompressed_direct_buf_off;
ZSTD_inBuffer input = { compressed_bytes, compressed_direct_buf_len, compressed_direct_buf_off };
ZSTD_outBuffer output = { uncompressed_bytes, uncompressed_direct_buf_len, 0 };

10
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
View File

@ -414,13 +414,11 @@ private void compressDecompressLoop(int rawDataSize) throws IOException {
outBuf.clear();
while (!decompressor.finished()) {
decompressor.decompress(inBuf, outBuf);
if (outBuf.remaining() == 0) {
outBuf.flip();
while (outBuf.remaining() > 0) {
assertEquals(expected.get(), outBuf.get());
}
outBuf.clear();
outBuf.flip();
while (outBuf.remaining() > 0) {
assertEquals(expected.get(), outBuf.get());
}
outBuf.clear();
}
outBuf.flip();
while (outBuf.remaining() > 0) {