Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-17096. Fix ZStandardCompressor input buffer offset (#2104). Contributed by Stephen Jung (Stripe).

This commit is contained in:
Stephen Jung 2020-11-10 11:38:22 -08:00 committed by GitHub
parent 61f8c5767e
commit 45434c93e8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java
View File

@ -159,7 +159,7 @@ public class ZStandardCompressor implements Compressor {
} }
// have we consumed all input // have we consumed all input
if (keepUncompressedBuf && uncompressedDirectBufLen > 0) { if (keepUncompressedBuf && uncompressedDirectBufLen - uncompressedDirectBufOff > 0) {
return false; return false;
} }
@ -223,7 +223,7 @@ public class ZStandardCompressor implements Compressor {
compressedDirectBuf.limit(n); compressedDirectBuf.limit(n);
// Check if we have consumed all input buffer // Check if we have consumed all input buffer
if (uncompressedDirectBufLen <= 0) { if (uncompressedDirectBufLen - uncompressedDirectBufOff <= 0) {
// consumed all input buffer // consumed all input buffer
keepUncompressedBuf = false; keepUncompressedBuf = false;
uncompressedDirectBuf.clear(); uncompressedDirectBuf.clear();

4
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
View File

@ -219,13 +219,13 @@ JNIEXPORT jint Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_defla
return (jint) 0; return (jint) 0;
} }
bytes_read += input.pos; bytes_read += input.pos - uncompressed_direct_buf_off;
bytes_written += output.pos; bytes_written += output.pos;
(*env)->SetLongField(env, this, ZStandardCompressor_bytesRead, bytes_read); (*env)->SetLongField(env, this, ZStandardCompressor_bytesRead, bytes_read);
(*env)->SetLongField(env, this, ZStandardCompressor_bytesWritten, bytes_written); (*env)->SetLongField(env, this, ZStandardCompressor_bytesWritten, bytes_written);
(*env)->SetIntField(env, this, ZStandardCompressor_uncompressedDirectBufOff, input.pos); (*env)->SetIntField(env, this, ZStandardCompressor_uncompressedDirectBufOff, input.pos);
(*env)->SetIntField(env, this, ZStandardCompressor_uncompressedDirectBufLen, input.size - input.pos); (*env)->SetIntField(env, this, ZStandardCompressor_uncompressedDirectBufLen, input.size);
return (jint) output.pos; return (jint) output.pos;
} }