Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-16768. SnappyCompressor test cases wrongly assume that the compressed data is always smaller than the input data. (#2003)

This commit is contained in:
Akira Ajisaka 2020-05-11 14:44:18 +09:00 committed by GitHub
parent aab9e0b16e
commit 328eae9a14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 102 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java
View File

@ -126,7 +126,7 @@ public class CompressDecompressTester<T extends Compressor, E extends Decompress
builder.add(new TesterPair<T, E>(name, compressor, decompressor));
}
public void test() throws InstantiationException, IllegalAccessException {
public void test() throws Exception {
pairs = builder.build();
pairs = assertionDelegate.filterOnAssumeWhat(pairs);
@ -287,47 +287,45 @@ public class CompressDecompressTester<T extends Compressor, E extends Decompress
@Override
public void assertCompression(String name, Compressor compressor,
Decompressor decompressor, byte[] rawData) {
Decompressor decompressor, byte[] rawData) throws Exception {
int cSize = 0;
int decompressedSize = 0;
byte[] compressedResult = new byte[rawData.length];
// Snappy compression can increase data size
int maxCompressedLength = 32 + rawData.length + rawData.length/6;
byte[] compressedResult = new byte[maxCompressedLength];
byte[] decompressedBytes = new byte[rawData.length];
try {
assertTrue(
joiner.join(name, "compressor.needsInput before error !!!"),
compressor.needsInput());
assertTrue(
assertTrue(
joiner.join(name, "compressor.needsInput before error !!!"),
compressor.needsInput());
assertEquals(
joiner.join(name, "compressor.getBytesWritten before error !!!"),
compressor.getBytesWritten() == 0);
compressor.setInput(rawData, 0, rawData.length);
compressor.finish();
while (!compressor.finished()) {
cSize += compressor.compress(compressedResult, 0,
compressedResult.length);
}
compressor.reset();
assertTrue(
joiner.join(name, "decompressor.needsInput() before error !!!"),
decompressor.needsInput());
decompressor.setInput(compressedResult, 0, cSize);
assertFalse(
joiner.join(name, "decompressor.needsInput() after error !!!"),
decompressor.needsInput());
while (!decompressor.finished()) {
decompressedSize = decompressor.decompress(decompressedBytes, 0,
decompressedBytes.length);
}
decompressor.reset();
assertTrue(joiner.join(name, " byte size not equals error !!!"),
decompressedSize == rawData.length);
assertArrayEquals(
joiner.join(name, " byte arrays not equals error !!!"), rawData,
decompressedBytes);
} catch (Exception ex) {
fail(joiner.join(name, ex.getMessage()));
0, compressor.getBytesWritten());
compressor.setInput(rawData, 0, rawData.length);
compressor.finish();
while (!compressor.finished()) {
cSize += compressor.compress(compressedResult, 0,
compressedResult.length);
}
compressor.reset();
assertTrue(
joiner.join(name, "decompressor.needsInput() before error !!!"),
decompressor.needsInput());
decompressor.setInput(compressedResult, 0, cSize);
assertFalse(
joiner.join(name, "decompressor.needsInput() after error !!!"),
decompressor.needsInput());
while (!decompressor.finished()) {
decompressedSize = decompressor.decompress(decompressedBytes, 0,
decompressedBytes.length);
}
decompressor.reset();
assertEquals(joiner.join(name, " byte size not equals error !!!"),
rawData.length, decompressedSize);
assertArrayEquals(
joiner.join(name, " byte arrays not equals error !!!"), rawData,
decompressedBytes);
}
}),
@ -519,6 +517,6 @@ public class CompressDecompressTester<T extends Compressor, E extends Decompress
protected final Logger logger = Logger.getLogger(getClass());
abstract void assertCompression(String name, Compressor compressor,
Decompressor decompressor, byte[] originalRawData);
Decompressor decompressor, byte[] originalRawData) throws Exception;
}
}

96
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.io.compress.snappy;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@ -44,11 +45,16 @@ import org.apache.hadoop.test.MultithreadedTestUtil;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.junit.Assume.*;
public class TestSnappyCompressorDecompressor {
public static final Logger LOG =
LoggerFactory.getLogger(TestSnappyCompressorDecompressor.class);
@Before
public void before() {
assumeTrue(SnappyCodec.isNativeCodeLoaded());
@ -167,40 +173,41 @@ public class TestSnappyCompressorDecompressor {
}
@Test
public void testSnappyCompressDecompress() {
public void testSnappyCompressDecompress() throws Exception {
int BYTE_SIZE = 1024 * 54;
byte[] bytes = BytesGenerator.get(BYTE_SIZE);
SnappyCompressor compressor = new SnappyCompressor();
try {
compressor.setInput(bytes, 0, bytes.length);
assertTrue("SnappyCompressDecompress getBytesRead error !!!",
compressor.getBytesRead() > 0);
assertTrue(
"SnappyCompressDecompress getBytesWritten before compress error !!!",
compressor.getBytesWritten() == 0);
compressor.setInput(bytes, 0, bytes.length);
assertTrue("SnappyCompressDecompress getBytesRead error !!!",
compressor.getBytesRead() > 0);
assertEquals(
"SnappyCompressDecompress getBytesWritten before compress error !!!",
0, compressor.getBytesWritten());
byte[] compressed = new byte[BYTE_SIZE];
int cSize = compressor.compress(compressed, 0, compressed.length);
assertTrue(
"SnappyCompressDecompress getBytesWritten after compress error !!!",
compressor.getBytesWritten() > 0);
// snappy compression may increase data size.
// This calculation comes from "Snappy::MaxCompressedLength(size_t)"
int maxSize = 32 + BYTE_SIZE + BYTE_SIZE / 6;
byte[] compressed = new byte[maxSize];
int cSize = compressor.compress(compressed, 0, compressed.length);
LOG.info("input size: {}", BYTE_SIZE);
LOG.info("compressed size: {}", cSize);
assertTrue(
"SnappyCompressDecompress getBytesWritten after compress error !!!",
compressor.getBytesWritten() > 0);
SnappyDecompressor decompressor = new SnappyDecompressor(BYTE_SIZE);
// set as input for decompressor only compressed data indicated with cSize
decompressor.setInput(compressed, 0, cSize);
byte[] decompressed = new byte[BYTE_SIZE];
decompressor.decompress(decompressed, 0, decompressed.length);
SnappyDecompressor decompressor = new SnappyDecompressor();
// set as input for decompressor only compressed data indicated with cSize
decompressor.setInput(compressed, 0, cSize);
byte[] decompressed = new byte[BYTE_SIZE];
decompressor.decompress(decompressed, 0, decompressed.length);
assertTrue("testSnappyCompressDecompress finished error !!!",
decompressor.finished());
Assert.assertArrayEquals(bytes, decompressed);
compressor.reset();
decompressor.reset();
assertTrue("decompressor getRemaining error !!!",
decompressor.getRemaining() == 0);
} catch (Exception e) {
fail("testSnappyCompressDecompress ex error!!!");
}
assertTrue("testSnappyCompressDecompress finished error !!!",
decompressor.finished());
Assert.assertArrayEquals(bytes, decompressed);
compressor.reset();
decompressor.reset();
assertEquals("decompressor getRemaining error !!!",
0, decompressor.getRemaining());
}
@Test
@ -278,7 +285,38 @@ public class TestSnappyCompressorDecompressor {
fail("testSnappyBlockCompression ex error !!!");
}
}
@Test
// The buffer size is smaller than the input.
public void testSnappyCompressDecompressWithSmallBuffer() throws Exception {
int inputSize = 1024 * 50;
int bufferSize = 512;
ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] buffer = new byte[bufferSize];
byte[] input = BytesGenerator.get(inputSize);
SnappyCompressor compressor = new SnappyCompressor();
compressor.setInput(input, 0, inputSize);
compressor.finish();
while (!compressor.finished()) {
int len = compressor.compress(buffer, 0, buffer.length);
out.write(buffer, 0, len);
}
byte[] compressed = out.toByteArray();
assertThat(compressed).hasSizeGreaterThan(0);
out.reset();
SnappyDecompressor decompressor = new SnappyDecompressor();
decompressor.setInput(compressed, 0, compressed.length);
while (!decompressor.finished()) {
int len = decompressor.decompress(buffer, 0, buffer.length);
out.write(buffer, 0, len);
}
byte[] decompressed = out.toByteArray();
assertThat(decompressed).isEqualTo(input);
}
private void compressDecompressLoop(int rawDataSize) throws IOException {
byte[] rawData = BytesGenerator.get(rawDataSize);
byte[] compressedResult = new byte[rawDataSize+20];