Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-16768. SnappyCompressor test cases wrongly assume that the compressed data is always smaller than the input data. (#3264)

Browse Source 
* HADOOP-16768. SnappyCompressor test cases wrongly assume that the compressed data is always smaller than the input data. (#2003)

(cherry picked from commit 328eae9a146b2dd9857a17a0db6fcddb1de23a0d)

* Add assertj-core dependency to hadoop-common for test
* Bump assertj-core version to 3.12.2 to use assertThat.hasSizeGreaterThan
This commit is contained in:
Akira Ajisaka 2021-08-04 15:46:00 +09:00 committed by GitHub
parent e259d85a99
commit b838647b16
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 109 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-common-project/hadoop-common/pom.xml
View File

@ -160,6 +160,11 @@
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>commons-beanutils</groupId> <groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId> <artifactId>commons-beanutils</artifactId>

72
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java
View File

@ -126,7 +126,7 @@ private void addPair(T compressor, E decompressor, String name) {
builder.add(new TesterPair<T, E>(name, compressor, decompressor)); builder.add(new TesterPair<T, E>(name, compressor, decompressor));
} }
public void test() throws InstantiationException, IllegalAccessException { public void test() throws Exception {
pairs = builder.build(); pairs = builder.build();
pairs = assertionDelegate.filterOnAssumeWhat(pairs); pairs = assertionDelegate.filterOnAssumeWhat(pairs);
@ -287,47 +287,45 @@ private boolean checkSetInputArrayIndexOutOfBoundsException(
@Override @Override
public void assertCompression(String name, Compressor compressor, public void assertCompression(String name, Compressor compressor,
Decompressor decompressor, byte[] rawData) { Decompressor decompressor, byte[] rawData) throws Exception {
int cSize = 0; int cSize = 0;
int decompressedSize = 0; int decompressedSize = 0;
byte[] compressedResult = new byte[rawData.length]; // Snappy compression can increase data size
int maxCompressedLength = 32 + rawData.length + rawData.length/6;
byte[] compressedResult = new byte[maxCompressedLength];
byte[] decompressedBytes = new byte[rawData.length]; byte[] decompressedBytes = new byte[rawData.length];
try { assertTrue(
assertTrue( joiner.join(name, "compressor.needsInput before error !!!"),
joiner.join(name, "compressor.needsInput before error !!!"), compressor.needsInput());
compressor.needsInput()); assertEquals(
assertTrue(
joiner.join(name, "compressor.getBytesWritten before error !!!"), joiner.join(name, "compressor.getBytesWritten before error !!!"),
compressor.getBytesWritten() == 0); 0, compressor.getBytesWritten());
compressor.setInput(rawData, 0, rawData.length); compressor.setInput(rawData, 0, rawData.length);
compressor.finish(); compressor.finish();
while (!compressor.finished()) { while (!compressor.finished()) {
cSize += compressor.compress(compressedResult, 0, cSize += compressor.compress(compressedResult, 0,
compressedResult.length); compressedResult.length);
}
compressor.reset();
assertTrue(
joiner.join(name, "decompressor.needsInput() before error !!!"),
decompressor.needsInput());
decompressor.setInput(compressedResult, 0, cSize);
assertFalse(
joiner.join(name, "decompressor.needsInput() after error !!!"),
decompressor.needsInput());
while (!decompressor.finished()) {
decompressedSize = decompressor.decompress(decompressedBytes, 0,
decompressedBytes.length);
}
decompressor.reset();
assertTrue(joiner.join(name, " byte size not equals error !!!"),
decompressedSize == rawData.length);
assertArrayEquals(
joiner.join(name, " byte arrays not equals error !!!"), rawData,
decompressedBytes);
} catch (Exception ex) {
fail(joiner.join(name, ex.getMessage()));
} }
compressor.reset();
assertTrue(
joiner.join(name, "decompressor.needsInput() before error !!!"),
decompressor.needsInput());
decompressor.setInput(compressedResult, 0, cSize);
assertFalse(
joiner.join(name, "decompressor.needsInput() after error !!!"),
decompressor.needsInput());
while (!decompressor.finished()) {
decompressedSize = decompressor.decompress(decompressedBytes, 0,
decompressedBytes.length);
}
decompressor.reset();
assertEquals(joiner.join(name, " byte size not equals error !!!"),
rawData.length, decompressedSize);
assertArrayEquals(
joiner.join(name, " byte arrays not equals error !!!"), rawData,
decompressedBytes);
} }
}), }),
@ -519,6 +517,6 @@ abstract static class TesterCompressionStrategy {
protected final Logger logger = Logger.getLogger(getClass()); protected final Logger logger = Logger.getLogger(getClass());
abstract void assertCompression(String name, Compressor compressor, abstract void assertCompression(String name, Compressor compressor,
Decompressor decompressor, byte[] originalRawData); Decompressor decompressor, byte[] originalRawData) throws Exception;
} }
} }

96
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java
View File

@ -17,6 +17,7 @@
*/ */
package org.apache.hadoop.io.compress.snappy; package org.apache.hadoop.io.compress.snappy;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
@ -44,11 +45,16 @@
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.junit.Assume.*; import static org.junit.Assume.*;
public class TestSnappyCompressorDecompressor { public class TestSnappyCompressorDecompressor {
public static final Logger LOG =
LoggerFactory.getLogger(TestSnappyCompressorDecompressor.class);
@Before @Before
public void before() { public void before() {
assumeTrue(SnappyCodec.isNativeCodeLoaded()); assumeTrue(SnappyCodec.isNativeCodeLoaded());
@ -167,40 +173,41 @@ public void testSnappyDecompressorCompressAIOBException() {
} }
@Test @Test
public void testSnappyCompressDecompress() { public void testSnappyCompressDecompress() throws Exception {
int BYTE_SIZE = 1024 * 54; int BYTE_SIZE = 1024 * 54;
byte[] bytes = BytesGenerator.get(BYTE_SIZE); byte[] bytes = BytesGenerator.get(BYTE_SIZE);
SnappyCompressor compressor = new SnappyCompressor(); SnappyCompressor compressor = new SnappyCompressor();
try { compressor.setInput(bytes, 0, bytes.length);
compressor.setInput(bytes, 0, bytes.length); assertTrue("SnappyCompressDecompress getBytesRead error !!!",
assertTrue("SnappyCompressDecompress getBytesRead error !!!", compressor.getBytesRead() > 0);
compressor.getBytesRead() > 0); assertEquals(
assertTrue( "SnappyCompressDecompress getBytesWritten before compress error !!!",
"SnappyCompressDecompress getBytesWritten before compress error !!!", 0, compressor.getBytesWritten());
compressor.getBytesWritten() == 0);
byte[] compressed = new byte[BYTE_SIZE]; // snappy compression may increase data size.
int cSize = compressor.compress(compressed, 0, compressed.length); // This calculation comes from "Snappy::MaxCompressedLength(size_t)"
assertTrue( int maxSize = 32 + BYTE_SIZE + BYTE_SIZE / 6;
"SnappyCompressDecompress getBytesWritten after compress error !!!", byte[] compressed = new byte[maxSize];
compressor.getBytesWritten() > 0); int cSize = compressor.compress(compressed, 0, compressed.length);
LOG.info("input size: {}", BYTE_SIZE);
LOG.info("compressed size: {}", cSize);
assertTrue(
"SnappyCompressDecompress getBytesWritten after compress error !!!",
compressor.getBytesWritten() > 0);
SnappyDecompressor decompressor = new SnappyDecompressor(BYTE_SIZE); SnappyDecompressor decompressor = new SnappyDecompressor();
// set as input for decompressor only compressed data indicated with cSize // set as input for decompressor only compressed data indicated with cSize
decompressor.setInput(compressed, 0, cSize); decompressor.setInput(compressed, 0, cSize);
byte[] decompressed = new byte[BYTE_SIZE]; byte[] decompressed = new byte[BYTE_SIZE];
decompressor.decompress(decompressed, 0, decompressed.length); decompressor.decompress(decompressed, 0, decompressed.length);
assertTrue("testSnappyCompressDecompress finished error !!!", assertTrue("testSnappyCompressDecompress finished error !!!",
decompressor.finished()); decompressor.finished());
Assert.assertArrayEquals(bytes, decompressed); Assert.assertArrayEquals(bytes, decompressed);
compressor.reset(); compressor.reset();
decompressor.reset(); decompressor.reset();
assertTrue("decompressor getRemaining error !!!", assertEquals("decompressor getRemaining error !!!",
decompressor.getRemaining() == 0); 0, decompressor.getRemaining());
} catch (Exception e) {
fail("testSnappyCompressDecompress ex error!!!");
}
} }
@Test @Test
@ -278,7 +285,38 @@ public void testSnappyBlockCompression() {
fail("testSnappyBlockCompression ex error !!!"); fail("testSnappyBlockCompression ex error !!!");
} }
} }
@Test
// The buffer size is smaller than the input.
public void testSnappyCompressDecompressWithSmallBuffer() throws Exception {
int inputSize = 1024 * 50;
int bufferSize = 512;
ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] buffer = new byte[bufferSize];
byte[] input = BytesGenerator.get(inputSize);
SnappyCompressor compressor = new SnappyCompressor();
compressor.setInput(input, 0, inputSize);
compressor.finish();
while (!compressor.finished()) {
int len = compressor.compress(buffer, 0, buffer.length);
out.write(buffer, 0, len);
}
byte[] compressed = out.toByteArray();
assertThat(compressed).hasSizeGreaterThan(0);
out.reset();
SnappyDecompressor decompressor = new SnappyDecompressor();
decompressor.setInput(compressed, 0, compressed.length);
while (!decompressor.finished()) {
int len = decompressor.decompress(buffer, 0, buffer.length);
out.write(buffer, 0, len);
}
byte[] decompressed = out.toByteArray();
assertThat(decompressed).isEqualTo(input);
}
private void compressDecompressLoop(int rawDataSize) throws IOException { private void compressDecompressLoop(int rawDataSize) throws IOException {
byte[] rawData = BytesGenerator.get(rawDataSize); byte[] rawData = BytesGenerator.get(rawDataSize);
byte[] compressedResult = new byte[rawDataSize+20]; byte[] compressedResult = new byte[rawDataSize+20];

3
hadoop-project/pom.xml
View File

@ -184,6 +184,7 @@
<hbase.one.version>1.2.6</hbase.one.version> <hbase.one.version>1.2.6</hbase.one.version>
<hbase.two.version>2.0.0-beta-1</hbase.two.version> <hbase.two.version>2.0.0-beta-1</hbase.two.version>
<junit.version>4.13.2</junit.version> <junit.version>4.13.2</junit.version>
<assertj.version>3.12.2</assertj.version>
<woodstox.version>5.3.0</woodstox.version> <woodstox.version>5.3.0</woodstox.version>
<json-smart.version>2.4.2</json-smart.version> <json-smart.version>2.4.2</json-smart.version>
<nimbus-jose-jwt.version>9.8.1</nimbus-jose-jwt.version> <nimbus-jose-jwt.version>9.8.1</nimbus-jose-jwt.version>
@ -1492,7 +1493,7 @@
<dependency> <dependency>
<groupId>org.assertj</groupId> <groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId> <artifactId>assertj-core</artifactId>
<version>3.8.0</version> <version>${assertj.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>