Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13138. Unable to append to a SequenceFile with Compression.NONE. Contributed by Vinayakumar B. 

(cherry picked from commit 0a527196dcb35a699e12121165bc3738766b569e)
(cherry picked from commit 42f64b3bcc)
This commit is contained in:
Chris Nauroth 2016-05-18 09:32:00 -07:00
parent 01a0730996
commit 4ce5a58ca9
2 changed files with 72 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
View File

@ -1115,9 +1115,12 @@ public static Option compression(CompressionType value,
CompressionOption readerCompressionOption = new CompressionOption(
reader.getCompressionType(), reader.getCompressionCodec());
// Codec comparison will be ignored if the compression is NONE
if (readerCompressionOption.value != compressionTypeOption.value
|| !readerCompressionOption.codec.getClass().getName()
.equals(compressionTypeOption.codec.getClass().getName())) {
|| (readerCompressionOption.value != CompressionType.NONE
&& readerCompressionOption.codec
.getClass() != compressionTypeOption.codec
.getClass())) {
throw new IllegalArgumentException(
"Compression option provided does not match the file");
}

67
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
View File

@ -248,6 +248,73 @@ public void testAppendBlockCompression() throws Exception {
fs.deleteOnExit(file);
}
@Test(timeout = 30000)
public void testAppendNoneCompression() throws Exception {
Path file = new Path(ROOT_PATH, "testseqappendnonecompr.seq");
fs.delete(file, true);
Option compressOption = Writer.compression(CompressionType.NONE);
Writer writer =
SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
SequenceFile.Writer.keyClass(Long.class),
SequenceFile.Writer.valueClass(String.class), compressOption);
writer.append(1L, "one");
writer.append(2L, "two");
writer.close();
verify2Values(file);
writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
SequenceFile.Writer.keyClass(Long.class),
SequenceFile.Writer.valueClass(String.class),
SequenceFile.Writer.appendIfExists(true), compressOption);
writer.append(3L, "three");
writer.append(4L, "four");
writer.close();
verifyAll4Values(file);
// Verify failure if the compression details are different or not Provided
try {
writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
SequenceFile.Writer.keyClass(Long.class),
SequenceFile.Writer.valueClass(String.class),
SequenceFile.Writer.appendIfExists(true));
writer.close();
fail("Expected IllegalArgumentException for compression options");
} catch (IllegalArgumentException iae) {
// Expected exception. Ignore it
}
// Verify failure if the compression details are different
try {
Option wrongCompressOption =
Writer.compression(CompressionType.RECORD, new GzipCodec());
writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
SequenceFile.Writer.keyClass(Long.class),
SequenceFile.Writer.valueClass(String.class),
SequenceFile.Writer.appendIfExists(true), wrongCompressOption);
writer.close();
fail("Expected IllegalArgumentException for compression options");
} catch (IllegalArgumentException iae) {
// Expected exception. Ignore it
}
// Codec should be ignored
Option noneWithCodec =
Writer.compression(CompressionType.NONE, new DefaultCodec());
writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
SequenceFile.Writer.keyClass(Long.class),
SequenceFile.Writer.valueClass(String.class),
SequenceFile.Writer.appendIfExists(true), noneWithCodec);
writer.close();
fs.deleteOnExit(file);
}
@Test(timeout = 30000)
public void testAppendSort() throws Exception {
GenericTestUtils.assumeInNativeProfile();