HADOOP-13138. Unable to append to a SequenceFile with Compression.NONE. Contributed by Vinayakumar B.
This commit is contained in:
parent
cf552aa87b
commit
bad85f3e3a
|
@ -1115,9 +1115,12 @@ public class SequenceFile {
|
|||
CompressionOption readerCompressionOption = new CompressionOption(
|
||||
reader.getCompressionType(), reader.getCompressionCodec());
|
||||
|
||||
// Codec comparison will be ignored if the compression is NONE
|
||||
if (readerCompressionOption.value != compressionTypeOption.value
|
||||
|| !readerCompressionOption.codec.getClass().getName()
|
||||
.equals(compressionTypeOption.codec.getClass().getName())) {
|
||||
|| (readerCompressionOption.value != CompressionType.NONE
|
||||
&& readerCompressionOption.codec
|
||||
.getClass() != compressionTypeOption.codec
|
||||
.getClass())) {
|
||||
throw new IllegalArgumentException(
|
||||
"Compression option provided does not match the file");
|
||||
}
|
||||
|
|
|
@ -248,6 +248,73 @@ public class TestSequenceFileAppend {
|
|||
fs.deleteOnExit(file);
|
||||
}
|
||||
|
||||
@Test(timeout = 30000)
|
||||
public void testAppendNoneCompression() throws Exception {
|
||||
Path file = new Path(ROOT_PATH, "testseqappendnonecompr.seq");
|
||||
fs.delete(file, true);
|
||||
|
||||
Option compressOption = Writer.compression(CompressionType.NONE);
|
||||
Writer writer =
|
||||
SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
|
||||
SequenceFile.Writer.keyClass(Long.class),
|
||||
SequenceFile.Writer.valueClass(String.class), compressOption);
|
||||
|
||||
writer.append(1L, "one");
|
||||
writer.append(2L, "two");
|
||||
writer.close();
|
||||
|
||||
verify2Values(file);
|
||||
|
||||
writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
|
||||
SequenceFile.Writer.keyClass(Long.class),
|
||||
SequenceFile.Writer.valueClass(String.class),
|
||||
SequenceFile.Writer.appendIfExists(true), compressOption);
|
||||
|
||||
writer.append(3L, "three");
|
||||
writer.append(4L, "four");
|
||||
writer.close();
|
||||
|
||||
verifyAll4Values(file);
|
||||
|
||||
// Verify failure if the compression details are different or not Provided
|
||||
try {
|
||||
writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
|
||||
SequenceFile.Writer.keyClass(Long.class),
|
||||
SequenceFile.Writer.valueClass(String.class),
|
||||
SequenceFile.Writer.appendIfExists(true));
|
||||
writer.close();
|
||||
fail("Expected IllegalArgumentException for compression options");
|
||||
} catch (IllegalArgumentException iae) {
|
||||
// Expected exception. Ignore it
|
||||
}
|
||||
|
||||
// Verify failure if the compression details are different
|
||||
try {
|
||||
Option wrongCompressOption =
|
||||
Writer.compression(CompressionType.RECORD, new GzipCodec());
|
||||
|
||||
writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
|
||||
SequenceFile.Writer.keyClass(Long.class),
|
||||
SequenceFile.Writer.valueClass(String.class),
|
||||
SequenceFile.Writer.appendIfExists(true), wrongCompressOption);
|
||||
writer.close();
|
||||
fail("Expected IllegalArgumentException for compression options");
|
||||
} catch (IllegalArgumentException iae) {
|
||||
// Expected exception. Ignore it
|
||||
}
|
||||
|
||||
// Codec should be ignored
|
||||
Option noneWithCodec =
|
||||
Writer.compression(CompressionType.NONE, new DefaultCodec());
|
||||
|
||||
writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
|
||||
SequenceFile.Writer.keyClass(Long.class),
|
||||
SequenceFile.Writer.valueClass(String.class),
|
||||
SequenceFile.Writer.appendIfExists(true), noneWithCodec);
|
||||
writer.close();
|
||||
fs.deleteOnExit(file);
|
||||
}
|
||||
|
||||
@Test(timeout = 30000)
|
||||
public void testAppendSort() throws Exception {
|
||||
GenericTestUtils.assumeInNativeProfile();
|
||||
|
|
Loading…
Reference in New Issue