Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-9220. Reading small file (< 512 bytes) that is open for append fails due to incorrect checksum. Contributed by Jing Zhao.

Browse Source 
(cherry picked from commit c7c36cbd6218f46c33d7fb2f60cd52cb29e6d720)
(cherry picked from commit 4cf7f8441a288cefd44f126d60dae1998239892a)

Conflicts:

	hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
	hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
This commit is contained in:
Kihwal Lee 2015-10-15 11:26:14 -05:00 committed by Junping Du
parent fc01ec866a
commit d8416aec3c
3 changed files with 48 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -49,6 +49,9 @@ Release 2.6.4 - UNRELEASED
HDFS-8722. Optimize datanode writes for small writes and flushes.
(Kihwal Lee)
HDFS-9220. Reading small file (< 512 bytes) that is open for append fails
due to incorrect checksum (Jing Zhao via kihwal)
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
View File

@ -742,11 +742,10 @@ private int receivePacket() throws IOException {
final int offset = checksumBuf.arrayOffset() +
checksumBuf.position() + skip;
final int end = offset + checksumLen - skip;
// If offset > end, there is no more checksum to write.
// If offset >= end, there is no more checksum to write.
// I.e. a partial chunk checksum rewrite happened and there is no
// more to write after that.
if (offset > end) {
assert crcBytes != null;
if (offset >= end && doCrcRecalc) {
lastCrc = crcBytes;
} else {
final int remainingBytes = checksumLen - skip;

43
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
View File

@ -28,6 +28,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@ -390,4 +391,46 @@ public void testComplexAppend() throws IOException {
//
assertTrue("testComplexAppend Worker encountered exceptions.", globalStatus);
}
/**
* Make sure when the block length after appending is less than 512 bytes, the
* checksum re-calculation and overwrite are performed correctly.
*/
@Test
public void testAppendLessThanChecksumChunk() throws Exception {
final byte[] buf = new byte[1024];
final MiniDFSCluster cluster = new MiniDFSCluster
.Builder(new HdfsConfiguration()).numDataNodes(1).build();
cluster.waitActive();
DistributedFileSystem fs = cluster.getFileSystem();
FSDataOutputStream out = null;
FSDataInputStream in = null;
try {
final int len1 = 200;
final int len2 = 300;
final Path p = new Path("/foo");
out = fs.create(p);
out.write(buf, 0, len1);
out.close();
out = fs.append(p);
out.write(buf, 0, len2);
// flush but leave open
out.hflush();
// read data to verify the replica's content and checksum are correct
in = fs.open(p);
final int length = in.read(0, buf, 0, len1 + len2);
assertTrue(length > 0);
} finally {
if (in != null) {
in.close();
}
if (out != null) {
out.close();
}
cluster.shutdown();
}
}
}