diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 4e7b91955a1..7ff3c780924 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1080,6 +1080,9 @@ Release 2.7.0 - UNRELEASED HDFS-6565. Use jackson instead jetty json in hdfs-client. (Akira Ajisaka via wheat9) + HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file + includes non-snapshotted content. (Charles Lamb via atm) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index abcd847dbfb..aac7b511433 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -2220,6 +2220,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, // get block checksum for each block long remaining = length; + if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) { + remaining = Math.min(length, blockLocations.getFileLength()); + } for(int i = 0; i < locatedblocks.size() && remaining > 0; i++) { if (refetchBlocks) { // refetch to get fresh tokens blockLocations = callGetBlockLocations(namenode, src, 0, length); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java index 98aafc1b308..d53140f14ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java @@ -20,8 +20,8 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot; import java.io.ByteArrayOutputStream; import java.io.PrintStream; - import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hdfs.AppendTestUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -29,8 +29,9 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - +import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -103,17 +104,35 @@ public class TestSnapshotFileLength { Path file1snap1 = SnapshotTestHelper.getSnapshotPath(sub, snapshot1, file1Name); + final FileChecksum snapChksum1 = hdfs.getFileChecksum(file1snap1); + assertThat("file and snapshot file checksums are not equal", + hdfs.getFileChecksum(file1), is(snapChksum1)); + // Append to the file. FSDataOutputStream out = hdfs.append(file1); + // Nothing has been appended yet. All checksums should still be equal. + assertThat("file and snapshot checksums (open for append) are not equal", + hdfs.getFileChecksum(file1), is(snapChksum1)); + assertThat("snapshot checksum (post-open for append) has changed", + hdfs.getFileChecksum(file1snap1), is(snapChksum1)); try { AppendTestUtil.write(out, 0, toAppend); // Test reading from snapshot of file that is open for append byte[] dataFromSnapshot = DFSTestUtil.readFileBuffer(hdfs, file1snap1); assertThat("Wrong data size in snapshot.", dataFromSnapshot.length, is(origLen)); + // Verify that checksum didn't change + assertThat("snapshot file checksum (pre-close) has changed", + hdfs.getFileChecksum(file1), is(snapChksum1)); + assertThat("snapshot checksum (post-append) has changed", + hdfs.getFileChecksum(file1snap1), is(snapChksum1)); } finally { out.close(); } + assertThat("file and snapshot file checksums (post-close) are equal", + hdfs.getFileChecksum(file1), not(snapChksum1)); + assertThat("snapshot file checksum (post-close) has changed", + hdfs.getFileChecksum(file1snap1), is(snapChksum1)); // Make sure we can read the entire file via its non-snapshot path. fileStatus = hdfs.getFileStatus(file1);