Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14514. Actual read size of open file in encryption zone still larger than listing size even after enabling HDFS-11402 in Hadoop 2. Contributed by Siyao Meng, Stephen O'Donnell. 

Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
Co-authored-by: Stephen O'Donnell <sodonnell@cloudera.com>
This commit is contained in:
Siyao Meng 2019-05-29 17:27:39 -07:00 committed by Wei-Chiu Chuang
parent e3d4f6bfed
commit cecba551aa
2 changed files with 21 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
View File

@ -844,26 +844,18 @@ public class DFSInputStream extends FSInputStream
@Override @Override
public int doRead(BlockReader blockReader, int off, int len) public int doRead(BlockReader blockReader, int off, int len)
throws IOException { throws IOException {
int oldpos = buf.position(); ByteBuffer tmpBuf = buf.duplicate();
int oldlimit = buf.limit(); tmpBuf.limit(tmpBuf.position() + len);
boolean success = false; int nRead = blockReader.read(tmpBuf);
try { updateReadStatistics(readStatistics, nRead, blockReader);
int ret = blockReader.read(buf); dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
success = true; nRead);
updateReadStatistics(readStatistics, ret, blockReader); if (nRead == 0) {
dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(), DFSClient.LOG.warn("zero");
ret); } else if (nRead > 0) {
if (ret == 0) { buf.position(buf.position() + nRead);
DFSClient.LOG.warn("zero");
}
return ret;
} finally {
if (!success) {
// Reset to original state so that retries work correctly.
buf.position(oldpos);
buf.limit(oldlimit);
}
} }
return nRead;
} }
@Override @Override

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
View File

@ -189,6 +189,8 @@ public class TestEncryptionZones {
testRootDir = new File(testRoot).getAbsoluteFile(); testRootDir = new File(testRoot).getAbsoluteFile();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
getKeyProviderURI()); getKeyProviderURI());
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES,
true);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
// Lower the batch size for testing // Lower the batch size for testing
conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
@ -1418,6 +1420,14 @@ public class TestEncryptionZones {
assertEquals("Got unexpected ez path", zone.toString(), assertEquals("Got unexpected ez path", zone.toString(),
dfsAdmin.getEncryptionZoneForPath(snap1Zone).getPath().toString()); dfsAdmin.getEncryptionZoneForPath(snap1Zone).getPath().toString());
// Append the file
DFSTestUtil.appendFile(fs, zoneFile, len);
// Verify file content in the snapshot
final Path snapshottedZoneFile = new Path(
snap1.toString() + "/" + zone.getName() + "/" + zoneFile.getName());
assertEquals("Contents of snapshotted file have changed unexpectedly",
contents, DFSTestUtil.readFile(fs, snapshottedZoneFile));
// Now delete the encryption zone, recreate the dir, and take another // Now delete the encryption zone, recreate the dir, and take another
// snapshot // snapshot
fsWrapper.delete(zone, true); fsWrapper.delete(zone, true);
@ -1470,8 +1480,6 @@ public class TestEncryptionZones {
assertEquals("Unexpected ez key", TEST_KEY2, listZone.getKeyName()); assertEquals("Unexpected ez key", TEST_KEY2, listZone.getKeyName());
// Verify contents of the snapshotted file // Verify contents of the snapshotted file
final Path snapshottedZoneFile = new Path(
snap1.toString() + "/" + zone.getName() + "/" + zoneFile.getName());
assertEquals("Contents of snapshotted file have changed unexpectedly", assertEquals("Contents of snapshotted file have changed unexpectedly",
contents, DFSTestUtil.readFile(fs, snapshottedZoneFile)); contents, DFSTestUtil.readFile(fs, snapshottedZoneFile));