Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-8809. HDFS fsck reports under construction blocks as CORRUPT. Contributed by Jing Zhao.

This commit is contained in:
Jing Zhao 2015-08-20 16:31:24 -07:00
parent 7642f64c24
commit c8bca62718
3 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -1194,6 +1194,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8922. Link the native_mini_dfs test library with libdl, since IBM Java HDFS-8922. Link the native_mini_dfs test library with libdl, since IBM Java
requires it (Ayappan via Colin P. McCabe) requires it (Ayappan via Colin P. McCabe)
HDFS-8809. HDFS fsck reports under construction blocks as "CORRUPT". (jing9)
Release 2.7.2 - UNRELEASED Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
View File

@ -528,6 +528,9 @@ private void collectBlocksSummary(String parent, HdfsFileStatus file, Result res
LocatedBlocks blocks) throws IOException { LocatedBlocks blocks) throws IOException {
String path = file.getFullName(parent); String path = file.getFullName(parent);
boolean isOpen = blocks.isUnderConstruction(); boolean isOpen = blocks.isUnderConstruction();
if (isOpen && !showOpenFiles) {
return;
}
int missing = 0; int missing = 0;
int corrupt = 0; int corrupt = 0;
long missize = 0; long missize = 0;
@ -536,8 +539,15 @@ private void collectBlocksSummary(String parent, HdfsFileStatus file, Result res
int misReplicatedPerFile = 0; int misReplicatedPerFile = 0;
StringBuilder report = new StringBuilder(); StringBuilder report = new StringBuilder();
int blockNumber = 0; int blockNumber = 0;
final LocatedBlock lastBlock = blocks.getLastLocatedBlock();
for (LocatedBlock lBlk : blocks.getLocatedBlocks()) { for (LocatedBlock lBlk : blocks.getLocatedBlocks()) {
ExtendedBlock block = lBlk.getBlock(); ExtendedBlock block = lBlk.getBlock();
if (!blocks.isLastBlockComplete() && lastBlock != null &&
lastBlock.getBlock().equals(block)) {
// this is the last block and this is not complete. ignore it since
// it is under construction
continue;
}
BlockManager bm = namenode.getNamesystem().getBlockManager(); BlockManager bm = namenode.getNamesystem().getBlockManager();
final BlockInfo storedBlock = bm.getStoredBlock( final BlockInfo storedBlock = bm.getStoredBlock(

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
View File

@ -65,6 +65,7 @@
import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSInputStream; import org.apache.hadoop.hdfs.DFSInputStream;
import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.DistributedFileSystem;
@ -612,6 +613,7 @@ public void testFsckOpenFiles() throws Exception {
out.write(randomString.getBytes()); out.write(randomString.getBytes());
writeCount++; writeCount++;
} }
((DFSOutputStream) out.getWrappedStream()).hflush();
// We expect the filesystem to be HEALTHY and show one open file // We expect the filesystem to be HEALTHY and show one open file
outStr = runFsck(conf, 0, true, topDir); outStr = runFsck(conf, 0, true, topDir);
System.out.println(outStr); System.out.println(outStr);