Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-10975. fsck -list-corruptfileblocks does not report corrupt EC files. Contributed by Takanobu Asanuma.

This commit is contained in:
Wei-Chiu Chuang 2016-10-21 15:43:12 -07:00
parent be72372248
commit df857f0d10
2 changed files with 14 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
View File

@ -660,6 +660,13 @@ private void collectBlocksSummary(String parent, HdfsFileStatus file,
decommissioningReplicas; decommissioningReplicas;
res.totalReplicas += totalReplicasPerBlock; res.totalReplicas += totalReplicasPerBlock;
boolean isMissing;
if (storedBlock.isStriped()) {
isMissing = totalReplicasPerBlock < minReplication;
} else {
isMissing = totalReplicasPerBlock == 0;
}
// count expected replicas // count expected replicas
short targetFileReplication; short targetFileReplication;
if (file.getErasureCodingPolicy() != null) { if (file.getErasureCodingPolicy() != null) {
@ -697,7 +704,7 @@ private void collectBlocksSummary(String parent, HdfsFileStatus file,
res.numMinReplicatedBlocks++; res.numMinReplicatedBlocks++;
// count missing replicas / under replicated blocks // count missing replicas / under replicated blocks
if (totalReplicasPerBlock < targetFileReplication && totalReplicasPerBlock > 0) { if (totalReplicasPerBlock < targetFileReplication && !isMissing) {
res.missingReplicas += (targetFileReplication - totalReplicasPerBlock); res.missingReplicas += (targetFileReplication - totalReplicasPerBlock);
res.numUnderReplicatedBlocks += 1; res.numUnderReplicatedBlocks += 1;
underReplicatedPerFile++; underReplicatedPerFile++;
@ -737,12 +744,6 @@ private void collectBlocksSummary(String parent, HdfsFileStatus file,
String blkName = block.toString(); String blkName = block.toString();
report.append(blockNumber + ". " + blkName + " len=" + report.append(blockNumber + ". " + blkName + " len=" +
block.getNumBytes()); block.getNumBytes());
boolean isMissing;
if (storedBlock.isStriped()) {
isMissing = totalReplicasPerBlock < minReplication;
} else {
isMissing = totalReplicasPerBlock == 0;
}
if (isMissing && !isCorrupt) { if (isMissing && !isCorrupt) {
// If the block is corrupted, it means all its available replicas are // If the block is corrupted, it means all its available replicas are
// corrupted in the case of replication, and it means the state of the // corrupted in the case of replication, and it means the state of the

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
View File

@ -2013,6 +2013,9 @@ public void testFsckCorruptECFile() throws Exception {
String outStr = runFsck(conf, 1, true, "/"); String outStr = runFsck(conf, 1, true, "/");
assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
assertTrue(outStr.contains("Under-erasure-coded block groups:\t0"));
outStr = runFsck(conf, -1, true, "/", "-list-corruptfileblocks");
assertTrue(outStr.contains("has 1 CORRUPT files"));
} }
@Test (timeout = 300000) @Test (timeout = 300000)
@ -2053,6 +2056,9 @@ public void testFsckMissingECFile() throws Exception {
"-locations"); "-locations");
assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
assertTrue(outStr.contains("Live_repl=" + (dataBlocks - 1))); assertTrue(outStr.contains("Live_repl=" + (dataBlocks - 1)));
assertTrue(outStr.contains("Under-erasure-coded block groups:\t0"));
outStr = runFsck(conf, -1, true, "/", "-list-corruptfileblocks");
assertTrue(outStr.contains("has 1 CORRUPT files"));
} }
private void waitForUnrecoverableBlockGroup(Configuration configuration) private void waitForUnrecoverableBlockGroup(Configuration configuration)