Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-5809. BlockPoolSliceScanner and high speed hdfs appending make datanode to drop into infinite loop (cmccabe) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1610790 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Colin McCabe 2014-07-15 18:17:13 +00:00
parent 65b0cfc96b
commit 7fe6ab7f91
3 changed files with 15 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -306,6 +306,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6678. MiniDFSCluster may still be partially running after initialization HDFS-6678. MiniDFSCluster may still be partially running after initialization
fails. (cnauroth) fails. (cnauroth)
HDFS-5809. BlockPoolSliceScanner and high speed hdfs appending make
datanode to drop into infinite loop (cmccabe)
Release 2.5.0 - UNRELEASED Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

21
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
View File

@ -310,17 +310,10 @@ class BlockPoolSliceScanner {
} }
} }
private synchronized void updateScanStatus(Block block, private synchronized void updateScanStatus(BlockScanInfo info,
ScanType type, ScanType type,
boolean scanOk) { boolean scanOk) {
BlockScanInfo info = blockMap.get(block);
if ( info != null ) {
delBlockInfo(info); delBlockInfo(info);
} else {
// It might already be removed. Thats ok, it will be caught next time.
info = new BlockScanInfo(block);
}
long now = Time.monotonicNow(); long now = Time.monotonicNow();
info.lastScanType = type; info.lastScanType = type;
@ -334,8 +327,8 @@ class BlockPoolSliceScanner {
} }
if (verificationLog != null) { if (verificationLog != null) {
verificationLog.append(now, block.getGenerationStamp(), verificationLog.append(now, info.getGenerationStamp(),
block.getBlockId()); info.getBlockId());
} }
} }
@ -434,11 +427,13 @@ class BlockPoolSliceScanner {
totalTransientErrors++; totalTransientErrors++;
} }
updateScanStatus(block.getLocalBlock(), ScanType.VERIFICATION_SCAN, true); updateScanStatus((BlockScanInfo)block.getLocalBlock(),
ScanType.VERIFICATION_SCAN, true);
return; return;
} catch (IOException e) { } catch (IOException e) {
updateScanStatus(block.getLocalBlock(), ScanType.VERIFICATION_SCAN, false); updateScanStatus((BlockScanInfo)block.getLocalBlock(),
ScanType.VERIFICATION_SCAN, false);
// If the block does not exists anymore, then its not an error // If the block does not exists anymore, then its not an error
if (!dataset.contains(block)) { if (!dataset.contains(block)) {
@ -497,7 +492,7 @@ class BlockPoolSliceScanner {
// Picks one block and verifies it // Picks one block and verifies it
private void verifyFirstBlock() { private void verifyFirstBlock() {
Block block = null; BlockScanInfo block = null;
synchronized (this) { synchronized (this) {
if (!blockInfoSet.isEmpty()) { if (!blockInfoSet.isEmpty()) {
block = blockInfoSet.first(); block = blockInfoSet.first();

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
View File

@ -116,7 +116,8 @@ public class DataNodeTestUtils {
public static void runBlockScannerForBlock(DataNode dn, ExtendedBlock b) { public static void runBlockScannerForBlock(DataNode dn, ExtendedBlock b) {
BlockPoolSliceScanner bpScanner = getBlockPoolScanner(dn, b); BlockPoolSliceScanner bpScanner = getBlockPoolScanner(dn, b);
bpScanner.verifyBlock(b); bpScanner.verifyBlock(new ExtendedBlock(b.getBlockPoolId(),
new BlockPoolSliceScanner.BlockScanInfo(b.getLocalBlock())));
} }
private static BlockPoolSliceScanner getBlockPoolScanner(DataNode dn, private static BlockPoolSliceScanner getBlockPoolScanner(DataNode dn,