Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-5809. BlockPoolSliceScanner and high speed hdfs appending make datanode to drop into infinite loop (cmccabe) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1610790 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Colin McCabe 2014-07-15 18:17:13 +00:00
parent 65b0cfc96b
commit 7fe6ab7f91
3 changed files with 15 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -306,6 +306,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6678. MiniDFSCluster may still be partially running after initialization
fails. (cnauroth)
HDFS-5809. BlockPoolSliceScanner and high speed hdfs appending make
datanode to drop into infinite loop (cmccabe)
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES

21
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
View File

@ -310,17 +310,10 @@ void deleteBlocks(Block[] blocks) {
}
}
private synchronized void updateScanStatus(Block block,
private synchronized void updateScanStatus(BlockScanInfo info,
ScanType type,
boolean scanOk) {
BlockScanInfo info = blockMap.get(block);
if ( info != null ) {
delBlockInfo(info);
} else {
// It might already be removed. Thats ok, it will be caught next time.
info = new BlockScanInfo(block);
}
long now = Time.monotonicNow();
info.lastScanType = type;
@ -334,8 +327,8 @@ private synchronized void updateScanStatus(Block block,
}
if (verificationLog != null) {
verificationLog.append(now, block.getGenerationStamp(),
block.getBlockId());
verificationLog.append(now, info.getGenerationStamp(),
info.getBlockId());
}
}
@ -434,11 +427,13 @@ void verifyBlock(ExtendedBlock block) {
totalTransientErrors++;
}
updateScanStatus(block.getLocalBlock(), ScanType.VERIFICATION_SCAN, true);
updateScanStatus((BlockScanInfo)block.getLocalBlock(),
ScanType.VERIFICATION_SCAN, true);
return;
} catch (IOException e) {
updateScanStatus(block.getLocalBlock(), ScanType.VERIFICATION_SCAN, false);
updateScanStatus((BlockScanInfo)block.getLocalBlock(),
ScanType.VERIFICATION_SCAN, false);
// If the block does not exists anymore, then its not an error
if (!dataset.contains(block)) {
@ -497,7 +492,7 @@ private synchronized boolean isFirstBlockProcessed() {
// Picks one block and verifies it
private void verifyFirstBlock() {
Block block = null;
BlockScanInfo block = null;
synchronized (this) {
if (!blockInfoSet.isEmpty()) {
block = blockInfoSet.first();

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
View File

@ -116,7 +116,8 @@ public static InterDatanodeProtocol createInterDatanodeProtocolProxy(
public static void runBlockScannerForBlock(DataNode dn, ExtendedBlock b) {
BlockPoolSliceScanner bpScanner = getBlockPoolScanner(dn, b);
bpScanner.verifyBlock(b);
bpScanner.verifyBlock(new ExtendedBlock(b.getBlockPoolId(),
new BlockPoolSliceScanner.BlockScanInfo(b.getLocalBlock())));
}
private static BlockPoolSliceScanner getBlockPoolScanner(DataNode dn,