Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-9876. shouldProcessOverReplicated should not count number of pending replicas. Contributed by Jing Zhao.

This commit is contained in:
Jing Zhao 2016-03-01 18:41:57 -08:00
parent 4abb2fa687
commit f2ba7da4f0
2 changed files with 23 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -455,6 +455,9 @@ Trunk (Unreleased)
HDFS-9867. Missing block exception should carry locatedBlocks information.
(Mingliang Liu via jing9)
HDFS-9876. shouldProcessOverReplicated should not count number of pending
replicas. (jing9)
BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS
HDFS-7347. Configurable erasure coding policy for individual files and

38
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -2893,7 +2893,7 @@ public class BlockManager implements BlockStatsMXBean {
} else {
updateNeededReplications(storedBlock, curReplicaDelta, 0);
}
if (shouldProcessOverReplicated(num, pendingNum, fileReplication)) {
if (shouldProcessOverReplicated(num, fileReplication)) {
processOverReplicatedBlock(storedBlock, fileReplication, node, delNodeHint);
}
// If the file replication has reached desired value
@ -2912,8 +2912,8 @@ public class BlockManager implements BlockStatsMXBean {
}
private boolean shouldProcessOverReplicated(NumberReplicas num,
int pendingNum, int expectedNum) {
int numCurrent = num.liveReplicas() + pendingNum;
int expectedNum) {
final int numCurrent = num.liveReplicas();
return numCurrent > expectedNum ||
(numCurrent == expectedNum && num.redundantInternalBlocks() > 0);
}
@ -3131,7 +3131,7 @@ public class BlockManager implements BlockStatsMXBean {
}
}
if (shouldProcessOverReplicated(num, 0, expectedReplication)) {
if (shouldProcessOverReplicated(num, expectedReplication)) {
if (num.replicasOnStaleNodes() > 0) {
// If any of the replicas of this block are on nodes that are
// considered "stale", then these replicas may in fact have
@ -3268,7 +3268,6 @@ public class BlockManager implements BlockStatsMXBean {
assert storedBlock instanceof BlockInfoStriped;
BlockInfoStriped sblk = (BlockInfoStriped) storedBlock;
short groupSize = sblk.getTotalBlockNum();
BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true);
// find all duplicated indices
BitSet found = new BitSet(groupSize); //indices found
@ -3283,14 +3282,6 @@ public class BlockManager implements BlockStatsMXBean {
found.set(index);
storage2index.put(storage, index);
}
// the number of target left replicas equals to the of number of the found
// indices.
int numOfTarget = found.cardinality();
final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(
bc.getStoragePolicyID());
final List<StorageType> excessTypes = storagePolicy.chooseExcess(
(short)numOfTarget, DatanodeStorageInfo.toStorageTypes(nonExcess));
// use delHint only if delHint is duplicated
final DatanodeStorageInfo delStorageHint =
@ -3302,6 +3293,19 @@ public class BlockManager implements BlockStatsMXBean {
}
}
// cardinality of found indicates the expected number of internal blocks
final int numOfTarget = found.cardinality();
final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(
bc.getStoragePolicyID());
final List<StorageType> excessTypes = storagePolicy.chooseExcess(
(short) numOfTarget, DatanodeStorageInfo.toStorageTypes(nonExcess));
if (excessTypes.isEmpty()) {
LOG.warn("excess types chosen for block {} among storages {} is empty",
storedBlock, nonExcess);
return;
}
BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true);
// for each duplicated index, delete some replicas until only one left
for (int targetIndex = duplicated.nextSetBit(0); targetIndex >= 0;
targetIndex = duplicated.nextSetBit(targetIndex + 1)) {
@ -3312,9 +3316,7 @@ public class BlockManager implements BlockStatsMXBean {
candidates.add(storage);
}
}
Block internalBlock = new Block(storedBlock);
internalBlock.setBlockId(storedBlock.getBlockId() + targetIndex);
while (candidates.size() > 1) {
if (candidates.size() > 1) {
List<DatanodeStorageInfo> replicasToDelete = placementPolicy
.chooseReplicasToDelete(nonExcess, candidates, (short) 1,
excessTypes, null, null);
@ -3749,7 +3751,7 @@ public class BlockManager implements BlockStatsMXBean {
final BlockInfo block = it.next();
int expectedReplication = this.getReplication(block);
NumberReplicas num = countNodes(block);
if (shouldProcessOverReplicated(num, 0, expectedReplication)) {
if (shouldProcessOverReplicated(num, expectedReplication)) {
// over-replicated block
processOverReplicatedBlock(block, (short) expectedReplication, null,
null);
@ -3886,7 +3888,7 @@ public class BlockManager implements BlockStatsMXBean {
neededReplications.add(block, n.liveReplicas() + pending,
n.readOnlyReplicas(),
n.decommissionedAndDecommissioning(), expected);
} else if (shouldProcessOverReplicated(n, 0, expected)) {
} else if (shouldProcessOverReplicated(n, expected)) {
processOverReplicatedBlock(block, expected, null, null);
}
}