svn merge -c 1429653 from trunk for HDFS-4351. In BlockPlacementPolicyDefault.chooseTarget(..), numOfReplicas needs to be updated when avoiding stale nodes.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1429654 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
b035a444e1
commit
abf7e8dcc0
|
@ -459,6 +459,9 @@ Release 2.0.3-alpha - Unreleased
|
|||
HDFS-4017. Unclosed FileInputStream in GetJournalEditServlet
|
||||
(Chao Shi via todd)
|
||||
|
||||
HDFS-4351. In BlockPlacementPolicyDefault.chooseTarget(..), numOfReplicas
|
||||
needs to be updated when avoiding stale nodes. (Andrew Wang via szetszwo)
|
||||
|
||||
Release 2.0.2-alpha - 2012-09-07
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
@ -234,13 +234,18 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
|
|||
+ totalReplicasExpected + "\n"
|
||||
+ e.getMessage());
|
||||
if (avoidStaleNodes) {
|
||||
// ecxludedNodes now has - initial excludedNodes, any nodes that were
|
||||
// chosen and nodes that were tried but were not chosen because they
|
||||
// were stale, decommissioned or for any other reason a node is not
|
||||
// chosen for write. Retry again now not avoiding stale node
|
||||
// Retry chooseTarget again, this time not avoiding stale nodes.
|
||||
|
||||
// excludedNodes contains the initial excludedNodes and nodes that were
|
||||
// not chosen because they were stale, decommissioned, etc.
|
||||
// We need to additionally exclude the nodes that were added to the
|
||||
// result list in the successful calls to choose*() above.
|
||||
for (Node node : results) {
|
||||
oldExcludedNodes.put(node, node);
|
||||
}
|
||||
// Set numOfReplicas, since it can get out of sync with the result list
|
||||
// if the NotEnoughReplicasException was thrown in chooseRandom().
|
||||
numOfReplicas = totalReplicasExpected - results.size();
|
||||
return chooseTarget(numOfReplicas, writer, oldExcludedNodes, blocksize,
|
||||
maxNodesPerRack, results, false);
|
||||
}
|
||||
|
@ -505,7 +510,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
|
|||
if (LOG.isDebugEnabled()) {
|
||||
threadLocalBuilder.get().append(node.toString()).append(": ")
|
||||
.append("Node ").append(NodeBase.getPath(node))
|
||||
.append(" is not chosen because the node is staled ");
|
||||
.append(" is not chosen because the node is stale ");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
|
@ -381,6 +381,24 @@ public class TestReplicationPolicy {
|
|||
assertTrue(cluster.isOnSameRack(targets[1], targets[2]));
|
||||
assertFalse(cluster.isOnSameRack(targets[0], targets[1]));
|
||||
}
|
||||
|
||||
/**
|
||||
* In this testcase, it tries to choose more targets than available nodes and
|
||||
* check the result, with stale node avoidance on the write path enabled.
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test
|
||||
public void testChooseTargetWithMoreThanAvailableNodesWithStaleness()
|
||||
throws Exception {
|
||||
try {
|
||||
namenode.getNamesystem().getBlockManager().getDatanodeManager()
|
||||
.setAvoidStaleDataNodesForWrite(true);
|
||||
testChooseTargetWithMoreThanAvailableNodes();
|
||||
} finally {
|
||||
namenode.getNamesystem().getBlockManager().getDatanodeManager()
|
||||
.setAvoidStaleDataNodesForWrite(false);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* In this testcase, it tries to choose more targets than available nodes and
|
||||
|
@ -388,7 +406,7 @@ public class TestReplicationPolicy {
|
|||
* @throws Exception
|
||||
*/
|
||||
@Test
|
||||
public void testChooseTargetWithMoreThanAvaiableNodes() throws Exception {
|
||||
public void testChooseTargetWithMoreThanAvailableNodes() throws Exception {
|
||||
// make data node 0 & 1 to be not qualified to choose: not enough disk space
|
||||
for(int i=0; i<2; i++) {
|
||||
dataNodes[i].updateHeartbeat(
|
||||
|
|
Loading…
Reference in New Issue