HDFS-9036. In BlockPlacementPolicyWithNodeGroup#chooseLocalStorage , random node is selected eventhough fallbackToLocalRack is true. (Contributed by J.Andreina)
(cherry picked from commit c715650385
)
This commit is contained in:
parent
1857621671
commit
7545488b12
|
@ -968,6 +968,10 @@ Release 2.8.0 - UNRELEASED
|
|||
HDFS-8581. ContentSummary on / skips further counts on yielding lock
|
||||
(J.Andreina via vinayakumarb)
|
||||
|
||||
HDFS-9036. In BlockPlacementPolicyWithNodeGroup#chooseLocalStorage , random
|
||||
node is selected eventhough fallbackToLocalRack is true.
|
||||
(J.Andreina via vinayakumarb)
|
||||
|
||||
Release 2.7.2 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
@ -31,7 +31,7 @@ import org.apache.hadoop.net.NodeBase;
|
|||
* for placing block replicas on environment with node-group layer.
|
||||
* The replica placement strategy is adjusted to:
|
||||
* If the writer is on a datanode, the 1st replica is placed on the local
|
||||
* node (or local node-group), otherwise a random datanode.
|
||||
* node(or local node-group or on local rack), otherwise a random datanode.
|
||||
* The 2nd replica is placed on a datanode that is on a different rack with 1st
|
||||
* replica node.
|
||||
* The 3rd replica is placed on a datanode which is on a different node-group
|
||||
|
@ -167,7 +167,7 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau
|
|||
/* choose one node from the nodegroup that <i>localMachine</i> is on.
|
||||
* if no such node is available, choose one node from the nodegroup where
|
||||
* a second replica is on.
|
||||
* if still no such node is available, choose a random node in the cluster.
|
||||
* if still no such node is available, return null.
|
||||
* @return the chosen node
|
||||
*/
|
||||
private DatanodeStorageInfo chooseLocalNodeGroup(
|
||||
|
@ -197,14 +197,12 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau
|
|||
excludedNodes, blocksize, maxNodesPerRack, results,
|
||||
avoidStaleNodes, storageTypes);
|
||||
} catch(NotEnoughReplicasException e2) {
|
||||
//otherwise randomly choose one from the network
|
||||
return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
|
||||
maxNodesPerRack, results, avoidStaleNodes, storageTypes);
|
||||
//otherwise return null
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
//otherwise randomly choose one from the network
|
||||
return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
|
||||
maxNodesPerRack, results, avoidStaleNodes, storageTypes);
|
||||
//otherwise return null
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -486,6 +486,25 @@ public class TestReplicationPolicyWithNodeGroup {
|
|||
verifyNoTwoTargetsOnSameNodeGroup(targets);
|
||||
}
|
||||
|
||||
/**
|
||||
* In this testcase, client is dataNodes[7], but it is not qualified
|
||||
* to be chosen. And there is no other node available on client Node group.
|
||||
* So the 1st replica should be placed on client local rack dataNodes[6]
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test
|
||||
public void testChooseTargetForLocalStorage() throws Exception {
|
||||
updateHeartbeatWithUsage(dataNodes[7],
|
||||
2* HdfsServerConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
|
||||
(HdfsServerConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L,
|
||||
0L, 0L, 0, 0); // no space
|
||||
|
||||
DatanodeStorageInfo[] targets;
|
||||
targets = chooseTarget(1, dataNodes[7]);
|
||||
assertEquals(targets.length, 1);
|
||||
assertTrue(targets[0].getDatanodeDescriptor().equals(dataNodes[6]));
|
||||
}
|
||||
|
||||
/**
|
||||
* This testcase tests re-replication, when dataNodes[0] is already chosen.
|
||||
* So the 1st replica can be placed on random rack.
|
||||
|
|
Loading…
Reference in New Issue