Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix cost balancing so assignment/balancing doesn't get stuck

This commit is contained in:
Nelson Ray 2013-02-26 18:58:16 -08:00
parent 2e1e1753d5
commit d1634fe5ef
3 changed files with 16 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
server/src/main/java/com/metamx/druid/master/BalancerCostAnalyzer.java
View File

@ -188,19 +188,26 @@ public class BalancerCostAnalyzer
* A DataSegment that we are proposing to move.
* @param serverHolders
* An iterable of ServerHolders for a particular tier.
* @param assign
* A boolean that is true if used in assignment else false in balancing.
* @return A ServerHolder with the new home for a segment.
*/
public ServerHolder findNewSegmentHome(final DataSegment proposalSegment, final Iterable<ServerHolder> serverHolders)
public ServerHolder findNewSegmentHome(
final DataSegment proposalSegment,
final Iterable<ServerHolder> serverHolders,
final boolean assign
)
{
final long proposalSegmentSize = proposalSegment.getSize();
double minCost = Double.MAX_VALUE;
ServerHolder toServer = null;
for (ServerHolder server : serverHolders) {
/** Don't calculate cost if the server doesn't have enough space or is serving/loading the segment. */
/** Don't calculate cost if the server doesn't have enough space or is loading the segment */
if (proposalSegmentSize > server.getAvailableSize()
|| server.isServingSegment(proposalSegment)
|| server.isLoadingSegment(proposalSegment)) {
|| server.isLoadingSegment(proposalSegment)
/** or if the ask is assignment and the server is serving the segment. */
|| (assign && server.isServingSegment(proposalSegment)) ) {
continue;
}

6
server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java
View File

@ -111,7 +111,7 @@ public class DruidMasterBalancer implements DruidMasterHelper
while (iter < maxSegmentsToMove) {
iter++;
final BalancerSegmentHolder segmentToMove = analyzer.pickSegmentToMove(serverHolderList, numSegments);
final ServerHolder holder = analyzer.findNewSegmentHome(segmentToMove.getSegment(), serverHolderList);
final ServerHolder holder = analyzer.findNewSegmentHome(segmentToMove.getSegment(), serverHolderList, false);
if (holder == null) {
continue;
}
@ -128,11 +128,13 @@ public class DruidMasterBalancer implements DruidMasterHelper
stats.addToTieredStat("initialCost", tier, (long) initialTotalCost);
stats.addToTieredStat("normalization", tier, (long) normalization);
stats.addToTieredStat("normalizedInitialCostTimesOneThousand", tier, (long) (normalizedInitialCost * 1000));
stats.addToTieredStat("movedCount", tier, currentlyMovingSegments.get(tier).size());
log.info(
"Initial Total Cost: [%f], Initial Normalized Cost: [%f], Segments Moved: [%d]",
"Initial Total Cost: [%f], Normalization: [%f], Initial Normalized Cost: [%f], Segments Moved: [%d]",
initialTotalCost,
normalization,
normalizedInitialCost,
currentlyMovingSegments.get(tier).size()
);

2
server/src/main/java/com/metamx/druid/master/rules/LoadRule.java
View File

@ -91,7 +91,7 @@ public abstract class LoadRule implements Rule
final MasterStats stats = new MasterStats();
while (totalReplicants < expectedReplicants) {
final ServerHolder holder = analyzer.findNewSegmentHome(segment, serverHolderList);
final ServerHolder holder = analyzer.findNewSegmentHome(segment, serverHolderList, true);
if (holder == null) {
log.warn(