Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

formatted files

This commit is contained in:
Dhruv Parthasarathy 2013-07-29 18:10:08 -07:00
parent ac0f0afdbf
commit 4e4d582e07
6 changed files with 57 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/src/main/java/com/metamx/druid/master/BalancerStrategy.java
View File

@ -25,7 +25,9 @@ import java.util.List;
public interface BalancerStrategy public interface BalancerStrategy
{ {
public ServerHolder findNewSegmentHome(final DataSegment proposalSegment,final List<ServerHolder> serverHolders); public ServerHolder findNewSegmentHome(final DataSegment proposalSegment, final List<ServerHolder> serverHolders);
public BalancerSegmentHolder pickSegmentToMove(final List<ServerHolder> serverHolders); public BalancerSegmentHolder pickSegmentToMove(final List<ServerHolder> serverHolders);
public void emitStats(String tier, MasterStats stats, List<ServerHolder> serverHolderList); public void emitStats(String tier, MasterStats stats, List<ServerHolder> serverHolderList);
} }

29
server/src/main/java/com/metamx/druid/master/CostBalancerStrategy.java
View File

@ -39,7 +39,7 @@ public class CostBalancerStrategy implements BalancerStrategy
public CostBalancerStrategy(DateTime referenceTimestamp) public CostBalancerStrategy(DateTime referenceTimestamp)
{ {
this.referenceTimestamp=referenceTimestamp; this.referenceTimestamp = referenceTimestamp;
} }
@Override @Override
@ -208,25 +208,26 @@ public class CostBalancerStrategy implements BalancerStrategy
} }
@Override @Override
public void emitStats( String tier, public void emitStats(
String tier,
MasterStats stats, List<ServerHolder> serverHolderList MasterStats stats, List<ServerHolder> serverHolderList
) )
{ {
final double initialTotalCost = calculateInitialTotalCost(serverHolderList); final double initialTotalCost = calculateInitialTotalCost(serverHolderList);
final double normalization = calculateNormalization(serverHolderList); final double normalization = calculateNormalization(serverHolderList);
final double normalizedInitialCost = initialTotalCost / normalization; final double normalizedInitialCost = initialTotalCost / normalization;
stats.addToTieredStat("initialCost", tier, (long) initialTotalCost); stats.addToTieredStat("initialCost", tier, (long) initialTotalCost);
stats.addToTieredStat("normalization", tier, (long) normalization); stats.addToTieredStat("normalization", tier, (long) normalization);
stats.addToTieredStat("normalizedInitialCostTimesOneThousand", tier, (long) (normalizedInitialCost * 1000)); stats.addToTieredStat("normalizedInitialCostTimesOneThousand", tier, (long) (normalizedInitialCost * 1000));
log.info( log.info(
"[%s]: Initial Total Cost: [%f], Normalization: [%f], Initial Normalized Cost: [%f]", "[%s]: Initial Total Cost: [%f], Normalization: [%f], Initial Normalized Cost: [%f]",
tier, tier,
initialTotalCost, initialTotalCost,
normalization, normalization,
normalizedInitialCost normalizedInitialCost
); );
} }

9
server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java
View File

@ -115,7 +115,7 @@ public class DruidMasterBalancer implements DruidMasterHelper
for (int iter = 0; iter < maxSegmentsToMove; iter++) { for (int iter = 0; iter < maxSegmentsToMove; iter++) {
final BalancerSegmentHolder segmentToMove = strategy.pickSegmentToMove(serverHolderList); final BalancerSegmentHolder segmentToMove = strategy.pickSegmentToMove(serverHolderList);
if (segmentToMove!=null && params.getAvailableSegments().contains(segmentToMove.getSegment())) { if (segmentToMove != null && params.getAvailableSegments().contains(segmentToMove.getSegment())) {
final ServerHolder holder = strategy.findNewSegmentHome(segmentToMove.getSegment(), serverHolderList); final ServerHolder holder = strategy.findNewSegmentHome(segmentToMove.getSegment(), serverHolderList);
if (holder != null) { if (holder != null) {
@ -124,16 +124,15 @@ public class DruidMasterBalancer implements DruidMasterHelper
} }
} }
stats.addToTieredStat("movedCount", tier, currentlyMovingSegments.get(tier).size()); stats.addToTieredStat("movedCount", tier, currentlyMovingSegments.get(tier).size());
if (params.getEmitStats()) if (params.getEmitStats()) {
{
strategy.emitStats(tier, stats, serverHolderList); strategy.emitStats(tier, stats, serverHolderList);
} }
log.info( log.info(
"[%s]: Segments Moved: [%d]",tier,currentlyMovingSegments.get(tier).size() "[%s]: Segments Moved: [%d]", tier, currentlyMovingSegments.get(tier).size()
); );
} }
return params.buildFromExisting() return params.buildFromExisting()
.withMasterStats(stats) .withMasterStats(stats)

21
server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java
View File

@ -90,8 +90,8 @@ public class DruidMasterRuntimeParams
this.mergeSegmentsLimit = mergeSegmentsLimit; this.mergeSegmentsLimit = mergeSegmentsLimit;
this.maxSegmentsToMove = maxSegmentsToMove; this.maxSegmentsToMove = maxSegmentsToMove;
this.balancerReferenceTimestamp = balancerReferenceTimestamp; this.balancerReferenceTimestamp = balancerReferenceTimestamp;
this.emitStats =emitBalancingCostParams; this.emitStats = emitBalancingCostParams;
this.strategyFactory=strategyFactory; this.strategyFactory = strategyFactory;
} }
public boolean getEmitStats() public boolean getEmitStats()
@ -248,8 +248,8 @@ public class DruidMasterRuntimeParams
this.mergeSegmentsLimit = 0; this.mergeSegmentsLimit = 0;
this.maxSegmentsToMove = 0; this.maxSegmentsToMove = 0;
this.balancerReferenceTimestamp = null; this.balancerReferenceTimestamp = null;
this.emitBalancingCostParams=false; this.emitBalancingCostParams = false;
this.strategyFactory=new CostBalancerStrategyFactory(); this.strategyFactory = new CostBalancerStrategyFactory();
} }
Builder( Builder(
@ -286,7 +286,7 @@ public class DruidMasterRuntimeParams
this.mergeSegmentsLimit = mergeSegmentsLimit; this.mergeSegmentsLimit = mergeSegmentsLimit;
this.maxSegmentsToMove = maxSegmentsToMove; this.maxSegmentsToMove = maxSegmentsToMove;
this.balancerReferenceTimestamp = balancerReferenceTimestamp; this.balancerReferenceTimestamp = balancerReferenceTimestamp;
this.emitBalancingCostParams=emitBalancingCostParams; this.emitBalancingCostParams = emitBalancingCostParams;
} }
public DruidMasterRuntimeParams build() public DruidMasterRuntimeParams build()
@ -312,15 +312,18 @@ public class DruidMasterRuntimeParams
); );
} }
public Builder withBalancerStrategy(BalancerStrategyFactory strategyFactory){ public Builder withBalancerStrategy(BalancerStrategyFactory strategyFactory)
this.strategyFactory=strategyFactory; {
this.strategyFactory = strategyFactory;
return this; return this;
} }
public Builder withEmitBalancingCostParams(boolean param){ public Builder withEmitBalancingCostParams(boolean param)
emitBalancingCostParams=param; {
emitBalancingCostParams = param;
return this; return this;
} }
public Builder withStartTime(long time) public Builder withStartTime(long time)
{ {
startTime = time; startTime = time;

6
server/src/main/java/com/metamx/druid/master/ReservoirSegmentSampler.java
View File

@ -45,11 +45,9 @@ public class ReservoirSegmentSampler
numSoFar++; numSoFar++;
} }
} }
if (fromServerHolder!=null) if (fromServerHolder != null) {
{
return new BalancerSegmentHolder(fromServerHolder.getServer(), proposalSegment); return new BalancerSegmentHolder(fromServerHolder.getServer(), proposalSegment);
} } else {
else{
return null; return null;
} }
} }

24
server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java
View File

@ -184,7 +184,14 @@ public class DruidMasterBalancerTest
) )
) )
) )
.withLoadManagementPeons(ImmutableMap.<String, LoadQueuePeon>of("from", fromPeon, "to", toPeon)) .withLoadManagementPeons(
ImmutableMap.<String, LoadQueuePeon>of(
"from",
fromPeon,
"to",
toPeon
)
)
.withAvailableSegments(segments.values()) .withAvailableSegments(segments.values())
.withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE) .withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE)
.withBalancerReferenceTimestamp(new DateTime("2013-01-01")) .withBalancerReferenceTimestamp(new DateTime("2013-01-01"))
@ -195,7 +202,7 @@ public class DruidMasterBalancerTest
} }
@Test @Test
public void testRun2() public void testRun2()
{ {
// Mock some servers of different usages // Mock some servers of different usages
@ -263,7 +270,18 @@ public class DruidMasterBalancerTest
) )
) )
) )
.withLoadManagementPeons(ImmutableMap.<String, LoadQueuePeon>of("1", peon1, "2", peon2, "3", peon3, "4", peon4)) .withLoadManagementPeons(
ImmutableMap.<String, LoadQueuePeon>of(
"1",
peon1,
"2",
peon2,
"3",
peon3,
"4",
peon4
)
)
.withAvailableSegments(segments.values()) .withAvailableSegments(segments.values())
.withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE) .withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE)
.withBalancerReferenceTimestamp(new DateTime("2013-01-01")) .withBalancerReferenceTimestamp(new DateTime("2013-01-01"))