Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

formatted files

This commit is contained in:
Dhruv Parthasarathy 2013-07-29 18:10:08 -07:00
parent ac0f0afdbf
commit 4e4d582e07
6 changed files with 57 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/src/main/java/com/metamx/druid/master/BalancerStrategy.java
View File

@ -25,7 +25,9 @@ import java.util.List;
public interface BalancerStrategy
{
public ServerHolder findNewSegmentHome(final DataSegment proposalSegment,final List<ServerHolder> serverHolders);
public ServerHolder findNewSegmentHome(final DataSegment proposalSegment, final List<ServerHolder> serverHolders);
public BalancerSegmentHolder pickSegmentToMove(final List<ServerHolder> serverHolders);
public void emitStats(String tier, MasterStats stats, List<ServerHolder> serverHolderList);
}

29
server/src/main/java/com/metamx/druid/master/CostBalancerStrategy.java
View File

@ -39,7 +39,7 @@ public class CostBalancerStrategy implements BalancerStrategy
public CostBalancerStrategy(DateTime referenceTimestamp)
{
this.referenceTimestamp=referenceTimestamp;
this.referenceTimestamp = referenceTimestamp;
}
@Override
@ -208,25 +208,26 @@ public class CostBalancerStrategy implements BalancerStrategy
}
@Override
public void emitStats( String tier,
public void emitStats(
String tier,
MasterStats stats, List<ServerHolder> serverHolderList
)
{
final double initialTotalCost = calculateInitialTotalCost(serverHolderList);
final double normalization = calculateNormalization(serverHolderList);
final double normalizedInitialCost = initialTotalCost / normalization;
final double initialTotalCost = calculateInitialTotalCost(serverHolderList);
final double normalization = calculateNormalization(serverHolderList);
final double normalizedInitialCost = initialTotalCost / normalization;
stats.addToTieredStat("initialCost", tier, (long) initialTotalCost);
stats.addToTieredStat("normalization", tier, (long) normalization);
stats.addToTieredStat("normalizedInitialCostTimesOneThousand", tier, (long) (normalizedInitialCost * 1000));
stats.addToTieredStat("initialCost", tier, (long) initialTotalCost);
stats.addToTieredStat("normalization", tier, (long) normalization);
stats.addToTieredStat("normalizedInitialCostTimesOneThousand", tier, (long) (normalizedInitialCost * 1000));
log.info(
"[%s]: Initial Total Cost: [%f], Normalization: [%f], Initial Normalized Cost: [%f]",
tier,
initialTotalCost,
normalization,
normalizedInitialCost
);
"[%s]: Initial Total Cost: [%f], Normalization: [%f], Initial Normalized Cost: [%f]",
tier,
initialTotalCost,
normalization,
normalizedInitialCost
);
}

9
server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java
View File

@ -115,7 +115,7 @@ public class DruidMasterBalancer implements DruidMasterHelper
for (int iter = 0; iter < maxSegmentsToMove; iter++) {
final BalancerSegmentHolder segmentToMove = strategy.pickSegmentToMove(serverHolderList);
if (segmentToMove!=null && params.getAvailableSegments().contains(segmentToMove.getSegment())) {
if (segmentToMove != null && params.getAvailableSegments().contains(segmentToMove.getSegment())) {
final ServerHolder holder = strategy.findNewSegmentHome(segmentToMove.getSegment(), serverHolderList);
if (holder != null) {
@ -124,16 +124,15 @@ public class DruidMasterBalancer implements DruidMasterHelper
}
}
stats.addToTieredStat("movedCount", tier, currentlyMovingSegments.get(tier).size());
if (params.getEmitStats())
{
if (params.getEmitStats()) {
strategy.emitStats(tier, stats, serverHolderList);
}
log.info(
"[%s]: Segments Moved: [%d]",tier,currentlyMovingSegments.get(tier).size()
"[%s]: Segments Moved: [%d]", tier, currentlyMovingSegments.get(tier).size()
);
}
}
return params.buildFromExisting()
.withMasterStats(stats)

21
server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java
View File

@ -90,8 +90,8 @@ public class DruidMasterRuntimeParams
this.mergeSegmentsLimit = mergeSegmentsLimit;
this.maxSegmentsToMove = maxSegmentsToMove;
this.balancerReferenceTimestamp = balancerReferenceTimestamp;
this.emitStats =emitBalancingCostParams;
this.strategyFactory=strategyFactory;
this.emitStats = emitBalancingCostParams;
this.strategyFactory = strategyFactory;
}
public boolean getEmitStats()
@ -248,8 +248,8 @@ public class DruidMasterRuntimeParams
this.mergeSegmentsLimit = 0;
this.maxSegmentsToMove = 0;
this.balancerReferenceTimestamp = null;
this.emitBalancingCostParams=false;
this.strategyFactory=new CostBalancerStrategyFactory();
this.emitBalancingCostParams = false;
this.strategyFactory = new CostBalancerStrategyFactory();
}
Builder(
@ -286,7 +286,7 @@ public class DruidMasterRuntimeParams
this.mergeSegmentsLimit = mergeSegmentsLimit;
this.maxSegmentsToMove = maxSegmentsToMove;
this.balancerReferenceTimestamp = balancerReferenceTimestamp;
this.emitBalancingCostParams=emitBalancingCostParams;
this.emitBalancingCostParams = emitBalancingCostParams;
}
public DruidMasterRuntimeParams build()
@ -312,15 +312,18 @@ public class DruidMasterRuntimeParams
);
}
public Builder withBalancerStrategy(BalancerStrategyFactory strategyFactory){
this.strategyFactory=strategyFactory;
public Builder withBalancerStrategy(BalancerStrategyFactory strategyFactory)
{
this.strategyFactory = strategyFactory;
return this;
}
public Builder withEmitBalancingCostParams(boolean param){
emitBalancingCostParams=param;
public Builder withEmitBalancingCostParams(boolean param)
{
emitBalancingCostParams = param;
return this;
}
public Builder withStartTime(long time)
{
startTime = time;

6
server/src/main/java/com/metamx/druid/master/ReservoirSegmentSampler.java
View File

@ -45,11 +45,9 @@ public class ReservoirSegmentSampler
numSoFar++;
}
}
if (fromServerHolder!=null)
{
if (fromServerHolder != null) {
return new BalancerSegmentHolder(fromServerHolder.getServer(), proposalSegment);
}
else{
} else {
return null;
}
}

24
server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java
View File

@ -184,7 +184,14 @@ public class DruidMasterBalancerTest
)
)
)
.withLoadManagementPeons(ImmutableMap.<String, LoadQueuePeon>of("from", fromPeon, "to", toPeon))
.withLoadManagementPeons(
ImmutableMap.<String, LoadQueuePeon>of(
"from",
fromPeon,
"to",
toPeon
)
)
.withAvailableSegments(segments.values())
.withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE)
.withBalancerReferenceTimestamp(new DateTime("2013-01-01"))
@ -195,7 +202,7 @@ public class DruidMasterBalancerTest
}
@Test
@Test
public void testRun2()
{
// Mock some servers of different usages
@ -263,7 +270,18 @@ public class DruidMasterBalancerTest
)
)
)
.withLoadManagementPeons(ImmutableMap.<String, LoadQueuePeon>of("1", peon1, "2", peon2, "3", peon3, "4", peon4))
.withLoadManagementPeons(
ImmutableMap.<String, LoadQueuePeon>of(
"1",
peon1,
"2",
peon2,
"3",
peon3,
"4",
peon4
)
)
.withAvailableSegments(segments.values())
.withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE)
.withBalancerReferenceTimestamp(new DateTime("2013-01-01"))