diff --git a/server/src/main/java/com/metamx/druid/http/InfoResource.java b/server/src/main/java/com/metamx/druid/http/InfoResource.java index d858c72467a..8fd17f315de 100644 --- a/server/src/main/java/com/metamx/druid/http/InfoResource.java +++ b/server/src/main/java/com/metamx/druid/http/InfoResource.java @@ -102,6 +102,7 @@ public class InfoResource private final DatabaseRuleManager databaseRuleManager; private final IndexingServiceClient indexingServiceClient; + @Inject public InfoResource( DruidMaster master, @@ -129,6 +130,7 @@ public class InfoResource .build(); } + @GET @Path("/cluster") @Produces("application/json") diff --git a/server/src/main/java/com/metamx/druid/http/MasterMain.java b/server/src/main/java/com/metamx/druid/http/MasterMain.java index a1db1bf0888..53382fe81bb 100644 --- a/server/src/main/java/com/metamx/druid/http/MasterMain.java +++ b/server/src/main/java/com/metamx/druid/http/MasterMain.java @@ -283,7 +283,8 @@ public class MasterMain databaseRuleManager, master, jsonMapper, - indexingServiceClient + indexingServiceClient, + configManager ) ); diff --git a/server/src/main/java/com/metamx/druid/http/MasterResource.java b/server/src/main/java/com/metamx/druid/http/MasterResource.java index 9bb59d79d43..7bbf045d732 100644 --- a/server/src/main/java/com/metamx/druid/http/MasterResource.java +++ b/server/src/main/java/com/metamx/druid/http/MasterResource.java @@ -24,13 +24,10 @@ import com.metamx.druid.master.LoadPeonCallback; import javax.inject.Inject; import javax.ws.rs.Consumes; -import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; -import javax.ws.rs.Produces; import javax.ws.rs.core.Response; import java.util.List; -import java.util.Map; /** */ @@ -110,11 +107,5 @@ public class MasterResource return resp; } - @GET - @Path("/loadstatus") - @Produces("application/json") - public Map getLoadStatus() - { - return master.getLoadStatus(); - } + } \ No newline at end of file diff --git a/server/src/main/java/com/metamx/druid/http/MasterSegmentSettingsResource.java b/server/src/main/java/com/metamx/druid/http/MasterSegmentSettingsResource.java new file mode 100644 index 00000000000..ea14c587f78 --- /dev/null +++ b/server/src/main/java/com/metamx/druid/http/MasterSegmentSettingsResource.java @@ -0,0 +1,69 @@ +/* +* Druid - a distributed column store. +* Copyright (C) 2012 Metamarkets Group Inc. +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ +package com.metamx.druid.http; + +import com.metamx.druid.config.JacksonConfigManager; +import com.metamx.druid.master.MasterSegmentSettings; + +import javax.inject.Inject; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; + +/** + */ +@Path("/master/config") +public class MasterSegmentSettingsResource +{ + private final JacksonConfigManager manager; + + @Inject + public MasterSegmentSettingsResource( + JacksonConfigManager manager + ) + { + this.manager=manager; + } + @GET + @Produces("application/json") + public Response getDynamicConfigs() + { + Response.ResponseBuilder builder = Response.status(Response.Status.OK) + .entity( + manager.watch(MasterSegmentSettings.CONFIG_KEY,MasterSegmentSettings.class).get() + ); + return builder.build(); + } + + @POST + @Consumes("application/json") + public Response setDynamicConfigs( + final MasterSegmentSettings masterSegmentSettings + ) + { + if (!manager.set(MasterSegmentSettings.CONFIG_KEY, masterSegmentSettings)) { + return Response.status(Response.Status.BAD_REQUEST).build(); + } + return Response.status(Response.Status.OK).build(); + } + +} diff --git a/server/src/main/java/com/metamx/druid/http/MasterServletModule.java b/server/src/main/java/com/metamx/druid/http/MasterServletModule.java index 26d9ad15459..51d57492914 100644 --- a/server/src/main/java/com/metamx/druid/http/MasterServletModule.java +++ b/server/src/main/java/com/metamx/druid/http/MasterServletModule.java @@ -25,6 +25,7 @@ import com.google.inject.Provides; import com.google.inject.util.Providers; import com.metamx.druid.client.InventoryView; import com.metamx.druid.client.indexing.IndexingServiceClient; +import com.metamx.druid.config.JacksonConfigManager; import com.metamx.druid.db.DatabaseRuleManager; import com.metamx.druid.db.DatabaseSegmentManager; import com.metamx.druid.master.DruidMaster; @@ -43,6 +44,8 @@ public class MasterServletModule extends JerseyServletModule private final DruidMaster master; private final ObjectMapper jsonMapper; private final IndexingServiceClient indexingServiceClient; + private final JacksonConfigManager configManager; + public MasterServletModule( InventoryView serverInventoryView, @@ -50,7 +53,8 @@ public class MasterServletModule extends JerseyServletModule DatabaseRuleManager databaseRuleManager, DruidMaster master, ObjectMapper jsonMapper, - IndexingServiceClient indexingServiceClient + IndexingServiceClient indexingServiceClient, + JacksonConfigManager configManager ) { this.serverInventoryView = serverInventoryView; @@ -59,17 +63,20 @@ public class MasterServletModule extends JerseyServletModule this.master = master; this.jsonMapper = jsonMapper; this.indexingServiceClient = indexingServiceClient; + this.configManager = configManager; } @Override protected void configureServlets() { + bind(MasterSegmentSettingsResource.class); bind(InfoResource.class); bind(MasterResource.class); bind(InventoryView.class).toInstance(serverInventoryView); bind(DatabaseSegmentManager.class).toInstance(segmentInventoryManager); bind(DatabaseRuleManager.class).toInstance(databaseRuleManager); bind(DruidMaster.class).toInstance(master); + bind(JacksonConfigManager.class).toInstance(configManager); if (indexingServiceClient == null) { bind(IndexingServiceClient.class).toProvider(Providers.of(null)); } diff --git a/server/src/main/java/com/metamx/druid/master/DruidMaster.java b/server/src/main/java/com/metamx/druid/master/DruidMaster.java index 6ca5076b584..1303dbd7648 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMaster.java +++ b/server/src/main/java/com/metamx/druid/master/DruidMaster.java @@ -94,6 +94,7 @@ public class DruidMaster private final LoadQueueTaskMaster taskMaster; private final Map loadManagementPeons; private final AtomicReference leaderLatch; + private volatile AtomicReference segmentSettingsAtomicReference; public DruidMaster( DruidMasterConfig config, @@ -155,6 +156,7 @@ public class DruidMaster this.exec = scheduledExecutorFactory.create(1, "Master-Exec--%d"); this.leaderLatch = new AtomicReference(null); + this.segmentSettingsAtomicReference= new AtomicReference(null); this.loadManagementPeons = loadQueuePeonMap; } @@ -464,7 +466,7 @@ public class DruidMaster serverInventoryView.start(); final List> masterRunnables = Lists.newArrayList(); - + segmentSettingsAtomicReference = configManager.watch(MasterSegmentSettings.CONFIG_KEY, MasterSegmentSettings.class,new MasterSegmentSettings.Builder().build()); masterRunnables.add(Pair.of(new MasterComputeManagerRunnable(), config.getMasterPeriod())); if (indexingServiceClient != null) { @@ -649,19 +651,15 @@ public class DruidMaster } // Do master stuff. - DruidMasterRuntimeParams params = DruidMasterRuntimeParams.newBuilder() .withStartTime(startTime) .withDatasources(databaseSegmentManager.getInventory()) - .withMillisToWaitBeforeDeleting(config.getMillisToWaitBeforeDeleting()) + .withMasterSegmentSettings(segmentSettingsAtomicReference.get()) .withEmitter(emitter) - .withMergeBytesLimit(config.getMergeBytesLimit()) - .withMergeSegmentsLimit(config.getMergeSegmentsLimit()) - .withMaxSegmentsToMove(config.getMaxSegmentsToMove()) - .withEmitBalancingCostParams(config.getEmitStats()) .build(); + for (DruidMasterHelper helper : helpers) { params = helper.run(params); } @@ -756,6 +754,9 @@ public class DruidMaster .withLoadManagementPeons(loadManagementPeons) .withSegmentReplicantLookup(segmentReplicantLookup) .withBalancerReferenceTimestamp(DateTime.now()) + .withMasterSegmentSettings( + segmentSettingsAtomicReference.get() + ) .build(); } }, diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java b/server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java index 2745d079410..889383a8054 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java +++ b/server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java @@ -79,7 +79,7 @@ public class DruidMasterBalancer implements DruidMasterHelper final MasterStats stats = new MasterStats(); final DateTime referenceTimestamp = params.getBalancerReferenceTimestamp(); final BalancerStrategy strategy = params.getBalancerStrategyFactory().getBalancerStrategy(referenceTimestamp); - final int maxSegmentsToMove = params.getMaxSegmentsToMove(); + final int maxSegmentsToMove = params.getMasterSegmentSettings().getMaxSegmentsToMove(); for (Map.Entry> entry : params.getDruidCluster().getCluster().entrySet()) { @@ -124,7 +124,7 @@ public class DruidMasterBalancer implements DruidMasterHelper } } stats.addToTieredStat("movedCount", tier, currentlyMovingSegments.get(tier).size()); - if (params.getEmitStats()) { + if (params.getMasterSegmentSettings().isEmitBalancingStats()) { strategy.emitStats(tier, stats, serverHolderList); } diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java b/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java index e6605e096b0..95b5c8db666 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java +++ b/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java @@ -34,11 +34,6 @@ public abstract class DruidMasterConfig @Default("PT600s") public abstract Duration getMasterStartDelay(); - @Config("druid.master.emitStats") - public boolean getEmitStats(){ - return false; - } - @Config("druid.master.period") @Default("PT60s") public abstract Duration getMasterPeriod(); @@ -47,12 +42,6 @@ public abstract class DruidMasterConfig @Default("PT1800s") public abstract Duration getMasterSegmentMergerPeriod(); - @Config("druid.master.millisToWaitBeforeDeleting") - public long getMillisToWaitBeforeDeleting() - { - return 15 * 60 * 1000L; - } - @Config("druid.master.merger.on") public boolean isMergeSegments() { @@ -71,26 +60,16 @@ public abstract class DruidMasterConfig return null; } - @Config("druid.master.merge.threshold") - public long getMergeBytesLimit() - { - return 100000000L; - } - - @Config("druid.master.merge.maxSegments") - public int getMergeSegmentsLimit() - { - return Integer.MAX_VALUE; - } - - @Config("druid.master.balancer.maxSegmentsToMove") - @Default("5") - public abstract int getMaxSegmentsToMove(); - @Config("druid.master.replicant.lifetime") @Default("15") public abstract int getReplicantLifetime(); + @Config("druid.master.masterSegmentSettings") + public MasterSegmentSettings getMasterSegmentSettings() + { + return new MasterSegmentSettings.Builder().build(); + } + @Config("druid.master.replicant.throttleLimit") @Default("10") public abstract int getReplicantThrottleLimit(); diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java b/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java index 7e6229374e9..4ad153cefe7 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java +++ b/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java @@ -46,13 +46,9 @@ public class DruidMasterRuntimeParams private final Map loadManagementPeons; private final ReplicationThrottler replicationManager; private final ServiceEmitter emitter; - private final long millisToWaitBeforeDeleting; + private final MasterSegmentSettings masterSegmentSettings; private final MasterStats stats; - private final long mergeBytesLimit; - private final int mergeSegmentsLimit; - private final int maxSegmentsToMove; private final DateTime balancerReferenceTimestamp; - private final boolean emitStats; private final BalancerStrategyFactory strategyFactory; public DruidMasterRuntimeParams( @@ -65,13 +61,9 @@ public class DruidMasterRuntimeParams Map loadManagementPeons, ReplicationThrottler replicationManager, ServiceEmitter emitter, - long millisToWaitBeforeDeleting, + MasterSegmentSettings masterSegmentSettings, MasterStats stats, - long mergeBytesLimit, - int mergeSegmentsLimit, - int maxSegmentsToMove, DateTime balancerReferenceTimestamp, - boolean emitBalancingCostParams, BalancerStrategyFactory strategyFactory ) { @@ -84,21 +76,12 @@ public class DruidMasterRuntimeParams this.loadManagementPeons = loadManagementPeons; this.replicationManager = replicationManager; this.emitter = emitter; - this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting; + this.masterSegmentSettings = masterSegmentSettings; this.stats = stats; - this.mergeBytesLimit = mergeBytesLimit; - this.mergeSegmentsLimit = mergeSegmentsLimit; - this.maxSegmentsToMove = maxSegmentsToMove; this.balancerReferenceTimestamp = balancerReferenceTimestamp; - this.emitStats = emitBalancingCostParams; this.strategyFactory = strategyFactory; } - public boolean getEmitStats() - { - return emitStats; - } - public long getStartTime() { return startTime; @@ -144,9 +127,9 @@ public class DruidMasterRuntimeParams return emitter; } - public long getMillisToWaitBeforeDeleting() + public MasterSegmentSettings getMasterSegmentSettings() { - return millisToWaitBeforeDeleting; + return masterSegmentSettings; } public MasterStats getMasterStats() @@ -154,21 +137,6 @@ public class DruidMasterRuntimeParams return stats; } - public long getMergeBytesLimit() - { - return mergeBytesLimit; - } - - public int getMergeSegmentsLimit() - { - return mergeSegmentsLimit; - } - - public int getMaxSegmentsToMove() - { - return maxSegmentsToMove; - } - public DateTime getBalancerReferenceTimestamp() { return balancerReferenceTimestamp; @@ -181,7 +149,7 @@ public class DruidMasterRuntimeParams public boolean hasDeletionWaitTimeElapsed() { - return (System.currentTimeMillis() - getStartTime() > getMillisToWaitBeforeDeleting()); + return (System.currentTimeMillis() - getStartTime() > masterSegmentSettings.getMillisToWaitBeforeDeleting()); } public static Builder newBuilder() @@ -201,13 +169,9 @@ public class DruidMasterRuntimeParams loadManagementPeons, replicationManager, emitter, - millisToWaitBeforeDeleting, + masterSegmentSettings, stats, - mergeBytesLimit, - mergeSegmentsLimit, - maxSegmentsToMove, balancerReferenceTimestamp, - emitStats, strategyFactory ); } @@ -223,11 +187,8 @@ public class DruidMasterRuntimeParams private final Map loadManagementPeons; private ReplicationThrottler replicationManager; private ServiceEmitter emitter; - private long millisToWaitBeforeDeleting; + private MasterSegmentSettings masterSegmentSettings; private MasterStats stats; - private long mergeBytesLimit; - private int mergeSegmentsLimit; - private int maxSegmentsToMove; private DateTime balancerReferenceTimestamp; private boolean emitBalancingCostParams; private BalancerStrategyFactory strategyFactory; @@ -243,11 +204,8 @@ public class DruidMasterRuntimeParams this.loadManagementPeons = Maps.newHashMap(); this.replicationManager = null; this.emitter = null; - this.millisToWaitBeforeDeleting = 0; this.stats = new MasterStats(); - this.mergeBytesLimit = 0; - this.mergeSegmentsLimit = 0; - this.maxSegmentsToMove = 0; + this.masterSegmentSettings = new MasterSegmentSettings.Builder().build(); this.balancerReferenceTimestamp = null; this.emitBalancingCostParams = false; this.strategyFactory = new CostBalancerStrategyFactory(); @@ -263,13 +221,9 @@ public class DruidMasterRuntimeParams Map loadManagementPeons, ReplicationThrottler replicationManager, ServiceEmitter emitter, - long millisToWaitBeforeDeleting, + MasterSegmentSettings masterSegmentSettings, MasterStats stats, - long mergeBytesLimit, - int mergeSegmentsLimit, - int maxSegmentsToMove, DateTime balancerReferenceTimestamp, - boolean emitBalancingCostParams, BalancerStrategyFactory strategyFactory ) { @@ -282,11 +236,8 @@ public class DruidMasterRuntimeParams this.loadManagementPeons = loadManagementPeons; this.replicationManager = replicationManager; this.emitter = emitter; - this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting; + this.masterSegmentSettings = masterSegmentSettings; this.stats = stats; - this.mergeBytesLimit = mergeBytesLimit; - this.mergeSegmentsLimit = mergeSegmentsLimit; - this.maxSegmentsToMove = maxSegmentsToMove; this.balancerReferenceTimestamp = balancerReferenceTimestamp; this.emitBalancingCostParams = emitBalancingCostParams; this.strategyFactory=strategyFactory; @@ -304,13 +255,9 @@ public class DruidMasterRuntimeParams loadManagementPeons, replicationManager, emitter, - millisToWaitBeforeDeleting, + masterSegmentSettings, stats, - mergeBytesLimit, - mergeSegmentsLimit, - maxSegmentsToMove, balancerReferenceTimestamp, - emitBalancingCostParams, strategyFactory ); } @@ -381,33 +328,15 @@ public class DruidMasterRuntimeParams return this; } - public Builder withMillisToWaitBeforeDeleting(long millisToWaitBeforeDeleting) - { - this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting; - return this; - } - public Builder withMasterStats(MasterStats stats) { this.stats.accumulate(stats); return this; } - public Builder withMergeBytesLimit(long mergeBytesLimit) + public Builder withMasterSegmentSettings(MasterSegmentSettings configs) { - this.mergeBytesLimit = mergeBytesLimit; - return this; - } - - public Builder withMergeSegmentsLimit(int mergeSegmentsLimit) - { - this.mergeSegmentsLimit = mergeSegmentsLimit; - return this; - } - - public Builder withMaxSegmentsToMove(int maxSegmentsToMove) - { - this.maxSegmentsToMove = maxSegmentsToMove; + this.masterSegmentSettings = configs; return this; } diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentMerger.java b/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentMerger.java index e46c2a06f51..d178cf1a8ad 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentMerger.java +++ b/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentMerger.java @@ -21,7 +21,6 @@ package com.metamx.druid.master; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; import com.google.common.collect.HashMultiset; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; @@ -100,9 +99,9 @@ public class DruidMasterSegmentMerger implements DruidMasterHelper for (int i = 0; i < timelineObjects.size(); i++) { if (!segmentsToMerge.add(timelineObjects.get(i)) - || segmentsToMerge.getByteCount() > params.getMergeBytesLimit() - || segmentsToMerge.getSegmentCount() >= params.getMergeSegmentsLimit()) { - i -= segmentsToMerge.backtrack(params.getMergeBytesLimit()); + || segmentsToMerge.getByteCount() > params.getMasterSegmentSettings().getMergeBytesLimit() + || segmentsToMerge.getSegmentCount() >= params.getMasterSegmentSettings().getMergeSegmentsLimit()) { + i -= segmentsToMerge.backtrack(params.getMasterSegmentSettings().getMergeBytesLimit()); if (segmentsToMerge.getSegmentCount() > 1) { stats.addToGlobalStat("mergedCount", mergeSegments(segmentsToMerge, entry.getKey())); @@ -118,7 +117,7 @@ public class DruidMasterSegmentMerger implements DruidMasterHelper } // Finish any timelineObjects to merge that may have not hit threshold - segmentsToMerge.backtrack(params.getMergeBytesLimit()); + segmentsToMerge.backtrack(params.getMasterSegmentSettings().getMergeBytesLimit()); if (segmentsToMerge.getSegmentCount() > 1) { stats.addToGlobalStat("mergedCount", mergeSegments(segmentsToMerge, entry.getKey())); } diff --git a/server/src/main/java/com/metamx/druid/master/MasterSegmentSettings.java b/server/src/main/java/com/metamx/druid/master/MasterSegmentSettings.java new file mode 100644 index 00000000000..75a56fa9cb7 --- /dev/null +++ b/server/src/main/java/com/metamx/druid/master/MasterSegmentSettings.java @@ -0,0 +1,140 @@ +/* +* Druid - a distributed column store. +* Copyright (C) 2012 Metamarkets Group Inc. +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ +package com.metamx.druid.master; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class MasterSegmentSettings +{ + public static final String CONFIG_KEY = "master.dynamicConfigs"; + private long millisToWaitBeforeDeleting=15 * 60 * 1000L; + private long mergeBytesLimit= 100000000L; + private int mergeSegmentsLimit = Integer.MAX_VALUE; + private int maxSegmentsToMove = 5; + private boolean emitBalancingStats = false; + + @JsonCreator + public MasterSegmentSettings( + @JsonProperty("millisToWaitBeforeDeleting") Long millisToWaitBeforeDeleting, + @JsonProperty("mergeBytesLimit") Long mergeBytesLimit, + @JsonProperty("mergeSegmentsLimit") Integer mergeSegmentsLimit, + @JsonProperty("maxSegmentsToMove") Integer maxSegmentsToMove, + @JsonProperty("emitBalancingStats") Boolean emitBalancingStats + ) + { + this.maxSegmentsToMove=maxSegmentsToMove; + this.millisToWaitBeforeDeleting=millisToWaitBeforeDeleting; + this.mergeSegmentsLimit=mergeSegmentsLimit; + this.mergeBytesLimit=mergeBytesLimit; + this.emitBalancingStats = emitBalancingStats; + } + + public static String getConfigKey() + { + return CONFIG_KEY; + } + + @JsonProperty + public long getMillisToWaitBeforeDeleting() + { + return millisToWaitBeforeDeleting; + } + + @JsonProperty + public long getMergeBytesLimit() + { + return mergeBytesLimit; + } + + public boolean isEmitBalancingStats() + { + return emitBalancingStats; + } + + @JsonProperty + public int getMergeSegmentsLimit() + { + return mergeSegmentsLimit; + } + + @JsonProperty + public int getMaxSegmentsToMove() + { + return maxSegmentsToMove; + } + + + public static class Builder + { + public static final String CONFIG_KEY = "master.dynamicConfigs"; + private long millisToWaitBeforeDeleting; + private long mergeBytesLimit; + private int mergeSegmentsLimit; + private int maxSegmentsToMove; + private boolean emitBalancingStats; + + public Builder() + { + this.millisToWaitBeforeDeleting=15 * 60 * 1000L; + this.mergeBytesLimit= 100000000L; + this.mergeSegmentsLimit= Integer.MAX_VALUE; + this.maxSegmentsToMove = 5; + this.emitBalancingStats = false; + } + + public Builder(long millisToWaitBeforeDeleting, long mergeBytesLimit, int mergeSegmentsLimit, int maxSegmentsToMove, boolean emitBalancingStats) + { + this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting; + this.mergeBytesLimit = mergeBytesLimit; + this.mergeSegmentsLimit = mergeSegmentsLimit; + this.maxSegmentsToMove = maxSegmentsToMove; + this.emitBalancingStats = emitBalancingStats; + } + + public Builder withMillisToWaitBeforeDeleting(long millisToWaitBeforeDeleting) + { + this.millisToWaitBeforeDeleting=millisToWaitBeforeDeleting; + return this; + } + + public Builder withMergeBytesLimit(long mergeBytesLimit) + { + this.mergeBytesLimit=mergeBytesLimit; + return this; + } + + public Builder withMergeSegmentsLimit(int mergeSegmentsLimit) + { + this.mergeSegmentsLimit=mergeSegmentsLimit; + return this; + } + + public Builder withMaxSegmentsToMove(int maxSegmentsToMove) + { + this.maxSegmentsToMove=maxSegmentsToMove; + return this; + } + + public MasterSegmentSettings build() + { + return new MasterSegmentSettings(millisToWaitBeforeDeleting,mergeBytesLimit,mergeSegmentsLimit,maxSegmentsToMove, emitBalancingStats); + } + } +} diff --git a/server/src/main/resources/static/index.html b/server/src/main/resources/static/index.html index 8a6a68a7faa..35febbea6b8 100644 --- a/server/src/main/resources/static/index.html +++ b/server/src/main/resources/static/index.html @@ -37,6 +37,9 @@ + diff --git a/server/src/main/resources/static/js/.Rhistory b/server/src/main/resources/static/js/.Rhistory new file mode 100644 index 00000000000..e69de29bb2d diff --git a/server/src/main/resources/static/js/masterSegmentSettings.js b/server/src/main/resources/static/js/masterSegmentSettings.js new file mode 100644 index 00000000000..b992cc8cf19 --- /dev/null +++ b/server/src/main/resources/static/js/masterSegmentSettings.js @@ -0,0 +1,25 @@ +$(function () { + $.get('../master/config', function (data) { + document.getElementById("millis").value=data["millisToWaitBeforeDeleting"]; + document.getElementById("mergeBytes").value = data["mergeBytesLimit"]; + document.getElementById("mergeSegments").value = data["mergeSegmentsLimit"]; + document.getElementById("maxSegments").value = data["maxSegmentsToMove"]; + }); + + $("#submit").click( function () + { + values = {}; + list = $('form').serializeArray(); + for (var i=0;i< list.length;i++) + { + values[list[i]["name"]]=list[i]["value"]; + } + $.ajax({ + url:'../master/config', + type:"POST", + data: JSON.stringify(values), + contentType:"application/json; charset=utf-8", + dataType:"json" + }); + }); +}); diff --git a/server/src/main/resources/static/js/tablehelper-0.0.2.js b/server/src/main/resources/static/js/tablehelper-0.0.2.js index 3c060ba09a4..ce8537056fa 100644 --- a/server/src/main/resources/static/js/tablehelper-0.0.2.js +++ b/server/src/main/resources/static/js/tablehelper-0.0.2.js @@ -78,4 +78,4 @@ function initDataTable(el) { this.value = asInitVals[$("thead input").index(this)]; } }); -} \ No newline at end of file + } \ No newline at end of file diff --git a/server/src/main/resources/static/masterSegmentSettings.html b/server/src/main/resources/static/masterSegmentSettings.html new file mode 100644 index 00000000000..c4f06ec25db --- /dev/null +++ b/server/src/main/resources/static/masterSegmentSettings.html @@ -0,0 +1,23 @@ + + + + Configure Dynamic Master Parameters + + +
+ millisToWaitBeforeDeleting: +
+ mergeBytesLimit: +
+ mergeSegmentsLimit: +
+ maxSegmentsToMove: +
+ +
+ + + + + + \ No newline at end of file diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java b/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java index 1984cd8010e..8513c157de3 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java +++ b/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java @@ -192,7 +192,7 @@ public class DruidMasterBalancerTest ) ) .withAvailableSegments(segments.values()) - .withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE) + .withMasterSegmentSettings(new MasterSegmentSettings.Builder().withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE).build()) .withBalancerReferenceTimestamp(new DateTime("2013-01-01")) .build(); @@ -239,7 +239,6 @@ public class DruidMasterBalancerTest LoadQueuePeonTester fromPeon = new LoadQueuePeonTester(); LoadQueuePeonTester toPeon = new LoadQueuePeonTester(); - DruidMasterRuntimeParams params = DruidMasterRuntimeParams.newBuilder() .withDruidCluster( @@ -265,7 +264,10 @@ public class DruidMasterBalancerTest ) ) .withAvailableSegments(segments.values()) - .withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE) + .withMasterSegmentSettings( + new MasterSegmentSettings.Builder().withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE) + .build() + ) .withBalancerReferenceTimestamp(new DateTime("2013-01-01")) .build(); @@ -355,7 +357,11 @@ public class DruidMasterBalancerTest ) ) .withAvailableSegments(segments.values()) - .withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE) + .withMasterSegmentSettings( + new MasterSegmentSettings.Builder().withMaxSegmentsToMove( + MAX_SEGMENTS_TO_MOVE + ).build() + ) .withBalancerReferenceTimestamp(new DateTime("2013-01-01")) .build(); diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java b/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java index 70e6258ec88..27dc46636ac 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java +++ b/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java @@ -177,8 +177,7 @@ public class DruidMasterRuleRunnerTest .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster())) - .withMaxSegmentsToMove(5) - .withBalancerReferenceTimestamp(new DateTime("2013-01-01")) + .withMasterSegmentSettings(new MasterSegmentSettings.Builder().withMaxSegmentsToMove(5).build()) .build(); DruidMasterRuntimeParams afterParams = ruleRunner.run(params); @@ -523,7 +522,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withMasterSegmentSettings(new MasterSegmentSettings.Builder().withMillisToWaitBeforeDeleting(0L).build()) .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) @@ -597,7 +596,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withMasterSegmentSettings(new MasterSegmentSettings.Builder().withMillisToWaitBeforeDeleting(0L).build()) .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) @@ -678,7 +677,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withMasterSegmentSettings(new MasterSegmentSettings.Builder().withMillisToWaitBeforeDeleting(0L).build()) .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) @@ -755,7 +754,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withMasterSegmentSettings(new MasterSegmentSettings.Builder().withMillisToWaitBeforeDeleting(0L).build()) .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) @@ -844,7 +843,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withMasterSegmentSettings(new MasterSegmentSettings.Builder().withMillisToWaitBeforeDeleting(0L).build()) .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) @@ -1030,7 +1029,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withMasterSegmentSettings(new MasterSegmentSettings.Builder().withMillisToWaitBeforeDeleting(0L).build()) .withAvailableSegments(longerAvailableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java b/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java index f8bbdcc63f8..04caa13a8a4 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java +++ b/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java @@ -459,10 +459,15 @@ public class DruidMasterSegmentMergerTest final DruidMasterSegmentMerger merger = new DruidMasterSegmentMerger(indexingServiceClient, whitelistRef); final DruidMasterRuntimeParams params = DruidMasterRuntimeParams.newBuilder() .withAvailableSegments(ImmutableSet.copyOf(segments)) - .withMergeBytesLimit(mergeBytesLimit) - .withMergeSegmentsLimit(mergeSegmentsLimit) + .withMasterSegmentSettings( + new MasterSegmentSettings.Builder().withMergeBytesLimit( + mergeBytesLimit + ).withMergeSegmentsLimit( + mergeSegmentsLimit + ) + .build() + ) .build(); - merger.run(params); return retVal; } diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java b/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java index 213e350f76b..59b727d1355 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java +++ b/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java @@ -36,7 +36,6 @@ import org.junit.Before; import org.junit.Test; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ScheduledExecutorService; /** */ @@ -95,12 +94,6 @@ public class DruidMasterTest return null; } - @Override - public long getMillisToWaitBeforeDeleting() - { - return super.getMillisToWaitBeforeDeleting(); - } - @Override public String getMergerServiceName() { @@ -108,9 +101,9 @@ public class DruidMasterTest } @Override - public int getMaxSegmentsToMove() + public MasterSegmentSettings getMasterSegmentSettings() { - return 0; + return new MasterSegmentSettings.Builder().withMillisToWaitBeforeDeleting(super.getMasterSegmentSettings().getMillisToWaitBeforeDeleting()).withMaxSegmentsToMove(0).build(); } @Override diff --git a/server/src/test/java/com/metamx/druid/utils/DruidMasterBalancerProfiler.java b/server/src/test/java/com/metamx/druid/utils/DruidMasterBalancerProfiler.java index fbaa185e434..482274c0b28 100644 --- a/server/src/test/java/com/metamx/druid/utils/DruidMasterBalancerProfiler.java +++ b/server/src/test/java/com/metamx/druid/utils/DruidMasterBalancerProfiler.java @@ -36,6 +36,7 @@ import com.metamx.druid.master.DruidMasterRuntimeParams; import com.metamx.druid.master.LoadPeonCallback; import com.metamx.druid.master.LoadQueuePeon; import com.metamx.druid.master.LoadQueuePeonTester; +import com.metamx.druid.master.MasterSegmentSettings; import com.metamx.druid.master.ReplicationThrottler; import com.metamx.druid.master.SegmentReplicantLookup; import com.metamx.druid.master.ServerHolder; @@ -161,7 +162,7 @@ public class DruidMasterBalancerProfiler peonMap ) .withAvailableSegments(segmentMap.values()) - .withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE) + .withMasterSegmentSettings(new MasterSegmentSettings.Builder().withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE).build()) .withBalancerReferenceTimestamp(new DateTime("2013-01-01")) .withEmitter(emitter) .withDatabaseRuleManager(manager) @@ -236,7 +237,7 @@ public class DruidMasterBalancerProfiler ) .withLoadManagementPeons(ImmutableMap.of("from", fromPeon, "to", toPeon)) .withAvailableSegments(segments.values()) - .withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE) + .withMasterSegmentSettings(new MasterSegmentSettings.Builder().withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE).build()) .withBalancerReferenceTimestamp(new DateTime("2013-01-01")) .build(); DruidMasterBalancerTester tester = new DruidMasterBalancerTester(master);