From 31ffed466c8f2feac3ab6f41ca31d71a4c306e4b Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Mon, 29 Jul 2013 11:39:24 -0700 Subject: [PATCH] first set of changes --- .../com/metamx/druid/http/InfoResource.java | 29 +++++++ .../com/metamx/druid/master/DruidMaster.java | 14 +-- .../master/DruidMasterRuntimeParams.java | 75 ++++------------ .../metamx/druid/master/DynamicConfigs.java | 85 +++++++++++++++++++ .../druid/master/DruidMasterBalancerTest.java | 30 ++++++- .../master/DruidMasterRuleRunnerTest.java | 63 ++++++++++++-- .../master/DruidMasterSegmentMergerTest.java | 15 +++- .../metamx/druid/master/DruidMasterTest.java | 1 + 8 files changed, 234 insertions(+), 78 deletions(-) create mode 100644 server/src/main/java/com/metamx/druid/master/DynamicConfigs.java diff --git a/server/src/main/java/com/metamx/druid/http/InfoResource.java b/server/src/main/java/com/metamx/druid/http/InfoResource.java index d858c72467a..a03d5d9abf0 100644 --- a/server/src/main/java/com/metamx/druid/http/InfoResource.java +++ b/server/src/main/java/com/metamx/druid/http/InfoResource.java @@ -31,9 +31,11 @@ import com.metamx.druid.client.DruidDataSource; import com.metamx.druid.client.DruidServer; import com.metamx.druid.client.InventoryView; import com.metamx.druid.client.indexing.IndexingServiceClient; +import com.metamx.druid.config.JacksonConfigManager; import com.metamx.druid.db.DatabaseRuleManager; import com.metamx.druid.db.DatabaseSegmentManager; import com.metamx.druid.master.DruidMaster; +import com.metamx.druid.master.DynamicConfigs; import com.metamx.druid.master.rules.Rule; import org.joda.time.Interval; @@ -101,6 +103,8 @@ public class InfoResource private final DatabaseSegmentManager databaseSegmentManager; private final DatabaseRuleManager databaseRuleManager; private final IndexingServiceClient indexingServiceClient; + private final JacksonConfigManager configManager; + @Inject public InfoResource( @@ -108,6 +112,7 @@ public class InfoResource InventoryView serverInventoryView, DatabaseSegmentManager databaseSegmentManager, DatabaseRuleManager databaseRuleManager, + JacksonConfigManager configManager, @Nullable IndexingServiceClient indexingServiceClient ) @@ -116,6 +121,7 @@ public class InfoResource this.serverInventoryView = serverInventoryView; this.databaseSegmentManager = databaseSegmentManager; this.databaseRuleManager = databaseRuleManager; + this.configManager=configManager; this.indexingServiceClient = indexingServiceClient; } @@ -129,6 +135,15 @@ public class InfoResource .build(); } + @GET + @Path("/master/dynamicConfigs") + @Produces("application/json") + public Response getDynamicConfigs() + { + Response.ResponseBuilder builder = Response.status(Response.Status.OK); + return builder.build(); + } + @GET @Path("/cluster") @Produces("application/json") @@ -373,6 +388,20 @@ public class InfoResource return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); } + @POST + @Path("master/setDynamicConfigs") + @Produces("application/json") + public Response setDynamicConfigs( + final DynamicConfigs dynamicConfigs) + { + if (!configManager.set(DynamicConfigs.CONFIG_KEY, dynamicConfigs)) + { + return Response.status(Response.Status.BAD_REQUEST).build(); + } + return Response.status(Response.Status.OK).build(); + } + + @GET @Path("/datasources") @Produces("application/json") diff --git a/server/src/main/java/com/metamx/druid/master/DruidMaster.java b/server/src/main/java/com/metamx/druid/master/DruidMaster.java index 592e76f0d06..253efa13ad3 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMaster.java +++ b/server/src/main/java/com/metamx/druid/master/DruidMaster.java @@ -80,6 +80,7 @@ public class DruidMaster private volatile boolean started = false; private volatile boolean master = false; + private volatile boolean defaultConfigsSet=false; private final DruidMasterConfig config; private final ZkPathsConfig zkPaths; @@ -465,7 +466,11 @@ public class DruidMaster serverInventoryView.start(); final List> masterRunnables = Lists.newArrayList(); - + if (!defaultConfigsSet) + { + configManager.set(DynamicConfigs.CONFIG_KEY, new DynamicConfigs()); + defaultConfigsSet=true; + } masterRunnables.add(Pair.of(new MasterComputeManagerRunnable(), config.getMasterPeriod())); if (indexingServiceClient != null) { @@ -650,18 +655,17 @@ public class DruidMaster } // Do master stuff. + DynamicConfigs dynamicConfigs = configManager.watch(DynamicConfigs.CONFIG_KEY,DynamicConfigs.class).get(); DruidMasterRuntimeParams params = DruidMasterRuntimeParams.newBuilder() .withStartTime(startTime) .withDatasources(databaseSegmentManager.getInventory()) - .withMillisToWaitBeforeDeleting(config.getMillisToWaitBeforeDeleting()) + .withDynamicConfigs(dynamicConfigs) .withEmitter(emitter) - .withMergeBytesLimit(config.getMergeBytesLimit()) - .withMergeSegmentsLimit(config.getMergeSegmentsLimit()) - .withMaxSegmentsToMove(config.getMaxSegmentsToMove()) .build(); + for (DruidMasterHelper helper : helpers) { params = helper.run(params); } diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java b/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java index 85fe6a35d89..7650dcc9b4b 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java +++ b/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java @@ -46,11 +46,8 @@ public class DruidMasterRuntimeParams private final Map loadManagementPeons; private final ReplicationThrottler replicationManager; private final ServiceEmitter emitter; - private final long millisToWaitBeforeDeleting; + private final DynamicConfigs dynamicConfigs; private final MasterStats stats; - private final long mergeBytesLimit; - private final int mergeSegmentsLimit; - private final int maxSegmentsToMove; private final DateTime balancerReferenceTimestamp; public DruidMasterRuntimeParams( @@ -63,11 +60,8 @@ public class DruidMasterRuntimeParams Map loadManagementPeons, ReplicationThrottler replicationManager, ServiceEmitter emitter, - long millisToWaitBeforeDeleting, + DynamicConfigs dynamicConfigs, MasterStats stats, - long mergeBytesLimit, - int mergeSegmentsLimit, - int maxSegmentsToMove, DateTime balancerReferenceTimestamp ) { @@ -80,11 +74,8 @@ public class DruidMasterRuntimeParams this.loadManagementPeons = loadManagementPeons; this.replicationManager = replicationManager; this.emitter = emitter; - this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting; + this.dynamicConfigs = dynamicConfigs; this.stats = stats; - this.mergeBytesLimit = mergeBytesLimit; - this.mergeSegmentsLimit = mergeSegmentsLimit; - this.maxSegmentsToMove = maxSegmentsToMove; this.balancerReferenceTimestamp = balancerReferenceTimestamp; } @@ -135,7 +126,7 @@ public class DruidMasterRuntimeParams public long getMillisToWaitBeforeDeleting() { - return millisToWaitBeforeDeleting; + return dynamicConfigs.getMillisToWaitBeforeDeleting(); } public MasterStats getMasterStats() @@ -145,17 +136,17 @@ public class DruidMasterRuntimeParams public long getMergeBytesLimit() { - return mergeBytesLimit; + return dynamicConfigs.getMergeBytesLimit(); } public int getMergeSegmentsLimit() { - return mergeSegmentsLimit; + return dynamicConfigs.getMergeSegmentsLimit(); } public int getMaxSegmentsToMove() { - return maxSegmentsToMove; + return dynamicConfigs.getMaxSegmentsToMove(); } public DateTime getBalancerReferenceTimestamp() @@ -190,11 +181,8 @@ public class DruidMasterRuntimeParams loadManagementPeons, replicationManager, emitter, - millisToWaitBeforeDeleting, + dynamicConfigs, stats, - mergeBytesLimit, - mergeSegmentsLimit, - maxSegmentsToMove, balancerReferenceTimestamp ); } @@ -210,11 +198,8 @@ public class DruidMasterRuntimeParams private final Map loadManagementPeons; private ReplicationThrottler replicationManager; private ServiceEmitter emitter; - private long millisToWaitBeforeDeleting; + private DynamicConfigs dynamicConfigs; private MasterStats stats; - private long mergeBytesLimit; - private int mergeSegmentsLimit; - private int maxSegmentsToMove; private DateTime balancerReferenceTimestamp; Builder() @@ -228,11 +213,8 @@ public class DruidMasterRuntimeParams this.loadManagementPeons = Maps.newHashMap(); this.replicationManager = null; this.emitter = null; - this.millisToWaitBeforeDeleting = 0; this.stats = new MasterStats(); - this.mergeBytesLimit = 0; - this.mergeSegmentsLimit = 0; - this.maxSegmentsToMove = 0; + this.dynamicConfigs = new DynamicConfigs(); this.balancerReferenceTimestamp = null; } @@ -246,11 +228,8 @@ public class DruidMasterRuntimeParams Map loadManagementPeons, ReplicationThrottler replicationManager, ServiceEmitter emitter, - long millisToWaitBeforeDeleting, + DynamicConfigs dynamicConfigs, MasterStats stats, - long mergeBytesLimit, - int mergeSegmentsLimit, - int maxSegmentsToMove, DateTime balancerReferenceTimestamp ) { @@ -263,11 +242,8 @@ public class DruidMasterRuntimeParams this.loadManagementPeons = loadManagementPeons; this.replicationManager = replicationManager; this.emitter = emitter; - this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting; + this.dynamicConfigs=dynamicConfigs; this.stats = stats; - this.mergeBytesLimit = mergeBytesLimit; - this.mergeSegmentsLimit = mergeSegmentsLimit; - this.maxSegmentsToMove = maxSegmentsToMove; this.balancerReferenceTimestamp = balancerReferenceTimestamp; } @@ -283,11 +259,8 @@ public class DruidMasterRuntimeParams loadManagementPeons, replicationManager, emitter, - millisToWaitBeforeDeleting, + dynamicConfigs, stats, - mergeBytesLimit, - mergeSegmentsLimit, - maxSegmentsToMove, balancerReferenceTimestamp ); } @@ -346,33 +319,15 @@ public class DruidMasterRuntimeParams return this; } - public Builder withMillisToWaitBeforeDeleting(long millisToWaitBeforeDeleting) - { - this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting; - return this; - } - public Builder withMasterStats(MasterStats stats) { this.stats.accumulate(stats); return this; } - public Builder withMergeBytesLimit(long mergeBytesLimit) + public Builder withDynamicConfigs(DynamicConfigs configs) { - this.mergeBytesLimit = mergeBytesLimit; - return this; - } - - public Builder withMergeSegmentsLimit(int mergeSegmentsLimit) - { - this.mergeSegmentsLimit = mergeSegmentsLimit; - return this; - } - - public Builder withMaxSegmentsToMove(int maxSegmentsToMove) - { - this.maxSegmentsToMove = maxSegmentsToMove; + this.dynamicConfigs = configs; return this; } diff --git a/server/src/main/java/com/metamx/druid/master/DynamicConfigs.java b/server/src/main/java/com/metamx/druid/master/DynamicConfigs.java new file mode 100644 index 00000000000..2c0758e3b2d --- /dev/null +++ b/server/src/main/java/com/metamx/druid/master/DynamicConfigs.java @@ -0,0 +1,85 @@ +/* +* Druid - a distributed column store. +* Copyright (C) 2012 Metamarkets Group Inc. +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ +package com.metamx.druid.master; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class DynamicConfigs +{ + public static final String CONFIG_KEY = "master.dynamicConfigs"; + private long millisToWaitBeforeDeleting=15 * 60 * 1000L; + private long mergeBytesLimit= 100000000L; + private int mergeSegmentsLimit = Integer.MAX_VALUE; + private int maxSegmentsToMove = 5; + + @JsonCreator + public DynamicConfigs(@JsonProperty("millisToWaitBeforeDeleting") Integer millisToWaitBeforeDeleting, + @JsonProperty("mergeBytesLimit") Long mergeBytesLimit, + @JsonProperty("mergeSegmentsLimit") Integer mergeSegmentsLimit, + @JsonProperty("maxSegmentsToMove") Integer maxSegmentsToMove + ) + { + if (maxSegmentsToMove!=null) + { + this.maxSegmentsToMove=maxSegmentsToMove; + } + if (millisToWaitBeforeDeleting!=null) + { + this.millisToWaitBeforeDeleting=millisToWaitBeforeDeleting; + } + if (mergeSegmentsLimit!=null) + { + this.mergeSegmentsLimit=mergeSegmentsLimit; + } + if (mergeBytesLimit!=null) + { + this.mergeBytesLimit=mergeBytesLimit; + } + } + + public DynamicConfigs() + { + } + + public static String getConfigKey() + { + return CONFIG_KEY; + } + + public long getMillisToWaitBeforeDeleting() + { + return millisToWaitBeforeDeleting; + } + + public long getMergeBytesLimit() + { + return mergeBytesLimit; + } + + public int getMergeSegmentsLimit() + { + return mergeSegmentsLimit; + } + + public int getMaxSegmentsToMove() + { + return maxSegmentsToMove; + } +} diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java b/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java index c0d3f0eade8..61765e65386 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java +++ b/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java @@ -167,7 +167,6 @@ public class DruidMasterBalancerTest LoadQueuePeonTester fromPeon = new LoadQueuePeonTester(); LoadQueuePeonTester toPeon = new LoadQueuePeonTester(); - DruidMasterRuntimeParams params = DruidMasterRuntimeParams.newBuilder() .withDruidCluster( @@ -184,9 +183,25 @@ public class DruidMasterBalancerTest ) ) ) - .withLoadManagementPeons(ImmutableMap.of("from", fromPeon, "to", toPeon)) + .withLoadManagementPeons( + ImmutableMap.of( + "from", + fromPeon, + "to", + toPeon + ) + ) .withAvailableSegments(segments.values()) - .withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE) + .withDynamicConfigs( + new DynamicConfigs() + { + @Override + public int getMaxSegmentsToMove() + { + return MAX_SEGMENTS_TO_MOVE; + } + } + ) .withBalancerReferenceTimestamp(new DateTime("2013-01-01")) .build(); @@ -264,7 +279,14 @@ public class DruidMasterBalancerTest ) .withLoadManagementPeons(ImmutableMap.of("1", peon1, "2", peon2, "3", peon3, "4", peon4)) .withAvailableSegments(segments.values()) - .withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE) + .withDynamicConfigs(new DynamicConfigs() + { + @Override + public int getMaxSegmentsToMove() + { + return MAX_SEGMENTS_TO_MOVE; + } + }) .withBalancerReferenceTimestamp(new DateTime("2013-01-01")) .build(); diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java b/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java index 70e6258ec88..92cffba22bd 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java +++ b/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java @@ -177,7 +177,14 @@ public class DruidMasterRuleRunnerTest .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster())) - .withMaxSegmentsToMove(5) + .withDynamicConfigs(new DynamicConfigs() + { + @Override + public int getMaxSegmentsToMove() + { + return 5; + } + }) .withBalancerReferenceTimestamp(new DateTime("2013-01-01")) .build(); @@ -523,7 +530,14 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withDynamicConfigs( new DynamicConfigs() + { + @Override + public long getMillisToWaitBeforeDeleting() + { + return 0L; + } + }) .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) @@ -597,7 +611,14 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withDynamicConfigs( new DynamicConfigs() + { + @Override + public long getMillisToWaitBeforeDeleting() + { + return 0L; + } + }) .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) @@ -678,7 +699,14 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withDynamicConfigs( new DynamicConfigs() + { + @Override + public long getMillisToWaitBeforeDeleting() + { + return 0L; + } + }) .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) @@ -755,7 +783,14 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withDynamicConfigs( new DynamicConfigs() + { + @Override + public long getMillisToWaitBeforeDeleting() + { + return 0L; + } + }) .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) @@ -844,7 +879,14 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withDynamicConfigs( new DynamicConfigs() + { + @Override + public long getMillisToWaitBeforeDeleting() + { + return 0L; + } + }) .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) @@ -1030,7 +1072,14 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withMillisToWaitBeforeDeleting(0L) + .withDynamicConfigs( new DynamicConfigs() + { + @Override + public long getMillisToWaitBeforeDeleting() + { + return 0L; + } + }) .withAvailableSegments(longerAvailableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(segmentReplicantLookup) diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java b/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java index 44cac2ad19e..df86bda0ead 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java +++ b/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java @@ -419,10 +419,21 @@ public class DruidMasterSegmentMergerTest final AtomicReference whitelistRef = new AtomicReference(null); final DruidMasterSegmentMerger merger = new DruidMasterSegmentMerger(indexingServiceClient, whitelistRef); + final DynamicConfigs dynamicConfigs = new DynamicConfigs(null,mergeBytesLimit,mergeSegmentsLimit,null); final DruidMasterRuntimeParams params = DruidMasterRuntimeParams.newBuilder() .withAvailableSegments(ImmutableSet.copyOf(segments)) - .withMergeBytesLimit(mergeBytesLimit) - .withMergeSegmentsLimit(mergeSegmentsLimit) + .withDynamicConfigs( + new DynamicConfigs() + { + @Override + public long getMergeBytesLimit(){ + return mergeBytesLimit; + } + @Override + public int getMergeSegmentsLimit(){ + return mergeSegmentsLimit; + } + }) .build(); merger.run(params); diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java b/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java index a49dc85a582..273c664549f 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java +++ b/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java @@ -188,4 +188,5 @@ public class DruidMasterTest master.moveSegment("from", "to", "dummySegment", null); } + }