first set of changes

server/src/main/java/com/metamx/druid/http/InfoResource.java

@@ -31,9 +31,11 @@ import com.metamx.druid.client.DruidDataSource;
 import com.metamx.druid.client.DruidServer;
 import com.metamx.druid.client.InventoryView;
 import com.metamx.druid.client.indexing.IndexingServiceClient;
+import com.metamx.druid.config.JacksonConfigManager;
 import com.metamx.druid.db.DatabaseRuleManager;
 import com.metamx.druid.db.DatabaseSegmentManager;
 import com.metamx.druid.master.DruidMaster;
+import com.metamx.druid.master.DynamicConfigs;
 import com.metamx.druid.master.rules.Rule;
 import org.joda.time.Interval;
 
@@ -101,6 +103,8 @@ public class InfoResource
   private final DatabaseSegmentManager databaseSegmentManager;
   private final DatabaseRuleManager databaseRuleManager;
   private final IndexingServiceClient indexingServiceClient;
+  private final JacksonConfigManager configManager;
+
 
   @Inject
   public InfoResource(
@@ -108,6 +112,7 @@ public class InfoResource
       InventoryView serverInventoryView,
       DatabaseSegmentManager databaseSegmentManager,
       DatabaseRuleManager databaseRuleManager,
+      JacksonConfigManager configManager,
       @Nullable
       IndexingServiceClient indexingServiceClient
   )
@@ -116,6 +121,7 @@ public class InfoResource
     this.serverInventoryView = serverInventoryView;
     this.databaseSegmentManager = databaseSegmentManager;
     this.databaseRuleManager = databaseRuleManager;
+    this.configManager=configManager;
     this.indexingServiceClient = indexingServiceClient;
   }
 
@@ -129,6 +135,15 @@ public class InfoResource
                    .build();
   }
 
+  @GET
+  @Path("/master/dynamicConfigs")
+  @Produces("application/json")
+  public Response getDynamicConfigs()
+  {
+    Response.ResponseBuilder builder = Response.status(Response.Status.OK);
+    return builder.build();
+  }
+
   @GET
   @Path("/cluster")
   @Produces("application/json")
@@ -373,6 +388,20 @@ public class InfoResource
     return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
   }
 
+  @POST
+  @Path("master/setDynamicConfigs")
+  @Produces("application/json")
+  public Response setDynamicConfigs(
+      final DynamicConfigs dynamicConfigs)
+  {
+    if (!configManager.set(DynamicConfigs.CONFIG_KEY, dynamicConfigs))
+    {
+      return Response.status(Response.Status.BAD_REQUEST).build();
+    }
+    return Response.status(Response.Status.OK).build();
+  }
+
+
   @GET
   @Path("/datasources")
   @Produces("application/json")

server/src/main/java/com/metamx/druid/master/DruidMaster.java

@@ -80,6 +80,7 @@ public class DruidMaster
 
   private volatile boolean started = false;
   private volatile boolean master = false;
+  private volatile boolean defaultConfigsSet=false;
 
   private final DruidMasterConfig config;
   private final ZkPathsConfig zkPaths;
@@ -465,7 +466,11 @@ public class DruidMaster
         serverInventoryView.start();
 
         final List<Pair<? extends MasterRunnable, Duration>> masterRunnables = Lists.newArrayList();
-
+        if (!defaultConfigsSet)
+        {
+         configManager.set(DynamicConfigs.CONFIG_KEY, new DynamicConfigs());
+          defaultConfigsSet=true;
+        }
         masterRunnables.add(Pair.of(new MasterComputeManagerRunnable(), config.getMasterPeriod()));
         if (indexingServiceClient != null) {
 
@@ -650,18 +655,17 @@ public class DruidMaster
         }
 
         // Do master stuff.
+        DynamicConfigs dynamicConfigs = configManager.watch(DynamicConfigs.CONFIG_KEY,DynamicConfigs.class).get();
 
         DruidMasterRuntimeParams params =
             DruidMasterRuntimeParams.newBuilder()
                                     .withStartTime(startTime)
                                     .withDatasources(databaseSegmentManager.getInventory())
-                                    .withMillisToWaitBeforeDeleting(config.getMillisToWaitBeforeDeleting())
+                                    .withDynamicConfigs(dynamicConfigs)
                                     .withEmitter(emitter)
-                                    .withMergeBytesLimit(config.getMergeBytesLimit())
-                                    .withMergeSegmentsLimit(config.getMergeSegmentsLimit())
-                                    .withMaxSegmentsToMove(config.getMaxSegmentsToMove())
                                     .build();
 
+
         for (DruidMasterHelper helper : helpers) {
           params = helper.run(params);
         }

server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java

@@ -46,11 +46,8 @@ public class DruidMasterRuntimeParams
   private final Map<String, LoadQueuePeon> loadManagementPeons;
   private final ReplicationThrottler replicationManager;
   private final ServiceEmitter emitter;
-  private final long millisToWaitBeforeDeleting;
+  private final DynamicConfigs dynamicConfigs;
   private final MasterStats stats;
-  private final long mergeBytesLimit;
-  private final int mergeSegmentsLimit;
-  private final int maxSegmentsToMove;
   private final DateTime balancerReferenceTimestamp;
 
   public DruidMasterRuntimeParams(
@@ -63,11 +60,8 @@ public class DruidMasterRuntimeParams
       Map<String, LoadQueuePeon> loadManagementPeons,
       ReplicationThrottler replicationManager,
       ServiceEmitter emitter,
-      long millisToWaitBeforeDeleting,
+      DynamicConfigs dynamicConfigs,
       MasterStats stats,
-      long mergeBytesLimit,
-      int mergeSegmentsLimit,
-      int maxSegmentsToMove,
       DateTime balancerReferenceTimestamp
   )
   {
@@ -80,11 +74,8 @@ public class DruidMasterRuntimeParams
     this.loadManagementPeons = loadManagementPeons;
     this.replicationManager = replicationManager;
     this.emitter = emitter;
-    this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting;
+    this.dynamicConfigs = dynamicConfigs;
     this.stats = stats;
-    this.mergeBytesLimit = mergeBytesLimit;
-    this.mergeSegmentsLimit = mergeSegmentsLimit;
-    this.maxSegmentsToMove = maxSegmentsToMove;
     this.balancerReferenceTimestamp = balancerReferenceTimestamp;
   }
 
@@ -135,7 +126,7 @@ public class DruidMasterRuntimeParams
 
   public long getMillisToWaitBeforeDeleting()
   {
-    return millisToWaitBeforeDeleting;
+    return dynamicConfigs.getMillisToWaitBeforeDeleting();
   }
 
   public MasterStats getMasterStats()
@@ -145,17 +136,17 @@ public class DruidMasterRuntimeParams
 
   public long getMergeBytesLimit()
   {
-    return mergeBytesLimit;
+    return dynamicConfigs.getMergeBytesLimit();
   }
 
   public int getMergeSegmentsLimit()
   {
-    return mergeSegmentsLimit;
+    return dynamicConfigs.getMergeSegmentsLimit();
   }
 
   public int getMaxSegmentsToMove()
   {
-    return maxSegmentsToMove;
+    return dynamicConfigs.getMaxSegmentsToMove();
   }
 
   public DateTime getBalancerReferenceTimestamp()
@@ -190,11 +181,8 @@ public class DruidMasterRuntimeParams
         loadManagementPeons,
         replicationManager,
         emitter,
-        millisToWaitBeforeDeleting,
+        dynamicConfigs,
         stats,
-        mergeBytesLimit,
-        mergeSegmentsLimit,
-        maxSegmentsToMove,
         balancerReferenceTimestamp
     );
   }
@@ -210,11 +198,8 @@ public class DruidMasterRuntimeParams
     private final Map<String, LoadQueuePeon> loadManagementPeons;
     private ReplicationThrottler replicationManager;
     private ServiceEmitter emitter;
-    private long millisToWaitBeforeDeleting;
+    private DynamicConfigs dynamicConfigs;
     private MasterStats stats;
-    private long mergeBytesLimit;
-    private int mergeSegmentsLimit;
-    private int maxSegmentsToMove;
     private DateTime balancerReferenceTimestamp;
 
     Builder()
@@ -228,11 +213,8 @@ public class DruidMasterRuntimeParams
       this.loadManagementPeons = Maps.newHashMap();
       this.replicationManager = null;
       this.emitter = null;
-      this.millisToWaitBeforeDeleting = 0;
       this.stats = new MasterStats();
-      this.mergeBytesLimit = 0;
+      this.dynamicConfigs = new DynamicConfigs();
-      this.mergeSegmentsLimit = 0;
-      this.maxSegmentsToMove = 0;
       this.balancerReferenceTimestamp = null;
     }
 
@@ -246,11 +228,8 @@ public class DruidMasterRuntimeParams
         Map<String, LoadQueuePeon> loadManagementPeons,
         ReplicationThrottler replicationManager,
         ServiceEmitter emitter,
-        long millisToWaitBeforeDeleting,
+        DynamicConfigs dynamicConfigs,
         MasterStats stats,
-        long mergeBytesLimit,
-        int mergeSegmentsLimit,
-        int maxSegmentsToMove,
         DateTime balancerReferenceTimestamp
     )
     {
@@ -263,11 +242,8 @@ public class DruidMasterRuntimeParams
       this.loadManagementPeons = loadManagementPeons;
       this.replicationManager = replicationManager;
       this.emitter = emitter;
-      this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting;
+      this.dynamicConfigs=dynamicConfigs;
       this.stats = stats;
-      this.mergeBytesLimit = mergeBytesLimit;
-      this.mergeSegmentsLimit = mergeSegmentsLimit;
-      this.maxSegmentsToMove = maxSegmentsToMove;
       this.balancerReferenceTimestamp = balancerReferenceTimestamp;
     }
 
@@ -283,11 +259,8 @@ public class DruidMasterRuntimeParams
           loadManagementPeons,
           replicationManager,
           emitter,
-          millisToWaitBeforeDeleting,
+          dynamicConfigs,
           stats,
-          mergeBytesLimit,
-          mergeSegmentsLimit,
-          maxSegmentsToMove,
           balancerReferenceTimestamp
       );
     }
@@ -346,33 +319,15 @@ public class DruidMasterRuntimeParams
       return this;
     }
 
-    public Builder withMillisToWaitBeforeDeleting(long millisToWaitBeforeDeleting)
-    {
-      this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting;
-      return this;
-    }
-
     public Builder withMasterStats(MasterStats stats)
     {
       this.stats.accumulate(stats);
       return this;
     }
 
-    public Builder withMergeBytesLimit(long mergeBytesLimit)
+    public Builder withDynamicConfigs(DynamicConfigs configs)
     {
-      this.mergeBytesLimit = mergeBytesLimit;
+      this.dynamicConfigs = configs;
-      return this;
-    }
-
-    public Builder withMergeSegmentsLimit(int mergeSegmentsLimit)
-    {
-      this.mergeSegmentsLimit = mergeSegmentsLimit;
-      return this;
-    }
-
-    public Builder withMaxSegmentsToMove(int maxSegmentsToMove)
-    {
-      this.maxSegmentsToMove = maxSegmentsToMove;
       return this;
     }
 

server/src/main/java/com/metamx/druid/master/DynamicConfigs.java

@@ -0,0 +1,85 @@
+/*
+* Druid - a distributed column store.
+* Copyright (C) 2012  Metamarkets Group Inc.
+*
+* This program is free software; you can redistribute it and/or
+* modify it under the terms of the GNU General Public License
+* as published by the Free Software Foundation; either version 2
+* of the License, or (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+package com.metamx.druid.master;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class DynamicConfigs
+{
+  public static final String CONFIG_KEY = "master.dynamicConfigs";
+  private long millisToWaitBeforeDeleting=15 * 60 * 1000L;
+  private long mergeBytesLimit= 100000000L;
+  private int mergeSegmentsLimit = Integer.MAX_VALUE;
+  private int maxSegmentsToMove = 5;
+
+  @JsonCreator
+  public DynamicConfigs(@JsonProperty("millisToWaitBeforeDeleting") Integer millisToWaitBeforeDeleting,
+                        @JsonProperty("mergeBytesLimit") Long mergeBytesLimit,
+                        @JsonProperty("mergeSegmentsLimit") Integer mergeSegmentsLimit,
+                        @JsonProperty("maxSegmentsToMove") Integer maxSegmentsToMove
+  )
+  {
+    if (maxSegmentsToMove!=null)
+    {
+      this.maxSegmentsToMove=maxSegmentsToMove;
+    }
+    if (millisToWaitBeforeDeleting!=null)
+    {
+      this.millisToWaitBeforeDeleting=millisToWaitBeforeDeleting;
+    }
+    if (mergeSegmentsLimit!=null)
+    {
+      this.mergeSegmentsLimit=mergeSegmentsLimit;
+    }
+    if (mergeBytesLimit!=null)
+    {
+      this.mergeBytesLimit=mergeBytesLimit;
+    }
+  }
+
+  public DynamicConfigs()
+  {
+  }
+
+  public static String getConfigKey()
+  {
+    return CONFIG_KEY;
+  }
+
+  public long getMillisToWaitBeforeDeleting()
+  {
+    return millisToWaitBeforeDeleting;
+  }
+
+  public long getMergeBytesLimit()
+  {
+    return mergeBytesLimit;
+  }
+
+  public int getMergeSegmentsLimit()
+  {
+    return mergeSegmentsLimit;
+  }
+
+  public int getMaxSegmentsToMove()
+  {
+    return maxSegmentsToMove;
+  }
+}

server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java

@@ -167,7 +167,6 @@ public class DruidMasterBalancerTest
 
     LoadQueuePeonTester fromPeon = new LoadQueuePeonTester();
     LoadQueuePeonTester toPeon = new LoadQueuePeonTester();
-
     DruidMasterRuntimeParams params =
         DruidMasterRuntimeParams.newBuilder()
                                 .withDruidCluster(
@@ -184,9 +183,25 @@ public class DruidMasterBalancerTest
                                         )
                                     )
                                 )
-                                .withLoadManagementPeons(ImmutableMap.<String, LoadQueuePeon>of("from", fromPeon, "to", toPeon))
+                                .withLoadManagementPeons(
+                                    ImmutableMap.<String, LoadQueuePeon>of(
+                                        "from",
+                                        fromPeon,
+                                        "to",
+                                        toPeon
+                                    )
+                                )
                                 .withAvailableSegments(segments.values())
-                                .withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE)
+                                .withDynamicConfigs(
+                                    new DynamicConfigs()
+                                    {
+                                      @Override
+                                      public int getMaxSegmentsToMove()
+                                      {
+                                        return MAX_SEGMENTS_TO_MOVE;
+                                      }
+                                    }
+                                )
                                 .withBalancerReferenceTimestamp(new DateTime("2013-01-01"))
                                 .build();
 
@@ -264,7 +279,14 @@ public class DruidMasterBalancerTest
                                 )
                                 .withLoadManagementPeons(ImmutableMap.<String, LoadQueuePeon>of("1", peon1, "2", peon2, "3", peon3, "4", peon4))
                                 .withAvailableSegments(segments.values())
-                                .withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE)
+                                .withDynamicConfigs(new DynamicConfigs()
+                                {
+                                  @Override
+                                  public int getMaxSegmentsToMove()
+                                  {
+                                    return MAX_SEGMENTS_TO_MOVE;
+                                  }
+                                })
                                 .withBalancerReferenceTimestamp(new DateTime("2013-01-01"))
                                 .build();
 

server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java

@@ -177,7 +177,14 @@ public class DruidMasterRuleRunnerTest
             .withAvailableSegments(availableSegments)
             .withDatabaseRuleManager(databaseRuleManager)
             .withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster()))
-            .withMaxSegmentsToMove(5)
+            .withDynamicConfigs(new DynamicConfigs()
+            {
+              @Override
+              public int getMaxSegmentsToMove()
+              {
+                return 5;
+              }
+            })
             .withBalancerReferenceTimestamp(new DateTime("2013-01-01"))
             .build();
 
@@ -523,7 +530,14 @@ public class DruidMasterRuleRunnerTest
 
     DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
         .withDruidCluster(druidCluster)
-        .withMillisToWaitBeforeDeleting(0L)
+        .withDynamicConfigs( new DynamicConfigs()
+        {
+          @Override
+          public long getMillisToWaitBeforeDeleting()
+          {
+            return 0L;
+          }
+        })
         .withAvailableSegments(availableSegments)
         .withDatabaseRuleManager(databaseRuleManager)
         .withSegmentReplicantLookup(segmentReplicantLookup)
@@ -597,7 +611,14 @@ public class DruidMasterRuleRunnerTest
 
     DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
         .withDruidCluster(druidCluster)
-        .withMillisToWaitBeforeDeleting(0L)
+        .withDynamicConfigs( new DynamicConfigs()
+        {
+          @Override
+          public long getMillisToWaitBeforeDeleting()
+          {
+            return 0L;
+          }
+        })
         .withAvailableSegments(availableSegments)
         .withDatabaseRuleManager(databaseRuleManager)
         .withSegmentReplicantLookup(segmentReplicantLookup)
@@ -678,7 +699,14 @@ public class DruidMasterRuleRunnerTest
 
     DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
         .withDruidCluster(druidCluster)
-        .withMillisToWaitBeforeDeleting(0L)
+        .withDynamicConfigs( new DynamicConfigs()
+        {
+          @Override
+          public long getMillisToWaitBeforeDeleting()
+          {
+            return 0L;
+          }
+        })
         .withAvailableSegments(availableSegments)
         .withDatabaseRuleManager(databaseRuleManager)
         .withSegmentReplicantLookup(segmentReplicantLookup)
@@ -755,7 +783,14 @@ public class DruidMasterRuleRunnerTest
 
     DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
         .withDruidCluster(druidCluster)
-        .withMillisToWaitBeforeDeleting(0L)
+        .withDynamicConfigs( new DynamicConfigs()
+        {
+          @Override
+          public long getMillisToWaitBeforeDeleting()
+          {
+            return 0L;
+          }
+        })
         .withAvailableSegments(availableSegments)
         .withDatabaseRuleManager(databaseRuleManager)
         .withSegmentReplicantLookup(segmentReplicantLookup)
@@ -844,7 +879,14 @@ public class DruidMasterRuleRunnerTest
 
     DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
         .withDruidCluster(druidCluster)
-        .withMillisToWaitBeforeDeleting(0L)
+        .withDynamicConfigs( new DynamicConfigs()
+        {
+          @Override
+          public long getMillisToWaitBeforeDeleting()
+          {
+            return 0L;
+          }
+        })
         .withAvailableSegments(availableSegments)
         .withDatabaseRuleManager(databaseRuleManager)
         .withSegmentReplicantLookup(segmentReplicantLookup)
@@ -1030,7 +1072,14 @@ public class DruidMasterRuleRunnerTest
 
     DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
         .withDruidCluster(druidCluster)
-        .withMillisToWaitBeforeDeleting(0L)
+        .withDynamicConfigs( new DynamicConfigs()
+        {
+          @Override
+          public long getMillisToWaitBeforeDeleting()
+          {
+            return 0L;
+          }
+        })
         .withAvailableSegments(longerAvailableSegments)
         .withDatabaseRuleManager(databaseRuleManager)
         .withSegmentReplicantLookup(segmentReplicantLookup)

server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java

@@ -419,10 +419,21 @@ public class DruidMasterSegmentMergerTest
 
     final AtomicReference<MergerWhitelist> whitelistRef = new AtomicReference<MergerWhitelist>(null);
     final DruidMasterSegmentMerger merger = new DruidMasterSegmentMerger(indexingServiceClient, whitelistRef);
+    final DynamicConfigs dynamicConfigs = new DynamicConfigs(null,mergeBytesLimit,mergeSegmentsLimit,null);
     final DruidMasterRuntimeParams params = DruidMasterRuntimeParams.newBuilder()
                                                                     .withAvailableSegments(ImmutableSet.copyOf(segments))
-                                                                    .withMergeBytesLimit(mergeBytesLimit)
+                                                                    .withDynamicConfigs(
-                                                                    .withMergeSegmentsLimit(mergeSegmentsLimit)
+                                                                        new DynamicConfigs()
+                                                                        {
+                                                                           @Override
+                                                                           public long getMergeBytesLimit(){
+                                                                             return mergeBytesLimit;
+                                                                           }
+                                                                           @Override
+                                                                           public int getMergeSegmentsLimit(){
+                                                                             return mergeSegmentsLimit;
+                                                                          }
+                                                                        })
                                                                     .build();
 
     merger.run(params);

server/src/test/java/com/metamx/druid/master/DruidMasterTest.java

@@ -188,4 +188,5 @@ public class DruidMasterTest
 
     master.moveSegment("from", "to", "dummySegment", null);
   }
+
 }