From b0d9889c83cd093ad09fc3e745d55197dadde576 Mon Sep 17 00:00:00 2001 From: Dhruv Parthasarathy Date: Mon, 29 Jul 2013 17:49:09 -0700 Subject: [PATCH] added web console for changing dynamicConfigs --- .../com/metamx/druid/http/InfoResource.java | 4 +- .../com/metamx/druid/http/MasterMain.java | 3 +- .../druid/http/MasterServletModule.java | 7 +- .../com/metamx/druid/master/DruidMaster.java | 4 +- .../master/DruidMasterRuntimeParams.java | 2 +- .../metamx/druid/master/DynamicConfigs.java | 7 +- .../main/resources/static/dynamicConfigs.html | 68 +++++++++++++++++++ server/src/main/resources/static/index.html | 3 + .../resources/static/js/tablehelper-0.0.2.js | 2 +- .../druid/master/DruidMasterBalancerTest.java | 4 +- .../master/DruidMasterRuleRunnerTest.java | 14 ++-- .../master/DruidMasterSegmentMergerTest.java | 3 +- 12 files changed, 100 insertions(+), 21 deletions(-) create mode 100644 server/src/main/resources/static/dynamicConfigs.html diff --git a/server/src/main/java/com/metamx/druid/http/InfoResource.java b/server/src/main/java/com/metamx/druid/http/InfoResource.java index a03d5d9abf0..3a2d6ec32b9 100644 --- a/server/src/main/java/com/metamx/druid/http/InfoResource.java +++ b/server/src/main/java/com/metamx/druid/http/InfoResource.java @@ -140,7 +140,7 @@ public class InfoResource @Produces("application/json") public Response getDynamicConfigs() { - Response.ResponseBuilder builder = Response.status(Response.Status.OK); + Response.ResponseBuilder builder = Response.status(Response.Status.OK).entity(configManager.watch(DynamicConfigs.CONFIG_KEY,DynamicConfigs.class).get()); return builder.build(); } @@ -390,7 +390,7 @@ public class InfoResource @POST @Path("master/setDynamicConfigs") - @Produces("application/json") + @Consumes("application/json") public Response setDynamicConfigs( final DynamicConfigs dynamicConfigs) { diff --git a/server/src/main/java/com/metamx/druid/http/MasterMain.java b/server/src/main/java/com/metamx/druid/http/MasterMain.java index 63521717e79..6dc8fa8ad0a 100644 --- a/server/src/main/java/com/metamx/druid/http/MasterMain.java +++ b/server/src/main/java/com/metamx/druid/http/MasterMain.java @@ -274,7 +274,8 @@ public class MasterMain databaseRuleManager, master, jsonMapper, - indexingServiceClient + indexingServiceClient, + configManager ) ); diff --git a/server/src/main/java/com/metamx/druid/http/MasterServletModule.java b/server/src/main/java/com/metamx/druid/http/MasterServletModule.java index 26d9ad15459..31fa2fa0db0 100644 --- a/server/src/main/java/com/metamx/druid/http/MasterServletModule.java +++ b/server/src/main/java/com/metamx/druid/http/MasterServletModule.java @@ -25,6 +25,7 @@ import com.google.inject.Provides; import com.google.inject.util.Providers; import com.metamx.druid.client.InventoryView; import com.metamx.druid.client.indexing.IndexingServiceClient; +import com.metamx.druid.config.JacksonConfigManager; import com.metamx.druid.db.DatabaseRuleManager; import com.metamx.druid.db.DatabaseSegmentManager; import com.metamx.druid.master.DruidMaster; @@ -43,6 +44,7 @@ public class MasterServletModule extends JerseyServletModule private final DruidMaster master; private final ObjectMapper jsonMapper; private final IndexingServiceClient indexingServiceClient; + private final JacksonConfigManager configManager; public MasterServletModule( InventoryView serverInventoryView, @@ -50,7 +52,8 @@ public class MasterServletModule extends JerseyServletModule DatabaseRuleManager databaseRuleManager, DruidMaster master, ObjectMapper jsonMapper, - IndexingServiceClient indexingServiceClient + IndexingServiceClient indexingServiceClient, + JacksonConfigManager configManager ) { this.serverInventoryView = serverInventoryView; @@ -59,6 +62,7 @@ public class MasterServletModule extends JerseyServletModule this.master = master; this.jsonMapper = jsonMapper; this.indexingServiceClient = indexingServiceClient; + this.configManager = configManager; } @Override @@ -70,6 +74,7 @@ public class MasterServletModule extends JerseyServletModule bind(DatabaseSegmentManager.class).toInstance(segmentInventoryManager); bind(DatabaseRuleManager.class).toInstance(databaseRuleManager); bind(DruidMaster.class).toInstance(master); + bind(JacksonConfigManager.class).toInstance(configManager); if (indexingServiceClient == null) { bind(IndexingServiceClient.class).toProvider(Providers.of(null)); } diff --git a/server/src/main/java/com/metamx/druid/master/DruidMaster.java b/server/src/main/java/com/metamx/druid/master/DruidMaster.java index 253efa13ad3..409abe9c5cd 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMaster.java +++ b/server/src/main/java/com/metamx/druid/master/DruidMaster.java @@ -468,7 +468,8 @@ public class DruidMaster final List> masterRunnables = Lists.newArrayList(); if (!defaultConfigsSet) { - configManager.set(DynamicConfigs.CONFIG_KEY, new DynamicConfigs()); + configManager.watch(DynamicConfigs.CONFIG_KEY, DynamicConfigs.class); + configManager.set(DynamicConfigs.CONFIG_KEY, new DynamicConfigs(null,null,null,null)); defaultConfigsSet=true; } masterRunnables.add(Pair.of(new MasterComputeManagerRunnable(), config.getMasterPeriod())); @@ -760,6 +761,7 @@ public class DruidMaster .withLoadManagementPeons(loadManagementPeons) .withSegmentReplicantLookup(segmentReplicantLookup) .withBalancerReferenceTimestamp(DateTime.now()) + .withDynamicConfigs(configManager.watch(DynamicConfigs.CONFIG_KEY,DynamicConfigs.class).get()) .build(); } }, diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java b/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java index 7650dcc9b4b..f0610fd3dac 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java +++ b/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java @@ -214,7 +214,7 @@ public class DruidMasterRuntimeParams this.replicationManager = null; this.emitter = null; this.stats = new MasterStats(); - this.dynamicConfigs = new DynamicConfigs(); + this.dynamicConfigs = new DynamicConfigs(null,null,null,null); this.balancerReferenceTimestamp = null; } diff --git a/server/src/main/java/com/metamx/druid/master/DynamicConfigs.java b/server/src/main/java/com/metamx/druid/master/DynamicConfigs.java index 2c0758e3b2d..b6fa793824b 100644 --- a/server/src/main/java/com/metamx/druid/master/DynamicConfigs.java +++ b/server/src/main/java/com/metamx/druid/master/DynamicConfigs.java @@ -54,30 +54,31 @@ public class DynamicConfigs } } - public DynamicConfigs() - { - } public static String getConfigKey() { return CONFIG_KEY; } + @JsonProperty public long getMillisToWaitBeforeDeleting() { return millisToWaitBeforeDeleting; } + @JsonProperty public long getMergeBytesLimit() { return mergeBytesLimit; } + @JsonProperty public int getMergeSegmentsLimit() { return mergeSegmentsLimit; } + @JsonProperty public int getMaxSegmentsToMove() { return maxSegmentsToMove; diff --git a/server/src/main/resources/static/dynamicConfigs.html b/server/src/main/resources/static/dynamicConfigs.html new file mode 100644 index 00000000000..68ded614407 --- /dev/null +++ b/server/src/main/resources/static/dynamicConfigs.html @@ -0,0 +1,68 @@ + + + + Configure Dynamic Master Parameters + + + + + + + + + +
+
+
+ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/src/main/resources/static/index.html b/server/src/main/resources/static/index.html index 8a6a68a7faa..97e80fad9b9 100644 --- a/server/src/main/resources/static/index.html +++ b/server/src/main/resources/static/index.html @@ -37,6 +37,9 @@
Configure Compute Node Rules
+
+ Configure Dynamic Master Parameters +
Enable/Disable Datasources
diff --git a/server/src/main/resources/static/js/tablehelper-0.0.2.js b/server/src/main/resources/static/js/tablehelper-0.0.2.js index 3c060ba09a4..ce8537056fa 100644 --- a/server/src/main/resources/static/js/tablehelper-0.0.2.js +++ b/server/src/main/resources/static/js/tablehelper-0.0.2.js @@ -78,4 +78,4 @@ function initDataTable(el) { this.value = asInitVals[$("thead input").index(this)]; } }); -} \ No newline at end of file + } \ No newline at end of file diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java b/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java index 61765e65386..7e5f0059807 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java +++ b/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java @@ -193,7 +193,7 @@ public class DruidMasterBalancerTest ) .withAvailableSegments(segments.values()) .withDynamicConfigs( - new DynamicConfigs() + new DynamicConfigs(null,null,null,null) { @Override public int getMaxSegmentsToMove() @@ -279,7 +279,7 @@ public class DruidMasterBalancerTest ) .withLoadManagementPeons(ImmutableMap.of("1", peon1, "2", peon2, "3", peon3, "4", peon4)) .withAvailableSegments(segments.values()) - .withDynamicConfigs(new DynamicConfigs() + .withDynamicConfigs(new DynamicConfigs(null,null,null,null) { @Override public int getMaxSegmentsToMove() diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java b/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java index 92cffba22bd..89fdb3f26db 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java +++ b/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java @@ -177,7 +177,7 @@ public class DruidMasterRuleRunnerTest .withAvailableSegments(availableSegments) .withDatabaseRuleManager(databaseRuleManager) .withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster())) - .withDynamicConfigs(new DynamicConfigs() + .withDynamicConfigs(new DynamicConfigs(null,null,null,null) { @Override public int getMaxSegmentsToMove() @@ -530,7 +530,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withDynamicConfigs( new DynamicConfigs() + .withDynamicConfigs( new DynamicConfigs(null,null,null,null) { @Override public long getMillisToWaitBeforeDeleting() @@ -611,7 +611,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withDynamicConfigs( new DynamicConfigs() + .withDynamicConfigs( new DynamicConfigs(null,null,null,null) { @Override public long getMillisToWaitBeforeDeleting() @@ -699,7 +699,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withDynamicConfigs( new DynamicConfigs() + .withDynamicConfigs( new DynamicConfigs(null,null,null,null) { @Override public long getMillisToWaitBeforeDeleting() @@ -783,7 +783,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withDynamicConfigs( new DynamicConfigs() + .withDynamicConfigs( new DynamicConfigs(null,null,null,null) { @Override public long getMillisToWaitBeforeDeleting() @@ -879,7 +879,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withDynamicConfigs( new DynamicConfigs() + .withDynamicConfigs( new DynamicConfigs(null,null,null,null) { @Override public long getMillisToWaitBeforeDeleting() @@ -1072,7 +1072,7 @@ public class DruidMasterRuleRunnerTest DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder() .withDruidCluster(druidCluster) - .withDynamicConfigs( new DynamicConfigs() + .withDynamicConfigs( new DynamicConfigs(null,null,null,null) { @Override public long getMillisToWaitBeforeDeleting() diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java b/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java index df86bda0ead..39b357734b2 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java +++ b/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java @@ -419,11 +419,10 @@ public class DruidMasterSegmentMergerTest final AtomicReference whitelistRef = new AtomicReference(null); final DruidMasterSegmentMerger merger = new DruidMasterSegmentMerger(indexingServiceClient, whitelistRef); - final DynamicConfigs dynamicConfigs = new DynamicConfigs(null,mergeBytesLimit,mergeSegmentsLimit,null); final DruidMasterRuntimeParams params = DruidMasterRuntimeParams.newBuilder() .withAvailableSegments(ImmutableSet.copyOf(segments)) .withDynamicConfigs( - new DynamicConfigs() + new DynamicConfigs(null,null,null,null) { @Override public long getMergeBytesLimit(){