From 2306602e933e967d023b132dab7424fb5f9b8e54 Mon Sep 17 00:00:00 2001 From: Eric Tschetter Date: Tue, 26 Mar 2013 09:06:11 -0500 Subject: [PATCH 1/4] 1) Remove zookeeper 3.4.3 dependency from merger --- merger/pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/merger/pom.xml b/merger/pom.xml index d8ca1172bb0..ec6f64e76dd 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -160,11 +160,6 @@ com.github.sgroschupf zkclient - - org.apache.zookeeper - zookeeper - 3.4.3 - From a7d836a0dd06e1a8670021d2497b23b5055e2241 Mon Sep 17 00:00:00 2001 From: Fangjin Yang Date: Tue, 26 Mar 2013 10:25:52 -0700 Subject: [PATCH 2/4] fix NPE for master balancing; introduce max number of workers for indexing service --- .../SimpleResourceManagementStrategy.java | 9 +++ .../coordinator/setup/WorkerSetupData.java | 12 +++- .../coordinator/RemoteTaskRunnerTest.java | 2 +- .../scaling/EC2AutoScalingStrategyTest.java | 1 + .../SimpleResourceManagementStrategyTest.java | 10 ++- .../druid/master/DruidMasterLogger.java | 66 ++++++++++++------- 6 files changed, 69 insertions(+), 31 deletions(-) diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategy.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategy.java index 1d9e5f9f52e..80beace6430 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategy.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategy.java @@ -70,6 +70,15 @@ public class SimpleResourceManagementStrategy implements ResourceManagementStrat @Override public boolean doProvision(Collection pendingTasks, Collection zkWorkers) { + if (zkWorkers.size() >= workerSetupdDataRef.get().getMaxNumWorkers()) { + log.info( + "Cannot scale anymore. Num workers = %d, Max num workers = %d", + zkWorkers.size(), + workerSetupdDataRef.get().getMaxNumWorkers() + ); + return false; + } + List workerNodeIds = autoScalingStrategy.ipToIdLookup( Lists.newArrayList( Iterables.transform( diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/WorkerSetupData.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/WorkerSetupData.java index 7fc28437300..f647f694998 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/WorkerSetupData.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/WorkerSetupData.java @@ -20,8 +20,6 @@ package com.metamx.druid.merger.coordinator.setup; - - import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -35,6 +33,7 @@ public class WorkerSetupData private final String minVersion; private final int minNumWorkers; + private final int maxNumWorkers; private final EC2NodeData nodeData; private final GalaxyUserData userData; @@ -42,12 +41,14 @@ public class WorkerSetupData public WorkerSetupData( @JsonProperty("minVersion") String minVersion, @JsonProperty("minNumWorkers") int minNumWorkers, + @JsonProperty("maxNumWorkers") int maxNumWorkers, @JsonProperty("nodeData") EC2NodeData nodeData, @JsonProperty("userData") GalaxyUserData userData ) { this.minVersion = minVersion; this.minNumWorkers = minNumWorkers; + this.maxNumWorkers = maxNumWorkers; this.nodeData = nodeData; this.userData = userData; } @@ -64,6 +65,12 @@ public class WorkerSetupData return minNumWorkers; } + @JsonProperty + public int getMaxNumWorkers() + { + return maxNumWorkers; + } + @JsonProperty public EC2NodeData getNodeData() { @@ -82,6 +89,7 @@ public class WorkerSetupData return "WorkerSetupData{" + "minVersion='" + minVersion + '\'' + ", minNumWorkers=" + minNumWorkers + + ", maxNumWorkers=" + maxNumWorkers + ", nodeData=" + nodeData + ", userData=" + userData + '}'; diff --git a/merger/src/test/java/com/metamx/druid/merger/coordinator/RemoteTaskRunnerTest.java b/merger/src/test/java/com/metamx/druid/merger/coordinator/RemoteTaskRunnerTest.java index a3980820268..febe9394372 100644 --- a/merger/src/test/java/com/metamx/druid/merger/coordinator/RemoteTaskRunnerTest.java +++ b/merger/src/test/java/com/metamx/druid/merger/coordinator/RemoteTaskRunnerTest.java @@ -324,7 +324,7 @@ public class RemoteTaskRunnerTest pathChildrenCache, scheduledExec, new RetryPolicyFactory(new TestRetryPolicyConfig()), - new AtomicReference(new WorkerSetupData("0", 0, null, null)) + new AtomicReference(new WorkerSetupData("0", 0, 1, null, null)) ); // Create a single worker and wait for things for be ready diff --git a/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/EC2AutoScalingStrategyTest.java b/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/EC2AutoScalingStrategyTest.java index cd569cb77e8..e6ab815e262 100644 --- a/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/EC2AutoScalingStrategyTest.java +++ b/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/EC2AutoScalingStrategyTest.java @@ -105,6 +105,7 @@ public class EC2AutoScalingStrategyTest new WorkerSetupData( "0", 0, + 1, new EC2NodeData(AMI_ID, INSTANCE_ID, 1, 1, Lists.newArrayList(), "foo"), new GalaxyUserData("env", "version", "type") ) diff --git a/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategyTest.java b/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategyTest.java index 186723ef7df..145c9034e03 100644 --- a/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategyTest.java +++ b/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategyTest.java @@ -60,7 +60,11 @@ public class SimpleResourceManagementStrategyTest public void setUp() throws Exception { autoScalingStrategy = EasyMock.createMock(AutoScalingStrategy.class); - workerSetupData = new AtomicReference(null); + workerSetupData = new AtomicReference( + new WorkerSetupData( + "0", 0, 2, null, null + ) + ); testTask = new TestTask( "task1", @@ -247,7 +251,7 @@ public class SimpleResourceManagementStrategyTest @Test public void testDoSuccessfulTerminate() throws Exception { - workerSetupData.set(new WorkerSetupData("0", 0, null, null)); + workerSetupData.set(new WorkerSetupData("0", 0, 1, null, null)); EasyMock.expect(autoScalingStrategy.ipToIdLookup(EasyMock.>anyObject())) .andReturn(Lists.newArrayList()); @@ -277,7 +281,7 @@ public class SimpleResourceManagementStrategyTest @Test public void testSomethingTerminating() throws Exception { - workerSetupData.set(new WorkerSetupData("0", 0, null, null)); + workerSetupData.set(new WorkerSetupData("0", 0, 1, null, null)); EasyMock.expect(autoScalingStrategy.ipToIdLookup(EasyMock.>anyObject())) .andReturn(Lists.newArrayList("ip")).times(2); diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterLogger.java b/server/src/main/java/com/metamx/druid/master/DruidMasterLogger.java index 1328619db44..382d03966b6 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterLogger.java +++ b/server/src/main/java/com/metamx/druid/master/DruidMasterLogger.java @@ -29,7 +29,6 @@ import com.metamx.druid.collect.CountingMap; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; -import javax.annotation.Nullable; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; @@ -40,16 +39,20 @@ public class DruidMasterLogger implements DruidMasterHelper { private static final Logger log = new Logger(DruidMasterLogger.class); - private void emitTieredStats(final ServiceEmitter emitter, final String formatString, final Map statMap) + private void emitTieredStats( + final ServiceEmitter emitter, + final String formatString, + final Map statMap + ) { - if (statMap != null) { + if (statMap != null) { for (Map.Entry entry : statMap.entrySet()) { String tier = entry.getKey(); Number value = entry.getValue(); emitter.emit( new ServiceMetricEvent.Builder().build( - String.format(formatString, tier), value.doubleValue() - ) + String.format(formatString, tier), value.doubleValue() + ) ); } } @@ -82,30 +85,43 @@ public class DruidMasterLogger implements DruidMasterHelper } } - emitTieredStats(emitter, "master/%s/cost/raw", - stats.getPerTierStats().get("initialCost")); + emitTieredStats( + emitter, "master/%s/cost/raw", + stats.getPerTierStats().get("initialCost") + ); - emitTieredStats(emitter, "master/%s/cost/normalization", - stats.getPerTierStats().get("normalization")); + emitTieredStats( + emitter, "master/%s/cost/normalization", + stats.getPerTierStats().get("normalization") + ); - emitTieredStats(emitter, "master/%s/moved/count", - stats.getPerTierStats().get("movedCount")); + emitTieredStats( + emitter, "master/%s/moved/count", + stats.getPerTierStats().get("movedCount") + ); - emitTieredStats(emitter, "master/%s/deleted/count", - stats.getPerTierStats().get("deletedCount")); + emitTieredStats( + emitter, "master/%s/deleted/count", + stats.getPerTierStats().get("deletedCount") + ); - emitTieredStats(emitter, "master/%s/cost/normalized", - Maps.transformEntries(stats.getPerTierStats().get("normalizedInitialCostTimesOneThousand"), - new Maps.EntryTransformer() - { - @Override - public Number transformEntry( - @Nullable String key, @Nullable AtomicLong value - ) - { - return value.doubleValue() / 1000d; - } - })); + Map normalized = stats.getPerTierStats().get("normalizedInitialCostTimesOneThousand"); + if (normalized != null) { + emitTieredStats( + emitter, "master/%s/cost/normalized", + Maps.transformEntries( + normalized, + new Maps.EntryTransformer() + { + @Override + public Number transformEntry(String key, AtomicLong value) + { + return value.doubleValue() / 1000d; + } + } + ) + ); + } Map unneeded = stats.getPerTierStats().get("unneededCount"); if (unneeded != null) { From a97287fa50e7ed763159c91789c6695a7246391f Mon Sep 17 00:00:00 2001 From: Fangjin Yang Date: Tue, 26 Mar 2013 11:10:04 -0700 Subject: [PATCH 3/4] [maven-release-plugin] prepare release druid-0.3.28 --- client/pom.xml | 2 +- common/pom.xml | 2 +- druid-services/pom.xml | 4 ++-- examples/pom.xml | 2 +- examples/rand/pom.xml | 2 +- examples/twitter/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 5b44a907c1c..39f6d79cb87 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28-SNAPSHOT + 0.3.28 diff --git a/common/pom.xml b/common/pom.xml index 67857c543f6..3c8d2d2f292 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28-SNAPSHOT + 0.3.28 diff --git a/druid-services/pom.xml b/druid-services/pom.xml index 9a90a88481d..8c33b5c7c7a 100644 --- a/druid-services/pom.xml +++ b/druid-services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.3.28-SNAPSHOT + 0.3.28 com.metamx druid - 0.3.28-SNAPSHOT + 0.3.28 diff --git a/examples/pom.xml b/examples/pom.xml index 106c6e04c30..369477130e6 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28-SNAPSHOT + 0.3.28 diff --git a/examples/rand/pom.xml b/examples/rand/pom.xml index 33537a83c28..84e64115973 100644 --- a/examples/rand/pom.xml +++ b/examples/rand/pom.xml @@ -9,7 +9,7 @@ com.metamx druid-examples - 0.3.28-SNAPSHOT + 0.3.28 diff --git a/examples/twitter/pom.xml b/examples/twitter/pom.xml index 674309f7a9c..b0a89d02f3f 100644 --- a/examples/twitter/pom.xml +++ b/examples/twitter/pom.xml @@ -9,7 +9,7 @@ com.metamx druid-examples - 0.3.28-SNAPSHOT + 0.3.28 diff --git a/index-common/pom.xml b/index-common/pom.xml index 42d4a29b0c4..fd231a906fd 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28-SNAPSHOT + 0.3.28 diff --git a/indexer/pom.xml b/indexer/pom.xml index db6a1b2e3a6..fef409a4e65 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28-SNAPSHOT + 0.3.28 diff --git a/merger/pom.xml b/merger/pom.xml index 48a4ffab96b..a05729f5667 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28-SNAPSHOT + 0.3.28 diff --git a/pom.xml b/pom.xml index 2fd37a68160..f56f208f9c7 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.3.28-SNAPSHOT + 0.3.28 druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index e8ac06907af..8bb0d51465e 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28-SNAPSHOT + 0.3.28 diff --git a/server/pom.xml b/server/pom.xml index caecb6b70cb..bd255c92654 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28-SNAPSHOT + 0.3.28 From e785b87be0e02669ea85316889c516f7131e3b37 Mon Sep 17 00:00:00 2001 From: Fangjin Yang Date: Tue, 26 Mar 2013 11:10:15 -0700 Subject: [PATCH 4/4] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- druid-services/pom.xml | 4 ++-- examples/pom.xml | 2 +- examples/rand/pom.xml | 2 +- examples/twitter/pom.xml | 2 +- index-common/pom.xml | 2 +- indexer/pom.xml | 2 +- merger/pom.xml | 2 +- pom.xml | 2 +- realtime/pom.xml | 2 +- server/pom.xml | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 39f6d79cb87..d75801ecb54 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28 + 0.3.29-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 3c8d2d2f292..cfb8eb16446 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28 + 0.3.29-SNAPSHOT diff --git a/druid-services/pom.xml b/druid-services/pom.xml index 8c33b5c7c7a..aabd9f898e6 100644 --- a/druid-services/pom.xml +++ b/druid-services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.3.28 + 0.3.29-SNAPSHOT com.metamx druid - 0.3.28 + 0.3.29-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 369477130e6..a8ce4ab125a 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28 + 0.3.29-SNAPSHOT diff --git a/examples/rand/pom.xml b/examples/rand/pom.xml index 84e64115973..27d1e1d3256 100644 --- a/examples/rand/pom.xml +++ b/examples/rand/pom.xml @@ -9,7 +9,7 @@ com.metamx druid-examples - 0.3.28 + 0.3.29-SNAPSHOT diff --git a/examples/twitter/pom.xml b/examples/twitter/pom.xml index b0a89d02f3f..c3621f8f67c 100644 --- a/examples/twitter/pom.xml +++ b/examples/twitter/pom.xml @@ -9,7 +9,7 @@ com.metamx druid-examples - 0.3.28 + 0.3.29-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index fd231a906fd..c64af29bbd5 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28 + 0.3.29-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index fef409a4e65..8dd869f10a1 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28 + 0.3.29-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index a05729f5667..7b3e8058e25 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28 + 0.3.29-SNAPSHOT diff --git a/pom.xml b/pom.xml index f56f208f9c7..07aacf06a50 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.3.28 + 0.3.29-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 8bb0d51465e..924b7f71078 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28 + 0.3.29-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index bd255c92654..de7e66c1b93 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.28 + 0.3.29-SNAPSHOT