diff --git a/client/pom.xml b/client/pom.xml index 809d02343b9..cd8bacf5dc0 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 340f8a6d3a2..7f47d7fffed 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT diff --git a/druid-services/pom.xml b/druid-services/pom.xml index d1aab696a14..ca691b5e84a 100644 --- a/druid-services/pom.xml +++ b/druid-services/pom.xml @@ -24,11 +24,11 @@ druid-services druid-services druid-services - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT com.metamx druid - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 88151a158f5..3e790ab4be6 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT diff --git a/examples/rand/pom.xml b/examples/rand/pom.xml index 54d30ddd5a8..b4394161de6 100644 --- a/examples/rand/pom.xml +++ b/examples/rand/pom.xml @@ -9,7 +9,7 @@ com.metamx druid-examples - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT diff --git a/examples/twitter/pom.xml b/examples/twitter/pom.xml index 3b7c2d3582d..3f68c7c82a6 100644 --- a/examples/twitter/pom.xml +++ b/examples/twitter/pom.xml @@ -9,7 +9,7 @@ com.metamx druid-examples - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT diff --git a/index-common/pom.xml b/index-common/pom.xml index d5db4e3be63..ebe57f00fb1 100644 --- a/index-common/pom.xml +++ b/index-common/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT diff --git a/indexer/pom.xml b/indexer/pom.xml index b1d065b9977..a736362d25d 100644 --- a/indexer/pom.xml +++ b/indexer/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT diff --git a/merger/pom.xml b/merger/pom.xml index fab94d85089..d478400f00c 100644 --- a/merger/pom.xml +++ b/merger/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/http/IndexerCoordinatorResource.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/http/IndexerCoordinatorResource.java index 21adae8b09b..ee84e777101 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/http/IndexerCoordinatorResource.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/http/IndexerCoordinatorResource.java @@ -176,6 +176,9 @@ public class IndexerCoordinatorResource if (!configManager.set(WorkerSetupData.CONFIG_KEY, workerSetupData)) { return Response.status(Response.Status.BAD_REQUEST).build(); } + + log.info("Updating Worker Setup configs: %s", workerSetupData); + return Response.ok().build(); } diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategy.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategy.java index 082870c83c8..1d9e5f9f52e 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategy.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategy.java @@ -102,18 +102,22 @@ public class SimpleResourceManagementStrategy implements ResourceManagementStrat } } } else { - Duration durSinceLastProvision = new Duration(new DateTime(), lastProvisionTime); + Duration durSinceLastProvision = new Duration(lastProvisionTime, new DateTime()); + + log.info( + "%s still provisioning. Wait for all provisioned nodes to complete before requesting new worker. Current wait time: %s", + currentlyProvisioning, + durSinceLastProvision + ); + if (durSinceLastProvision.isLongerThan(config.getMaxScalingDuration())) { - log.makeAlert("Worker node provisioning taking too long") + log.makeAlert("Worker node provisioning taking too long!") .addData("millisSinceLastProvision", durSinceLastProvision.getMillis()) .addData("provisioningCount", currentlyProvisioning.size()) .emit(); - } - log.info( - "%s still provisioning. Wait for all provisioned nodes to complete before requesting new worker.", - currentlyProvisioning - ); + currentlyProvisioning.clear(); + } } return false; @@ -203,18 +207,21 @@ public class SimpleResourceManagementStrategy implements ResourceManagementStrat return true; } } else { - Duration durSinceLastTerminate = new Duration(new DateTime(), lastTerminateTime); - if (durSinceLastTerminate.isLongerThan(config.getMaxScalingDuration())) { - log.makeAlert("Worker node termination taking too long") - .addData("millisSinceLastTerminate", durSinceLastTerminate.getMillis()) - .addData("terminatingCount", currentlyTerminating.size()) - .emit(); - } + Duration durSinceLastTerminate = new Duration(lastTerminateTime, new DateTime()); log.info( "%s still terminating. Wait for all nodes to terminate before trying again.", currentlyTerminating ); + + if (durSinceLastTerminate.isLongerThan(config.getMaxScalingDuration())) { + log.makeAlert("Worker node termination taking too long!") + .addData("millisSinceLastTerminate", durSinceLastTerminate.getMillis()) + .addData("terminatingCount", currentlyTerminating.size()) + .emit(); + + currentlyTerminating.clear(); + } } return false; diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/EC2NodeData.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/EC2NodeData.java index 4c2b86f4f6f..6def924eb98 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/EC2NodeData.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/EC2NodeData.java @@ -91,4 +91,17 @@ public class EC2NodeData { return keyName; } + + @Override + public String toString() + { + return "EC2NodeData{" + + "amiId='" + amiId + '\'' + + ", instanceType='" + instanceType + '\'' + + ", minInstances=" + minInstances + + ", maxInstances=" + maxInstances + + ", securityGroupIds=" + securityGroupIds + + ", keyName='" + keyName + '\'' + + '}'; + } } diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/GalaxyUserData.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/GalaxyUserData.java index 76061637312..262fe3ac66d 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/GalaxyUserData.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/GalaxyUserData.java @@ -60,4 +60,14 @@ public class GalaxyUserData { return type; } + + @Override + public String toString() + { + return "GalaxyUserData{" + + "env='" + env + '\'' + + ", version='" + version + '\'' + + ", type='" + type + '\'' + + '}'; + } } diff --git a/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/WorkerSetupData.java b/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/WorkerSetupData.java index 18cd85e6962..7fc28437300 100644 --- a/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/WorkerSetupData.java +++ b/merger/src/main/java/com/metamx/druid/merger/coordinator/setup/WorkerSetupData.java @@ -75,4 +75,15 @@ public class WorkerSetupData { return userData; } + + @Override + public String toString() + { + return "WorkerSetupData{" + + "minVersion='" + minVersion + '\'' + + ", minNumWorkers=" + minNumWorkers + + ", nodeData=" + nodeData + + ", userData=" + userData + + '}'; + } } diff --git a/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategyTest.java b/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategyTest.java index 2052ae014bb..186723ef7df 100644 --- a/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategyTest.java +++ b/merger/src/test/java/com/metamx/druid/merger/coordinator/scaling/SimpleResourceManagementStrategyTest.java @@ -30,6 +30,10 @@ import com.metamx.druid.merger.coordinator.TaskRunnerWorkItem; import com.metamx.druid.merger.coordinator.ZkWorker; import com.metamx.druid.merger.coordinator.setup.WorkerSetupData; import com.metamx.druid.merger.worker.Worker; +import com.metamx.emitter.EmittingLogger; +import com.metamx.emitter.core.Event; +import com.metamx.emitter.service.ServiceEmitter; +import com.metamx.emitter.service.ServiceEventBuilder; import junit.framework.Assert; import org.easymock.EasyMock; import org.joda.time.DateTime; @@ -90,7 +94,7 @@ public class SimpleResourceManagementStrategyTest @Override public Duration getMaxScalingDuration() { - return null; + return new Duration(1000); } @Override @@ -184,6 +188,62 @@ public class SimpleResourceManagementStrategyTest EasyMock.verify(autoScalingStrategy); } + @Test + public void testProvisionAlert() throws Exception + { + ServiceEmitter emitter = EasyMock.createMock(ServiceEmitter.class); + EmittingLogger.registerEmitter(emitter); + emitter.emit(EasyMock.anyObject()); + EasyMock.expectLastCall(); + EasyMock.replay(emitter); + + EasyMock.expect(autoScalingStrategy.ipToIdLookup(EasyMock.>anyObject())) + .andReturn(Lists.newArrayList()).times(2); + EasyMock.expect(autoScalingStrategy.provision()).andReturn( + new AutoScalingData(Lists.newArrayList("fake"), Lists.newArrayList("faker")) + ); + EasyMock.replay(autoScalingStrategy); + + boolean provisionedSomething = simpleResourceManagementStrategy.doProvision( + Arrays.asList( + new TaskRunnerWorkItem(testTask, null, null, null).withQueueInsertionTime(new DateTime()) + ), + Arrays.asList( + new TestZkWorker(testTask) + ) + ); + + Assert.assertTrue(provisionedSomething); + Assert.assertTrue(simpleResourceManagementStrategy.getStats().toList().size() == 1); + DateTime createdTime = simpleResourceManagementStrategy.getStats().toList().get(0).getTimestamp(); + Assert.assertTrue( + simpleResourceManagementStrategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION + ); + + Thread.sleep(2000); + + provisionedSomething = simpleResourceManagementStrategy.doProvision( + Arrays.asList( + new TaskRunnerWorkItem(testTask, null, null, null).withQueueInsertionTime(new DateTime()) + ), + Arrays.asList( + new TestZkWorker(testTask) + ) + ); + + Assert.assertFalse(provisionedSomething); + Assert.assertTrue( + simpleResourceManagementStrategy.getStats().toList().get(0).getEvent() == ScalingStats.EVENT.PROVISION + ); + DateTime anotherCreatedTime = simpleResourceManagementStrategy.getStats().toList().get(0).getTimestamp(); + Assert.assertTrue( + createdTime.equals(anotherCreatedTime) + ); + + EasyMock.verify(autoScalingStrategy); + EasyMock.verify(emitter); + } + @Test public void testDoSuccessfulTerminate() throws Exception { diff --git a/pom.xml b/pom.xml index 27e639e7c18..5d2635955a7 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ com.metamx druid pom - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT druid druid diff --git a/realtime/pom.xml b/realtime/pom.xml index 24b19ccf813..0ded94e9d25 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index edbc04a6806..c8ec34d5f14 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ com.metamx druid - 0.3.19-SNAPSHOT + 0.3.20-SNAPSHOT