From 703b674800146488eb0bede4345a2dec354129f0 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 7 Oct 2013 12:16:50 -0700 Subject: [PATCH] add availability zone info to autoscaling --- .../overlord/scaling/EC2AutoScalingStrategy.java | 2 ++ .../druid/indexing/overlord/setup/WorkerSetupData.java | 10 ++++++++++ .../druid/indexing/overlord/RemoteTaskRunnerTest.java | 2 +- .../overlord/scaling/EC2AutoScalingStrategyTest.java | 1 + .../scaling/SimpleResourceManagementStrategyTest.java | 6 +++--- .../server/coordinator/DruidCoordinatorConfig.java | 2 +- 6 files changed, 18 insertions(+), 5 deletions(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/scaling/EC2AutoScalingStrategy.java b/indexing-service/src/main/java/io/druid/indexing/overlord/scaling/EC2AutoScalingStrategy.java index 2426d08890a..57c2d875ac5 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/scaling/EC2AutoScalingStrategy.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/scaling/EC2AutoScalingStrategy.java @@ -24,6 +24,7 @@ import com.amazonaws.services.ec2.model.DescribeInstancesRequest; import com.amazonaws.services.ec2.model.DescribeInstancesResult; import com.amazonaws.services.ec2.model.Filter; import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.Placement; import com.amazonaws.services.ec2.model.Reservation; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.RunInstancesResult; @@ -88,6 +89,7 @@ public class EC2AutoScalingStrategy implements AutoScalingStrategy ) .withInstanceType(workerConfig.getInstanceType()) .withSecurityGroupIds(workerConfig.getSecurityGroupIds()) + .withPlacement(new Placement(setupData.getAvailabilityZone())) .withKeyName(workerConfig.getKeyName()) .withUserData( Base64.encodeBase64String( diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/setup/WorkerSetupData.java b/indexing-service/src/main/java/io/druid/indexing/overlord/setup/WorkerSetupData.java index 7764b9f5fea..47bb145eefe 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/setup/WorkerSetupData.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/setup/WorkerSetupData.java @@ -31,6 +31,7 @@ public class WorkerSetupData private final String minVersion; private final int minNumWorkers; private final int maxNumWorkers; + private final String availabilityZone; private final EC2NodeData nodeData; private final GalaxyUserData userData; @@ -39,6 +40,7 @@ public class WorkerSetupData @JsonProperty("minVersion") String minVersion, @JsonProperty("minNumWorkers") int minNumWorkers, @JsonProperty("maxNumWorkers") int maxNumWorkers, + @JsonProperty("availabilityZone") String availabilityZone, @JsonProperty("nodeData") EC2NodeData nodeData, @JsonProperty("userData") GalaxyUserData userData ) @@ -46,6 +48,7 @@ public class WorkerSetupData this.minVersion = minVersion; this.minNumWorkers = minNumWorkers; this.maxNumWorkers = maxNumWorkers; + this.availabilityZone = availabilityZone; this.nodeData = nodeData; this.userData = userData; } @@ -68,6 +71,12 @@ public class WorkerSetupData return maxNumWorkers; } + @JsonProperty + public String getAvailabilityZone() + { + return availabilityZone; + } + @JsonProperty public EC2NodeData getNodeData() { @@ -87,6 +96,7 @@ public class WorkerSetupData "minVersion='" + minVersion + '\'' + ", minNumWorkers=" + minNumWorkers + ", maxNumWorkers=" + maxNumWorkers + + ", availabilityZone=" + availabilityZone + ", nodeData=" + nodeData + ", userData=" + userData + '}'; diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java index 6bf97fa85c8..f368e69758b 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java @@ -392,7 +392,7 @@ public class RemoteTaskRunnerTest }, cf, new SimplePathChildrenCacheFactory.Builder().build(), - DSuppliers.of(new AtomicReference(new WorkerSetupData("0", 0, 1, null, null))), + DSuppliers.of(new AtomicReference(new WorkerSetupData("0", 0, 1, null, null, null))), null ); diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/scaling/EC2AutoScalingStrategyTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/scaling/EC2AutoScalingStrategyTest.java index 368901fb605..d0b5edb5ca1 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/scaling/EC2AutoScalingStrategyTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/scaling/EC2AutoScalingStrategyTest.java @@ -99,6 +99,7 @@ public class EC2AutoScalingStrategyTest "0", 0, 1, + "", new EC2NodeData(AMI_ID, INSTANCE_ID, 1, 1, Lists.newArrayList(), "foo"), new GalaxyUserData("env", "version", "type") ) diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/scaling/SimpleResourceManagementStrategyTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/scaling/SimpleResourceManagementStrategyTest.java index 8cda83e48ea..02ac9a21778 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/scaling/SimpleResourceManagementStrategyTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/scaling/SimpleResourceManagementStrategyTest.java @@ -65,7 +65,7 @@ public class SimpleResourceManagementStrategyTest autoScalingStrategy = EasyMock.createMock(AutoScalingStrategy.class); workerSetupData = new AtomicReference( new WorkerSetupData( - "0", 0, 2, null, null + "0", 0, 2, null, null, null ) ); @@ -237,7 +237,7 @@ public class SimpleResourceManagementStrategyTest @Test public void testDoSuccessfulTerminate() throws Exception { - workerSetupData.set(new WorkerSetupData("0", 0, 1, null, null)); + workerSetupData.set(new WorkerSetupData("0", 0, 1, null, null, null)); EasyMock.expect(autoScalingStrategy.ipToIdLookup(EasyMock.>anyObject())) .andReturn(Lists.newArrayList()); @@ -267,7 +267,7 @@ public class SimpleResourceManagementStrategyTest @Test public void testSomethingTerminating() throws Exception { - workerSetupData.set(new WorkerSetupData("0", 0, 1, null, null)); + workerSetupData.set(new WorkerSetupData("0", 0, 1, null, null, null)); EasyMock.expect(autoScalingStrategy.ipToIdLookup(EasyMock.>anyObject())) .andReturn(Lists.newArrayList("ip")).times(2); diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorConfig.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorConfig.java index a1319a7b7f5..e4f50baa6fb 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorConfig.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorConfig.java @@ -31,7 +31,7 @@ public abstract class DruidCoordinatorConfig public abstract String getHost(); @Config("druid.coordinator.startDelay") - @Default("PT60s") + @Default("PT120s") public abstract Duration getCoordinatorStartDelay(); @Config("druid.coordinator.period")