Merge pull request #257 from metamx/az

Add availability zone info to autoscaling

indexing-service/src/main/java/io/druid/indexing/overlord/scaling/EC2AutoScalingStrategy.java

@@ -24,6 +24,7 @@ import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
 import com.amazonaws.services.ec2.model.DescribeInstancesResult;
 import com.amazonaws.services.ec2.model.Filter;
 import com.amazonaws.services.ec2.model.Instance;
+import com.amazonaws.services.ec2.model.Placement;
 import com.amazonaws.services.ec2.model.Reservation;
 import com.amazonaws.services.ec2.model.RunInstancesRequest;
 import com.amazonaws.services.ec2.model.RunInstancesResult;
@@ -88,6 +89,7 @@ public class EC2AutoScalingStrategy implements AutoScalingStrategy
           )
               .withInstanceType(workerConfig.getInstanceType())
               .withSecurityGroupIds(workerConfig.getSecurityGroupIds())
+              .withPlacement(new Placement(setupData.getAvailabilityZone()))
               .withKeyName(workerConfig.getKeyName())
               .withUserData(
                   Base64.encodeBase64String(

indexing-service/src/main/java/io/druid/indexing/overlord/setup/WorkerSetupData.java

@@ -31,6 +31,7 @@ public class WorkerSetupData
   private final String minVersion;
   private final int minNumWorkers;
   private final int maxNumWorkers;
+  private final String availabilityZone;
   private final EC2NodeData nodeData;
   private final GalaxyUserData userData;
 
@@ -39,6 +40,7 @@ public class WorkerSetupData
       @JsonProperty("minVersion") String minVersion,
       @JsonProperty("minNumWorkers") int minNumWorkers,
       @JsonProperty("maxNumWorkers") int maxNumWorkers,
+      @JsonProperty("availabilityZone") String availabilityZone,
       @JsonProperty("nodeData") EC2NodeData nodeData,
       @JsonProperty("userData") GalaxyUserData userData
   )
@@ -46,6 +48,7 @@ public class WorkerSetupData
     this.minVersion = minVersion;
     this.minNumWorkers = minNumWorkers;
     this.maxNumWorkers = maxNumWorkers;
+    this.availabilityZone = availabilityZone;
     this.nodeData = nodeData;
     this.userData = userData;
   }
@@ -68,6 +71,12 @@ public class WorkerSetupData
     return maxNumWorkers;
   }
 
+  @JsonProperty
+  public String getAvailabilityZone()
+  {
+    return availabilityZone;
+  }
+
   @JsonProperty
   public EC2NodeData getNodeData()
   {
@@ -87,6 +96,7 @@ public class WorkerSetupData
            "minVersion='" + minVersion + '\'' +
            ", minNumWorkers=" + minNumWorkers +
            ", maxNumWorkers=" + maxNumWorkers +
+           ", availabilityZone=" + availabilityZone +
            ", nodeData=" + nodeData +
            ", userData=" + userData +
            '}';

indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java

@@ -392,7 +392,7 @@ public class RemoteTaskRunnerTest
         },
         cf,
         new SimplePathChildrenCacheFactory.Builder().build(),
-        DSuppliers.of(new AtomicReference<WorkerSetupData>(new WorkerSetupData("0", 0, 1, null, null))),
+        DSuppliers.of(new AtomicReference<WorkerSetupData>(new WorkerSetupData("0", 0, 1, null, null, null))),
         null
     );
 

indexing-service/src/test/java/io/druid/indexing/overlord/scaling/EC2AutoScalingStrategyTest.java

@@ -99,6 +99,7 @@ public class EC2AutoScalingStrategyTest
             "0",
             0,
             1,
+            "",
             new EC2NodeData(AMI_ID, INSTANCE_ID, 1, 1, Lists.<String>newArrayList(), "foo"),
             new GalaxyUserData("env", "version", "type")
         )

indexing-service/src/test/java/io/druid/indexing/overlord/scaling/SimpleResourceManagementStrategyTest.java

@@ -65,7 +65,7 @@ public class SimpleResourceManagementStrategyTest
     autoScalingStrategy = EasyMock.createMock(AutoScalingStrategy.class);
     workerSetupData = new AtomicReference<WorkerSetupData>(
         new WorkerSetupData(
-            "0", 0, 2, null, null
+            "0", 0, 2, null, null, null
         )
     );
 
@@ -237,7 +237,7 @@ public class SimpleResourceManagementStrategyTest
   @Test
   public void testDoSuccessfulTerminate() throws Exception
   {
-    workerSetupData.set(new WorkerSetupData("0", 0, 1, null, null));
+    workerSetupData.set(new WorkerSetupData("0", 0, 1, null, null, null));
 
     EasyMock.expect(autoScalingStrategy.ipToIdLookup(EasyMock.<List<String>>anyObject()))
             .andReturn(Lists.<String>newArrayList());
@@ -267,7 +267,7 @@ public class SimpleResourceManagementStrategyTest
   @Test
   public void testSomethingTerminating() throws Exception
   {
-    workerSetupData.set(new WorkerSetupData("0", 0, 1, null, null));
+    workerSetupData.set(new WorkerSetupData("0", 0, 1, null, null, null));
 
     EasyMock.expect(autoScalingStrategy.ipToIdLookup(EasyMock.<List<String>>anyObject()))
             .andReturn(Lists.<String>newArrayList("ip")).times(2);

server/src/main/java/io/druid/server/coordinator/DruidCoordinatorConfig.java

@@ -31,7 +31,7 @@ public abstract class DruidCoordinatorConfig
   public abstract String getHost();
 
   @Config("druid.coordinator.startDelay")
-  @Default("PT60s")
+  @Default("PT120s")
   public abstract Duration getCoordinatorStartDelay();
 
   @Config("druid.coordinator.period")