From 9235b4506387439a3d6586b95aa52b20f24f5812 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 10 Mar 2015 11:28:12 -0700 Subject: [PATCH] EC2AutoScaler: Support for setting subnetId. --- .../autoscaling/ec2/EC2AutoScaler.java | 1 + .../overlord/autoscaling/ec2/EC2NodeData.java | 16 ++- .../autoscaling/EC2AutoScalerSerdeTest.java | 111 ++++++++++++++++++ .../autoscaling/EC2AutoScalerTest.java | 7 +- .../setup/WorkerBehaviorConfigTest.java | 3 +- 5 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java index 71c7bf06249..d3d88b7d415 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java @@ -117,6 +117,7 @@ public class EC2AutoScaler implements AutoScaler .withSecurityGroupIds(workerConfig.getSecurityGroupIds()) .withPlacement(new Placement(envConfig.getAvailabilityZone())) .withKeyName(workerConfig.getKeyName()) + .withSubnetId(workerConfig.getSubnetId()) .withUserData(userDataBase64) ); diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java index 81f7d9ec3bb..7a2700efec7 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java @@ -32,6 +32,7 @@ public class EC2NodeData private final int maxInstances; private final List securityGroupIds; private final String keyName; + private final String subnetId; @JsonCreator public EC2NodeData( @@ -40,7 +41,8 @@ public class EC2NodeData @JsonProperty("minInstances") int minInstances, @JsonProperty("maxInstances") int maxInstances, @JsonProperty("securityGroupIds") List securityGroupIds, - @JsonProperty("keyName") String keyName + @JsonProperty("keyName") String keyName, + @JsonProperty("subnetId") String subnetId ) { this.amiId = amiId; @@ -49,6 +51,7 @@ public class EC2NodeData this.maxInstances = maxInstances; this.securityGroupIds = securityGroupIds; this.keyName = keyName; + this.subnetId = subnetId; } @JsonProperty @@ -87,6 +90,12 @@ public class EC2NodeData return keyName; } + @JsonProperty + public String getSubnetId() + { + return subnetId; + } + @Override public String toString() { @@ -97,6 +106,7 @@ public class EC2NodeData ", maxInstances=" + maxInstances + ", securityGroupIds=" + securityGroupIds + ", keyName='" + keyName + '\'' + + ", subnetId='" + subnetId + '\'' + '}'; } @@ -130,6 +140,9 @@ public class EC2NodeData if (securityGroupIds != null ? !securityGroupIds.equals(that.securityGroupIds) : that.securityGroupIds != null) { return false; } + if (subnetId != null ? !subnetId.equals(that.subnetId) : that.subnetId != null) { + return false; + } return true; } @@ -143,6 +156,7 @@ public class EC2NodeData result = 31 * result + maxInstances; result = 31 * result + (securityGroupIds != null ? securityGroupIds.hashCode() : 0); result = 31 * result + (keyName != null ? keyName.hashCode() : 0); + result = 31 * result + (subnetId != null ? subnetId.hashCode() : 0); return result; } } diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java new file mode 100644 index 00000000000..1e306773fb5 --- /dev/null +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java @@ -0,0 +1,111 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.indexing.overlord.autoscaling; + +import com.fasterxml.jackson.databind.BeanProperty; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; +import com.google.common.collect.Lists; +import com.google.common.io.BaseEncoding; +import io.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler; +import io.druid.jackson.DefaultObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +public class EC2AutoScalerSerdeTest +{ + final String json = "{\n" + + " \"envConfig\" : {\n" + + " \"availabilityZone\" : \"westeros-east-1a\",\n" + + " \"nodeData\" : {\n" + + " \"amiId\" : \"ami-abc\",\n" + + " \"instanceType\" : \"t1.micro\",\n" + + " \"keyName\" : \"iron\",\n" + + " \"maxInstances\" : 1,\n" + + " \"minInstances\" : 1,\n" + + " \"securityGroupIds\" : [\"kingsguard\"],\n" + + " \"subnetId\" : \"redkeep\"\n" + + " },\n" + + " \"userData\" : {\n" + + " \"data\" : \"VERSION=:VERSION:\\n\"," + + " \"impl\" : \"string\",\n" + + " \"versionReplacementString\" : \":VERSION:\"\n" + + " }\n" + + " },\n" + + " \"maxNumWorkers\" : 3,\n" + + " \"minNumWorkers\" : 2,\n" + + " \"type\" : \"ec2\"\n" + + "}"; + + @Test + public void testSerde() throws Exception + { + final ObjectMapper objectMapper = new DefaultObjectMapper(); + objectMapper.setInjectableValues( + new InjectableValues() + { + @Override + public Object findInjectableValue( + Object o, DeserializationContext deserializationContext, BeanProperty beanProperty, Object o1 + ) + { + return null; + } + } + ); + + final EC2AutoScaler autoScaler = objectMapper.readValue(json, EC2AutoScaler.class); + + Assert.assertEquals(3, autoScaler.getMaxNumWorkers()); + Assert.assertEquals(2, autoScaler.getMinNumWorkers()); + Assert.assertEquals("westeros-east-1a", autoScaler.getEnvConfig().getAvailabilityZone()); + + // nodeData + Assert.assertEquals("ami-abc", autoScaler.getEnvConfig().getNodeData().getAmiId()); + Assert.assertEquals("t1.micro", autoScaler.getEnvConfig().getNodeData().getInstanceType()); + Assert.assertEquals("iron", autoScaler.getEnvConfig().getNodeData().getKeyName()); + Assert.assertEquals(1, autoScaler.getEnvConfig().getNodeData().getMaxInstances()); + Assert.assertEquals(1, autoScaler.getEnvConfig().getNodeData().getMinInstances()); + Assert.assertEquals( + Lists.newArrayList("kingsguard"), + autoScaler.getEnvConfig().getNodeData().getSecurityGroupIds() + ); + Assert.assertEquals("redkeep", autoScaler.getEnvConfig().getNodeData().getSubnetId()); + + // userData + Assert.assertEquals( + "VERSION=1234\n", + new String( + BaseEncoding.base64() + .decode(autoScaler.getEnvConfig().getUserData().withVersion("1234").getUserDataBase64()), + Charsets.UTF_8 + ) + ); + + // Round trip. + Assert.assertEquals( + "Round trip", + autoScaler, + objectMapper.readValue(objectMapper.writeValueAsBytes(autoScaler), EC2AutoScaler.class) + ); + } +} diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerTest.java index c2f7fe2b332..8b4e491bf4b 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerTest.java @@ -25,7 +25,10 @@ import com.amazonaws.services.ec2.model.Reservation; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.RunInstancesResult; import com.amazonaws.services.ec2.model.TerminateInstancesRequest; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; import com.google.common.collect.Lists; +import com.google.common.io.BaseEncoding; import io.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler; import io.druid.indexing.overlord.autoscaling.ec2.EC2EnvironmentConfig; import io.druid.indexing.overlord.autoscaling.ec2.EC2NodeData; @@ -89,8 +92,8 @@ public class EC2AutoScalerTest 1, new EC2EnvironmentConfig( "us-east-1a", - new EC2NodeData(AMI_ID, INSTANCE_ID, 1, 1, Lists.newArrayList(), "foo"), - new GalaxyEC2UserData(new DefaultObjectMapper(), "env", "version", "type") + new EC2NodeData(AMI_ID, INSTANCE_ID, 1, 1, Lists.newArrayList(), "foo", "mySubnet"), + new GalaxyEC2UserData(new DefaultObjectMapper(), "env", "version", "type") ), amazonEC2Client, managementConfig diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java index ed36a36d63d..50a9515835b 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java @@ -54,7 +54,8 @@ public class WorkerBehaviorConfigTest 3, 5, Arrays.asList("securityGroupIds"), - "keyNames" + "keyNames", + "subnetId" ), new StringEC2UserData( "availZone",