Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 757: Auto allocate Elastic IPs and deal with empty tags in describe security groups response

This commit is contained in:
Jan Paral 2011-12-18 16:47:13 -07:00
parent 39290d725c
commit 1f095fda4a
11 changed files with 335 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apis/ec2/src/main/java/org/jclouds/ec2/EC2PropertiesBuilder.java
View File

@ -22,6 +22,7 @@ import static org.jclouds.Constants.PROPERTY_API_VERSION;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_AUTH_TAG;
import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG;
import static org.jclouds.ec2.reference.EC2Constants.PROPERTY_EC2_AMI_OWNERS;
import static org.jclouds.ec2.reference.EC2Constants.PROPERTY_EC2_AUTO_ALLOCATE_ELASTIC_IPS;
import static org.jclouds.ec2.reference.EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT;
import java.util.Properties;
@ -42,6 +43,7 @@ public class EC2PropertiesBuilder extends PropertiesBuilder {
properties.setProperty(PROPERTY_API_VERSION, EC2AsyncClient.VERSION);
properties.setProperty(PROPERTY_EC2_AMI_OWNERS, "*");
properties.setProperty(PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT, "500");
properties.setProperty(PROPERTY_EC2_AUTO_ALLOCATE_ELASTIC_IPS, "false");
return properties;
}

15
apis/ec2/src/main/java/org/jclouds/ec2/compute/config/EC2ComputeServiceDependenciesModule.java
View File

@ -41,6 +41,7 @@ import org.jclouds.ec2.EC2AsyncClient;
import org.jclouds.ec2.EC2Client;
import org.jclouds.ec2.compute.EC2ComputeService;
import org.jclouds.ec2.compute.domain.RegionAndName;
import org.jclouds.ec2.compute.functions.AddElasticIpsToNodemetadata;
import org.jclouds.ec2.compute.functions.CreateSecurityGroupIfNeeded;
import org.jclouds.ec2.compute.functions.CreateUniqueKeyPair;
import org.jclouds.ec2.compute.functions.CredentialsForInstance;
@ -52,11 +53,13 @@ import org.jclouds.ec2.compute.predicates.SecurityGroupPresent;
import org.jclouds.ec2.domain.InstanceState;
import org.jclouds.ec2.domain.KeyPair;
import org.jclouds.ec2.domain.RunningInstance;
import org.jclouds.ec2.reference.EC2Constants;
import org.jclouds.predicates.RetryablePredicate;
import org.jclouds.rest.RestContext;
import org.jclouds.rest.internal.RestContextImpl;
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Predicate;
import com.google.common.base.Supplier;
import com.google.common.cache.LoadingCache;
@ -94,8 +97,6 @@ public class EC2ComputeServiceDependenciesModule extends AbstractModule {
bind(TemplateBuilder.class).to(EC2TemplateBuilderImpl.class);
bind(TemplateOptions.class).to(EC2TemplateOptions.class);
bind(ComputeService.class).to(EC2ComputeService.class);
bind(new TypeLiteral<Function<RunningInstance, NodeMetadata>>() {
}).to(RunningInstanceToNodeMetadata.class);
bind(new TypeLiteral<CacheLoader<RunningInstance, Credentials>>() {
}).to(CredentialsForInstance.class);
bind(new TypeLiteral<CacheLoader<RegionAndName, String>>() {
@ -112,6 +113,16 @@ public class EC2ComputeServiceDependenciesModule extends AbstractModule {
}).in(Scopes.SINGLETON);
}
@Provides
@Singleton
public Function<RunningInstance, NodeMetadata> bindNodeConverter(RunningInstanceToNodeMetadata baseConverter,
AddElasticIpsToNodemetadata addElasticIpsToNodemetadata,
@Named(EC2Constants.PROPERTY_EC2_AUTO_ALLOCATE_ELASTIC_IPS) boolean autoAllocateElasticIps) {
if (!autoAllocateElasticIps)
return baseConverter;
return Functions.compose(addElasticIpsToNodemetadata, baseConverter);
}
@Provides
@Singleton
Supplier<String> provideSuffix() {

105
apis/ec2/src/main/java/org/jclouds/ec2/compute/functions/AddElasticIpsToNodemetadata.java Normal file
View File

@ -0,0 +1,105 @@
/**
* Licensed to jclouds, Inc. (jclouds) under one or more
* contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. jclouds licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jclouds.ec2.compute.functions;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.jclouds.aws.util.AWSUtils;
import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.compute.domain.NodeMetadataBuilder;
import org.jclouds.ec2.EC2Client;
import org.jclouds.ec2.domain.PublicIpInstanceIdPair;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
/**
* This class searches for elastic ip addresses that are associated with the node, and adds them to
* the publicIpAddress collection if present.
*
* @author Adrian Cole
*/
@Singleton
public class AddElasticIpsToNodemetadata implements Function<NodeMetadata, NodeMetadata> {
private final EC2Client client;
@Inject
AddElasticIpsToNodemetadata(EC2Client client) {
this.client = client;
}
@Override
public NodeMetadata apply(NodeMetadata arg0) {
String[] parts = AWSUtils.parseHandle(arg0.getId());
String region = parts[0];
String instanceId = parts[1];
Iterable<PublicIpInstanceIdPair> elasticIpsAssociatedWithNode = ipAddressPairsAssignedToInstance(region,
instanceId);
Set<String> publicIps = extractIpAddressFromPairs(elasticIpsAssociatedWithNode);
return addPublicIpsToNode(publicIps, arg0);
}
@VisibleForTesting
NodeMetadata addPublicIpsToNode(Set<String> publicIps, NodeMetadata arg0) {
if (publicIps.size() > 0)
arg0 = NodeMetadataBuilder.fromNodeMetadata(arg0).publicAddresses(
Iterables.concat(publicIps, arg0.getPublicAddresses())).build();
return arg0;
}
@VisibleForTesting
Set<String> extractIpAddressFromPairs(Iterable<PublicIpInstanceIdPair> elasticIpsAssociatedWithNode) {
Set<String> publicIps = ImmutableSet.copyOf(Iterables.transform(elasticIpsAssociatedWithNode,
new Function<PublicIpInstanceIdPair, String>() {
@Override
public String apply(PublicIpInstanceIdPair arg0) {
return arg0.getPublicIp();
}
}));
return publicIps;
}
@VisibleForTesting
Iterable<PublicIpInstanceIdPair> ipAddressPairsAssignedToInstance(String region, final String instanceId) {
Iterable<PublicIpInstanceIdPair> elasticIpsAssociatedWithNode = Iterables.filter(client
.getElasticIPAddressServices().describeAddressesInRegion(region),
new Predicate<PublicIpInstanceIdPair>() {
@Override
public boolean apply(PublicIpInstanceIdPair in) {
return instanceId.equals(in.getInstanceId());
}
});
return elasticIpsAssociatedWithNode;
}
}

42
apis/ec2/src/main/java/org/jclouds/ec2/compute/strategy/EC2CreateNodesInGroupThenAddToSet.java
View File

@ -47,6 +47,7 @@ import org.jclouds.ec2.compute.domain.RegionAndName;
import org.jclouds.ec2.compute.predicates.InstancePresent;
import org.jclouds.ec2.domain.RunningInstance;
import org.jclouds.ec2.options.RunInstancesOptions;
import org.jclouds.ec2.reference.EC2Constants;
import org.jclouds.logging.Logger;
import com.google.common.annotations.VisibleForTesting;
@ -69,6 +70,10 @@ public class EC2CreateNodesInGroupThenAddToSet implements CreateNodesInGroupThen
@Named(ComputeServiceConstants.COMPUTE_LOGGER)
protected Logger logger = Logger.NULL;
@Inject
@Named(EC2Constants.PROPERTY_EC2_AUTO_ALLOCATE_ELASTIC_IPS)
boolean autoAllocateElasticIps = false;
@VisibleForTesting
final EC2Client client;
@VisibleForTesting
@ -114,6 +119,9 @@ public class EC2CreateNodesInGroupThenAddToSet implements CreateNodesInGroupThen
Map<NodeMetadata, Exception> badNodes, Multimap<NodeMetadata, CustomizationResponse> customizationResponses) {
// ensure we don't mutate the input template
template = templateBuilderProvider.get().fromTemplate(template).build();
Iterable<String> ips = allocateElasticIpsInRegion(count, template);
Iterable<? extends RunningInstance> started = createKeyPairAndSecurityGroupsAsNeededThenRunInstances(group,
count, template);
@ -127,6 +135,8 @@ public class EC2CreateNodesInGroupThenAddToSet implements CreateNodesInGroupThen
populateCredentials(started);
}
assignElasticIpsToInstances(count, started, ips, template);
return utils.customizeNodesAndAddToGoodMapOrPutExceptionIntoBadMap(template.getOptions(), transform(started,
runningInstanceToNodeMetadata), goodNodes, badNodes, customizationResponses);
}
@ -143,6 +153,38 @@ public class EC2CreateNodesInGroupThenAddToSet implements CreateNodesInGroupThen
credentialStore.put("node#" + instance.getRegion() + "/" + instance.getId(), credentials);
}
// TODO write test for this
protected Iterable<String> allocateElasticIpsInRegion(int count, Template template) {
Iterable<String> ips = ImmutableSet.<String> of();
if (!autoAllocateElasticIps)
return ips;
String region = AWSUtils.getRegionFromLocationOrNull(template.getLocation());
logger.debug("<< allocating elastic IPs for nodes in region (%s)", region);
for (int i=0; i<count; ++i) {
ips = Iterables.concat(ips, ImmutableSet.<String> of(
client.getElasticIPAddressServices().allocateAddressInRegion(region)));
}
return ips;
}
// TODO write test for this
protected void assignElasticIpsToInstances(int count, Iterable<? extends RunningInstance> started,
Iterable<String> ips, Template template) {
if (!autoAllocateElasticIps)
return;
String region = AWSUtils.getRegionFromLocationOrNull(template.getLocation());
for (int i=0; i<count; ++i) {
String ip = Iterables.get(ips, i);
String id = Iterables.get(started, i).getId();
client.getElasticIPAddressServices().associateAddressInRegion(region, ip, id);
}
}
// TODO write test for this
protected Iterable<? extends RunningInstance> createKeyPairAndSecurityGroupsAsNeededThenRunInstances(String group,
int count, Template template) {

21
apis/ec2/src/main/java/org/jclouds/ec2/compute/strategy/EC2DestroyNodeStrategy.java
View File

@ -20,6 +20,9 @@ package org.jclouds.ec2.compute.strategy;
import static com.google.common.base.Preconditions.checkNotNull;
import java.util.Iterator;
import java.util.Set;
import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Named;
@ -31,6 +34,7 @@ import org.jclouds.compute.reference.ComputeServiceConstants;
import org.jclouds.compute.strategy.DestroyNodeStrategy;
import org.jclouds.compute.strategy.GetNodeMetadataStrategy;
import org.jclouds.ec2.EC2Client;
import org.jclouds.ec2.reference.EC2Constants;
import org.jclouds.logging.Logger;
/**
@ -44,6 +48,9 @@ public class EC2DestroyNodeStrategy implements DestroyNodeStrategy {
protected Logger logger = Logger.NULL;
protected final EC2Client client;
protected final GetNodeMetadataStrategy getNode;
@Inject
@Named(EC2Constants.PROPERTY_EC2_AUTO_ALLOCATE_ELASTIC_IPS)
boolean autoAllocateElasticIps = false;
@Inject
protected EC2DestroyNodeStrategy(EC2Client client, GetNodeMetadataStrategy getNode) {
@ -56,10 +63,24 @@ public class EC2DestroyNodeStrategy implements DestroyNodeStrategy {
String[] parts = AWSUtils.parseHandle(id);
String region = parts[0];
String instanceId = parts[1];
Set<String> publicIps = getNode.getNode(id).getPublicAddresses();
releaseElasticIpInRegion(region, instanceId, publicIps);
destroyInstanceInRegion(region, instanceId);
return getNode.getNode(id);
}
protected void releaseElasticIpInRegion(String region, String instanceId, Set<String> publicIps) {
if (!autoAllocateElasticIps)
return;
Iterator<String> it = publicIps.iterator();
while (it.hasNext()) {
String publicIp = (String)it.next();
client.getElasticIPAddressServices().disassociateAddressInRegion(region, publicIp);
client.getElasticIPAddressServices().releaseAddressInRegion(region, publicIp);
}
}
protected void destroyInstanceInRegion(String region, String instanceId) {
client.getInstanceServices().terminateInstancesInRegion(region, instanceId);
}

5
apis/ec2/src/main/java/org/jclouds/ec2/reference/EC2Constants.java
View File

@ -37,4 +37,9 @@ public interface EC2Constants {
*/
public static final String PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT = "jclouds.ec2.timeout.securitygroup-present";
/**
* Whenever a node is created, automatically allocate and assign an elastic ip address, also
* deallocate when the node is destroyed.
*/
public static final String PROPERTY_EC2_AUTO_ALLOCATE_ELASTIC_IPS = "jclouds.ec2.auto-allocate-elastic-ips";
}

10
apis/ec2/src/main/java/org/jclouds/ec2/xml/DescribeSecurityGroupsResponseHandler.java
View File

@ -19,6 +19,7 @@
package org.jclouds.ec2.xml;
import static org.jclouds.util.SaxUtils.currentOrNull;
import static org.jclouds.util.SaxUtils.currentOrNegative;
import static org.jclouds.util.SaxUtils.equalsOrSuffix;
import java.util.Set;
@ -101,11 +102,14 @@ public class DescribeSecurityGroupsResponseHandler extends
} else if (equalsOrSuffix(qName, "groupDescription")) {
this.groupDescription = currentOrNull(currentText);
} else if (equalsOrSuffix(qName, "ipProtocol")) {
this.ipProtocol = IpProtocol.fromValue(currentOrNull(currentText));
// Algorete: ipProtocol can be an empty tag on EC2 clone (e.g. OpenStack EC2)
this.ipProtocol = IpProtocol.fromValue(currentOrNegative(currentText));
} else if (equalsOrSuffix(qName, "fromPort")) {
this.fromPort = Integer.parseInt(currentOrNull(currentText));
// Algorete: fromPort can be an empty tag on EC2 clone (e.g. OpenStack EC2)
this.fromPort = Integer.parseInt(currentOrNegative(currentText));
} else if (equalsOrSuffix(qName, "toPort")) {
this.toPort = Integer.parseInt(currentOrNull(currentText));
// Algorete: toPort can be an empty tag on EC2 clone (e.g. OpenStack EC2)
this.toPort = Integer.parseInt(currentOrNegative(currentText));
} else if (equalsOrSuffix(qName, "cidrIp")) {
this.ipRanges.add(currentOrNull(currentText));
} else if (equalsOrSuffix(qName, "ipPermissions")) {

70
apis/ec2/src/test/java/org/jclouds/ec2/compute/EC2ComputeServiceLiveTest.java
View File

@ -20,11 +20,17 @@ package org.jclouds.ec2.compute;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.assertFalse;
import java.io.IOException;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.jclouds.compute.BaseComputeServiceLiveTest;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.compute.ComputeServiceContextFactory;
import org.jclouds.compute.RunNodesException;
import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.compute.domain.OsFamily;
import org.jclouds.compute.domain.Template;
@ -39,20 +45,27 @@ import org.jclouds.ec2.domain.BlockDevice;
import org.jclouds.ec2.domain.InstanceType;
import org.jclouds.ec2.domain.IpProtocol;
import org.jclouds.ec2.domain.KeyPair;
import org.jclouds.ec2.domain.PublicIpInstanceIdPair;
import org.jclouds.ec2.domain.RunningInstance;
import org.jclouds.ec2.domain.SecurityGroup;
import org.jclouds.ec2.domain.Snapshot;
import org.jclouds.ec2.domain.Volume;
import org.jclouds.ec2.reference.EC2Constants;
import org.jclouds.ec2.services.ElasticBlockStoreClient;
import org.jclouds.ec2.services.InstanceClient;
import org.jclouds.ec2.services.KeyPairClient;
import org.jclouds.ec2.services.SecurityGroupClient;
import org.jclouds.http.HttpResponseException;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.net.IPSocket;
import org.jclouds.scriptbuilder.domain.Statements;
import org.jclouds.sshj.config.SshjSshClientModule;
import org.jclouds.util.InetAddresses2;
import org.testng.annotations.Test;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
@ -164,6 +177,63 @@ public class EC2ComputeServiceLiveTest extends BaseComputeServiceLiveTest {
}
}
@Test(enabled = true, dependsOnMethods = "testCompareSizes")
public void testAutoIpAllocation() throws Exception {
ComputeServiceContext context = null;
String group = this.group + "aip";
try {
Properties overrides = setupProperties();
overrides.setProperty(EC2Constants.PROPERTY_EC2_AUTO_ALLOCATE_ELASTIC_IPS, "true");
context = new ComputeServiceContextFactory().createContext(provider,
ImmutableSet.<Module> of(new Log4JLoggingModule()), overrides);
// create a node
Set<? extends NodeMetadata> nodes =
context.getComputeService().createNodesInGroup(group, 1);
assertTrue(nodes.size() == 1);
// Get public IPs (on EC2 we should get 2, on Nova only 1)
NodeMetadata node = Iterables.get(nodes, 0);
String region = node.getLocation().getParent().getId();
Set<String> publicIps = node.getPublicAddresses();
// Check that all ips are public and port 22 is accessible
for (String ip: publicIps) {
assertFalse(InetAddresses2.isPrivateIPAddress(ip));
IPSocket socket = new IPSocket(ip, 22);
assert socketTester.apply(socket) : String.format("failed to open socket %s on node %s", socket,
node);
}
// check that there is an elastic ip correlating to it
EC2Client ec2 = EC2Client.class.cast(context.getProviderSpecificContext().getApi());
Set<PublicIpInstanceIdPair> ipidpairs =
ec2.getElasticIPAddressServices().describeAddressesInRegion(region, publicIps.toArray(new String[0]));
assertTrue(ipidpairs.size() == 1);
// check that the elastic ip is in node.publicAddresses
PublicIpInstanceIdPair ipidpair = Iterables.get(ipidpairs, 0);
assertTrue(ipidpair.getInstanceId() == node.getId());
// delete the node
context.getComputeService().destroyNodesMatching(NodePredicates.inGroup(group));
// check that the ip is deallocated
try {
ec2.getElasticIPAddressServices().describeAddressesInRegion(region, ipidpair.getPublicIp());
assert false;
} catch (HttpResponseException e) {
// do nothing. .. it is expected to fail.
}
} finally {
if (context != null)
context.close();
}
}
/**
* Note we cannot use the micro size as it has no ephemeral space.
*/

25
apis/ec2/src/test/java/org/jclouds/ec2/xml/DescribeSecurityGroupsResponseHandlerTest.java
View File

@ -36,6 +36,8 @@ import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Multimap;
/**
* Tests behavior of {@code DescribeSecurityGroupsResponseHandler}
@ -65,6 +67,29 @@ public class DescribeSecurityGroupsResponseHandlerTest extends BaseEC2HandlerTes
assertEquals(result, expected);
}
// Response from OpenStack 1.1 EC2 API
public void testApplyInputStreamWithEmptyFields() {
InputStream is = getClass().getResourceAsStream("/describe_securitygroups_empty.xml");
Multimap<String, String> userIdGroupPairs = LinkedHashMultimap.create();
userIdGroupPairs.put("UYY3TLBUXIEON5NQVUUX6OMPWBZIQNFM", "jclouds#cluster#world");
Set<SecurityGroup> expected = ImmutableSet.of(
new SecurityGroup(defaultRegion, null, "jclouds#cluster#world", "UYY3TLBUXIEON5NQVUUX6OMPWBZIQNFM", "Cluster",
ImmutableSet.of(
new IpPermissionImpl(IpProtocol.TCP, 22, 22, ImmutableMultimap.<String, String> of(),
ImmutableSet.<String> of(), ImmutableSet.of("0.0.0.0/0")),
new IpPermissionImpl(IpProtocol.ALL, -1, -1, userIdGroupPairs,
ImmutableSet.<String> of(), ImmutableSet.<String> of()))));
DescribeSecurityGroupsResponseHandler handler = injector.getInstance(DescribeSecurityGroupsResponseHandler.class);
addDefaultRegionToHandler(handler);
Set<SecurityGroup> result = factory.create(handler).parse(is);
assertEquals(result, expected);
}
private void addDefaultRegionToHandler(ParseSax.HandlerWithResult<?> handler) {
GeneratedHttpRequest<?> request = createMock(GeneratedHttpRequest.class);
expect(request.getArgs()).andReturn(ImmutableList.<Object> of()).atLeastOnce();

39
apis/ec2/src/test/resources/describe_securitygroups_empty.xml Normal file
View File

@ -0,0 +1,39 @@
<?xml version="1.0" ?>
<DescribeSecurityGroupsResponse xmlns="http://ec2.amazonaws.com/doc/2010-06-15/">
<requestId>L6EFIZVPJS76T3K5-0UV</requestId>
<securityGroupInfo>
<item>
<ipPermissions>
<item>
<toPort>22</toPort>
<ipProtocol>tcp</ipProtocol>
<ipRanges>
<item>
<cidrIp>0.0.0.0/0</cidrIp>
</item>
</ipRanges>
<groups/>
<fromPort>22</fromPort>
</item>
<item>
<toPort/>
<ipProtocol/>
<ipRanges/>
<groups>
<item>
<groupName>jclouds#cluster#world</groupName>
<userId>UYY3TLBUXIEON5NQVUUX6OMPWBZIQNFM</userId>
</item>
</groups>
<fromPort/>
</item>
</ipPermissions>
<groupName>jclouds#cluster#world</groupName>
<groupDescription>Cluster</groupDescription>
<ownerId>UYY3TLBUXIEON5NQVUUX6OMPWBZIQNFM</ownerId>
</item>
</securityGroupInfo>
</DescribeSecurityGroupsResponse>

4
core/src/main/java/org/jclouds/util/SaxUtils.java
View File

@ -51,4 +51,8 @@ public class SaxUtils {
return returnVal.equals("") ? null : returnVal;
}
public static String currentOrNegative(StringBuilder currentText) {
String returnVal = currentText.toString().trim();
return returnVal.equals("") ? "-1" : returnVal;
}
}