JCLOUDS-602. Added live tests for new EBS volume fields.

While they're implemented in apis/ec2, the tests are in
providers/aws-ec2, generally, to make sure ec2-alike clones won't barf
on them. We're exercising creation of volumes, images and instances
with the new options. I also had to do some sketchy wait-and-loop'ing
in AMIAPILiveTest.testCreateAndListEBSBackedImage() due to what seems
to be a delay on new AMIs showing up in filtered DescribeImages calls,
though they'll show up instantly when you specify the image ID. Go figure.

apis/ec2/src/main/java/org/jclouds/ec2/options/RunInstancesOptions.java

@@ -144,7 +144,7 @@ public class RunInstancesOptions extends BaseEC2RequestOptions {
          if (mapping.getEbsIops() != null)
             formParameters.put(String.format("BlockDeviceMapping.%d.Ebs.Iops", i),
                     String.valueOf(mapping.getEbsIops()));
-         if (mapping.getEbsEncrypted() != null)
+         if (mapping.getEbsEncrypted() != null && mapping.getEbsEncrypted())
             formParameters.put(String.format("BlockDeviceMapping.%d.Ebs.Encrypted", i), String.valueOf(mapping.getEbsEncrypted()));
          i++;
       }

apis/ec2/src/test/java/org/jclouds/ec2/features/AMIApiLiveTest.java

@@ -33,6 +33,7 @@ import java.util.Iterator;
 import java.util.Properties;
 import java.util.Set;
 
+import org.jclouds.Constants;
 import org.jclouds.aws.AWSResponseException;
 import org.jclouds.compute.RunNodesException;
 import org.jclouds.compute.domain.Template;
@@ -45,7 +46,9 @@ import org.jclouds.ec2.domain.Image.ImageType;
 import org.jclouds.ec2.domain.RootDeviceType;
 import org.jclouds.ec2.domain.RunningInstance;
 import org.jclouds.ec2.domain.Snapshot;
+import org.jclouds.ec2.options.RegisterImageBackedByEbsOptions;
 import org.jclouds.ec2.predicates.InstanceStateRunning;
+import org.jclouds.ec2.predicates.SnapshotCompleted;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -182,7 +185,7 @@ public class AMIApiLiveTest extends BaseComputeServiceContextLiveTest {
 
       // Register a new image...
       ebsBackedImageId = client.registerUnixImageBackedByEbsInRegion(regionId, ebsBackedImageName, snapshot.getId(),
-            addNewBlockDevice("/dev/sda2", "myvirtual", 1, false, "gp2", null, false).withDescription("adrian"));
+              newBlockDeviceOption());
       imagesToDeregister.add(ebsBackedImageId);
       final Image ebsBackedImage = getOnlyElement(client.describeImagesInRegion(regionId, imageIds(ebsBackedImageId)));
       assertEquals(ebsBackedImage.getName(), ebsBackedImageName);
@@ -192,20 +195,35 @@ public class AMIApiLiveTest extends BaseComputeServiceContextLiveTest {
       assertEquals(ebsBackedImage.getDescription(), "adrian");
       assertEquals(
             ebsBackedImage.getEbsBlockDevices().entrySet(),
-            ImmutableMap.of("/dev/sda1", new Image.EbsBlockDevice(snapshot.getId(), snapshot.getVolumeSize(), true, "standard", null, false),
+            ImmutableMap.of("/dev/sda1", new Image.EbsBlockDevice(snapshot.getId(), snapshot.getVolumeSize(), true, null, null, false),
-                  "/dev/sda2", new Image.EbsBlockDevice(null, 1, false, "gp2", null, false)).entrySet());
+                  "/dev/sda2", newBlockDeviceInfo()).entrySet());
 
+      int describeCount = 0;
+      int after = 0;
+
+      // This loop is in here to deal with a lag between image creation and it showing up in filtered describeImage queries.
+      while (describeCount < 10 && after == 0) {
+         describeCount++;
+         Thread.sleep(30000);
          // List of images after - should be one larger than before
-      int after = client.describeImagesInRegionWithFilter(regionId,
+         after = client.describeImagesInRegionWithFilter(regionId,
                  ImmutableMultimap.<String, String>builder()
                          .put("name", ebsBackedImageName).build()).size();
-
+      }
       assertEquals(after, sizeBefore + 1);
    }
 
+   protected RegisterImageBackedByEbsOptions newBlockDeviceOption() {
+      return addNewBlockDevice("/dev/sda2", "myvirtual", 5, false, null, null, false).withDescription("adrian");
+   }
+
+   protected Image.EbsBlockDevice newBlockDeviceInfo() {
+      return new Image.EbsBlockDevice(null, 5, false, null, null, false);
+   }
+
    // Fires up an instance, finds its root volume ID, takes a snapshot, then
    // terminates the instance.
-   private Snapshot createSnapshot() throws RunNodesException {
+   protected Snapshot createSnapshot() throws RunNodesException {
 
       String instanceId = null;
       try {
@@ -222,6 +240,8 @@ public class AMIApiLiveTest extends BaseComputeServiceContextLiveTest {
          Snapshot snapshot = ec2Api.getElasticBlockStoreApi().get().createSnapshotInRegion(regionId,
                device.getVolumeId());
          snapshotsToDelete.add(snapshot.getId());
+         Predicate<Snapshot> snapshotted = retry(new SnapshotCompleted(ec2Api.getElasticBlockStoreApi().get()), 600, 10, SECONDS);
+         assert snapshotted.apply(snapshot);
          return snapshot;
       } finally {
          if (instanceId != null)

apis/ec2/src/test/java/org/jclouds/ec2/features/ElasticBlockStoreApiLiveTest.java

@@ -15,10 +15,11 @@
  * limitations under the License.
  */
 package org.jclouds.ec2.features;
+
 import static com.google.common.base.Preconditions.checkNotNull;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.jclouds.ec2.options.CreateVolumeOptions.Builder.fromSnapshotId;
-import static org.jclouds.ec2.options.CreateVolumeOptions.Builder.volumeType;
+import static org.jclouds.ec2.options.CreateVolumeOptions.Builder.withSize;
 import static org.jclouds.ec2.options.DescribeSnapshotsOptions.Builder.snapshotIds;
 import static org.jclouds.util.Predicates2.retry;
 import static org.testng.Assert.assertEquals;
@@ -31,6 +32,7 @@ import java.util.SortedSet;
 import com.google.common.base.Predicate;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import org.jclouds.aws.AWSResponseException;
@@ -48,20 +50,20 @@ import org.testng.annotations.Test;
 /**
  * Tests behavior of {@code ElasticBlockStoreApi}
  */
-@Test(groups = "live", singleThreaded = true, testName = "ElasticBlockStoreApiLiveTest")
+@Test(groups = "live", singleThreaded = true)
 public class ElasticBlockStoreApiLiveTest extends BaseComputeServiceContextLiveTest {
    public ElasticBlockStoreApiLiveTest() {
       provider = "ec2";
    }
 
-   private EC2Api ec2Api;
+   protected EC2Api ec2Api;
-   private ElasticBlockStoreApi client;
+   protected ElasticBlockStoreApi client;
 
-   private String defaultRegion;
+   protected String defaultRegion;
-   private String defaultZone;
+   protected String defaultZone;
 
-   private String volumeId;
+   protected String volumeId;
-   private Snapshot snapshot;
+   protected Snapshot snapshot;
 
    @Override
    @BeforeClass(groups = { "integration", "live" })
@@ -118,10 +120,9 @@ public class ElasticBlockStoreApiLiveTest extends BaseComputeServiceContextLiveT
    @Test
    void testCreateVolumeInAvailabilityZone() {
       Volume expected = client.createVolumeInAvailabilityZone(defaultZone,
-              volumeType("gp2"));
+              withSize(1));
       assertNotNull(expected);
       assertEquals(expected.getAvailabilityZone(), defaultZone);
-      assertEquals(expected.getVolumeType(), "gp2");
       this.volumeId = expected.getId();
 
       Set<Volume> result = Sets.newLinkedHashSet(client.describeVolumesInRegion(defaultRegion, expected.getId()));
@@ -159,10 +160,10 @@ public class ElasticBlockStoreApiLiveTest extends BaseComputeServiceContextLiveT
       assertEquals(volume.getAvailabilityZone(), defaultZone);
       assertEquals(result.getStatus(), Volume.Status.AVAILABLE);
 
-      client.deleteVolumeInRegion(snapshot.getRegion(), volume.getId());
+      client.deleteVolumeInRegion(snapshot.getRegion(), result.getId());
    }
 
-   @Test(dependsOnMethods = "testCreateSnapshotInRegion")
+   @Test(dependsOnMethods = "testCreateVolumeFromSnapshotInAvailabilityZone")
    void testCreateVolumeFromSnapshotInAvailabilityZoneWithOptions() {
       Volume volume = client.createVolumeInAvailabilityZone(defaultZone,
               fromSnapshotId(snapshot.getId()));
@@ -177,10 +178,10 @@ public class ElasticBlockStoreApiLiveTest extends BaseComputeServiceContextLiveT
       assertEquals(volume.getAvailabilityZone(), defaultZone);
       assertEquals(result.getStatus(), Volume.Status.AVAILABLE);
 
-      client.deleteVolumeInRegion(snapshot.getRegion(), volume.getId());
+      client.deleteVolumeInRegion(snapshot.getRegion(), result.getId());
    }
 
-   @Test(dependsOnMethods = "testCreateSnapshotInRegion")
+   @Test(dependsOnMethods = "testCreateVolumeFromSnapshotInAvailabilityZoneWithOptions")
    void testCreateVolumeFromSnapshotInAvailabilityZoneWithSize() {
       Volume volume = client.createVolumeFromSnapshotInAvailabilityZone(defaultZone, 2, snapshot.getId());
       assertNotNull(volume);
@@ -195,7 +196,7 @@ public class ElasticBlockStoreApiLiveTest extends BaseComputeServiceContextLiveT
       assertEquals(volume.getSize(), 2);
       assertEquals(result.getStatus(), Volume.Status.AVAILABLE);
 
-      client.deleteVolumeInRegion(snapshot.getRegion(), volume.getId());
+      client.deleteVolumeInRegion(snapshot.getRegion(), result.getId());
    }
 
    @Test
@@ -274,21 +275,21 @@ public class ElasticBlockStoreApiLiveTest extends BaseComputeServiceContextLiveT
       // snapshotId);
    }
 
-   @Test(dependsOnMethods = "testCreateSnapshotInRegion")
+   @Test(dependsOnMethods = "testCreateVolumeFromSnapshotInAvailabilityZoneWithSize")
    public void testGetCreateVolumePermissionForSnapshot() {
       client.getCreateVolumePermissionForSnapshotInRegion(snapshot.getRegion(), snapshot.getId());
    }
 
-   @Test(dependsOnMethods = "testCreateSnapshotInRegion")
+   @Test(dependsOnMethods = "testGetCreateVolumePermissionForSnapshot")
    void testDeleteVolumeInRegion() {
       client.deleteVolumeInRegion(defaultRegion, volumeId);
-      Set<Volume> result = Sets.newLinkedHashSet(client.describeVolumesInRegion(defaultRegion, volumeId));
+      assertEquals(client.describeVolumesInRegionWithFilter(defaultRegion,
-      assertEquals(result.size(), 1);
+                      ImmutableMultimap.<String, String>builder()
-      Volume volume = result.iterator().next();
+                              .put("volume-id", volumeId).build()),
-      assertEquals(volume.getStatus(), Volume.Status.DELETING);
+              ImmutableSet.of());
    }
 
-   @Test(dependsOnMethods = "testGetCreateVolumePermissionForSnapshot")
+   @Test(dependsOnMethods = "testDeleteVolumeInRegion")
    void testDeleteSnapshotInRegion() {
       client.deleteSnapshotInRegion(snapshot.getRegion(), snapshot.getId());
       assert client.describeSnapshotsInRegion(snapshot.getRegion(), snapshotIds(snapshot.getId())).size() == 0;

providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/features/AWSAMIApiLiveTest.java

@@ -16,18 +16,38 @@
  */
 package org.jclouds.aws.ec2.features;
 
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.getOnlyElement;
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.jclouds.aws.ec2.options.AWSDescribeImagesOptions.Builder.filters;
+import static org.jclouds.ec2.options.RegisterImageBackedByEbsOptions.Builder.addNewBlockDevice;
+import static org.jclouds.ec2.options.RunInstancesOptions.Builder.withBlockDeviceMappings;
+import static org.jclouds.util.Predicates2.retry;
+import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
 
+import java.util.Properties;
 import java.util.Set;
 
-import org.jclouds.aws.domain.Region;
+import com.google.common.base.Predicate;
-import org.jclouds.ec2.domain.Image;
-import org.jclouds.ec2.features.AMIApiLiveTest;
-import org.testng.annotations.Test;
-
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import org.jclouds.Constants;
+import org.jclouds.aws.domain.Region;
+import org.jclouds.compute.RunNodesException;
+import org.jclouds.ec2.domain.BlockDevice;
+import org.jclouds.ec2.domain.BlockDeviceMapping;
+import org.jclouds.ec2.domain.Image;
+import org.jclouds.ec2.domain.RunningInstance;
+import org.jclouds.ec2.domain.Snapshot;
+import org.jclouds.ec2.domain.Volume;
+import org.jclouds.ec2.features.AMIApiLiveTest;
+import org.jclouds.ec2.options.RegisterImageBackedByEbsOptions;
+import org.jclouds.ec2.options.RunInstancesOptions;
+import org.jclouds.ec2.predicates.SnapshotCompleted;
+import org.testng.annotations.Test;
 
 /**
  * Tests behavior of {@code AMIApi}
@@ -39,6 +59,13 @@ public class AWSAMIApiLiveTest extends AMIApiLiveTest {
       provider = "aws-ec2";
    }
 
+   @Override
+   protected Properties setupProperties() {
+      Properties overrides = super.setupProperties();
+      overrides.put(Constants.PROPERTY_API_VERSION, "2014-05-01");
+      return overrides;
+   }
+
    public void testDescribeImagesCC() {
       Set<? extends Image> ccResults = client.describeImagesInRegion(Region.US_EAST_1,
             filters(ImmutableMultimap.<String, String> builder()//
@@ -53,4 +80,50 @@ public class AWSAMIApiLiveTest extends AMIApiLiveTest {
       assertNotNull(ccResults);
       assert ccResults.size() >= 34 : ccResults;
    }
+
+   protected RegisterImageBackedByEbsOptions newBlockDeviceOption() {
+      return addNewBlockDevice("/dev/sda2", "myvirtual", 5, false, "gp2", null, false).withDescription("adrian");
+   }
+
+   protected Image.EbsBlockDevice newBlockDeviceInfo() {
+      return new Image.EbsBlockDevice(null, 5, false, "gp2", null, false);
+   }
+
+
+   @Override
+   protected Snapshot createSnapshot() throws RunNodesException {
+
+      String instanceId = null;
+      try {
+         BlockDeviceMapping mapping = new BlockDeviceMapping.MapNewVolumeToDevice("/dev/sdb", 1, true, "gp2", null, false);
+         RunInstancesOptions options = withBlockDeviceMappings(ImmutableSet
+                 .<BlockDeviceMapping> of(mapping));
+
+         RunningInstance instance = getOnlyElement(concat(ec2Api.getInstanceApi().get().runInstancesInRegion(
+                 regionId, null, imageId, 1, 1, options)));
+         instanceId = instance.getId();
+
+         assertTrue(runningTester.apply(instance), instanceId + "didn't achieve the state running!");
+
+         instance = getOnlyElement(concat(ec2Api.getInstanceApi().get().describeInstancesInRegion(regionId,
+                 instanceId)));
+         BlockDevice gp2Device = instance.getEbsBlockDevices().get("/dev/sdb");
+         assertNotNull(gp2Device, "device /dev/sdb not present on " + instance);
+         Volume gp2Volume = Iterables.getOnlyElement(ec2Api.getElasticBlockStoreApi().get().describeVolumesInRegion(regionId, gp2Device.getVolumeId()));
+         assertNotNull(gp2Volume, "/dev/sdb volume is null");
+         assertEquals(gp2Volume.getVolumeType(), "gp2");
+         BlockDevice device = instance.getEbsBlockDevices().get("/dev/sda1");
+         assertNotNull(device, "device: /dev/sda1 not present on: " + instance);
+         Snapshot snapshot = ec2Api.getElasticBlockStoreApi().get().createSnapshotInRegion(regionId,
+                 device.getVolumeId());
+         snapshotsToDelete.add(snapshot.getId());
+         Predicate<Snapshot> snapshotted = retry(new SnapshotCompleted(ec2Api.getElasticBlockStoreApi().get()), 600, 10, SECONDS);
+         assert snapshotted.apply(snapshot);
+         return snapshot;
+      } finally {
+         if (instanceId != null)
+            ec2Api.getInstanceApi().get().terminateInstancesInRegion(regionId, instanceId);
+      }
+   }
+
 }

providers/aws-ec2/src/test/java/org/jclouds/aws/ec2/features/AWSElasticBlockStoreApiLiveTest.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.aws.ec2.features;
+
+import static org.jclouds.ec2.options.CreateVolumeOptions.Builder.volumeType;
+import static org.jclouds.ec2.options.CreateVolumeOptions.Builder.withSize;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+import java.util.Properties;
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+import org.jclouds.Constants;
+import org.jclouds.ec2.domain.Volume;
+import org.jclouds.ec2.features.ElasticBlockStoreApiLiveTest;
+import org.testng.annotations.Test;
+
+/**
+ * Tests behavior of {@code ElasticBlockStoreApi}
+ */
+@Test(groups = "live", singleThreaded = true)
+public class AWSElasticBlockStoreApiLiveTest extends ElasticBlockStoreApiLiveTest {
+
+   public AWSElasticBlockStoreApiLiveTest() {
+      provider = "aws-ec2";
+   }
+
+   @Override
+   protected Properties setupProperties() {
+      Properties overrides = super.setupProperties();
+      overrides.put(Constants.PROPERTY_API_VERSION, "2014-05-01");
+      return overrides;
+   }
+
+   @Test
+   void testCreateVolumeInAvailabilityZoneWithVolumeType() {
+      Volume expected = client.createVolumeInAvailabilityZone(defaultZone,
+              volumeType("gp2"), withSize(1));
+      assertNotNull(expected);
+      assertEquals(expected.getAvailabilityZone(), defaultZone);
+      assertEquals(expected.getVolumeType(), "gp2");
+
+      Set<Volume> result = Sets.newLinkedHashSet(client.describeVolumesInRegion(defaultRegion, expected.getId()));
+      assertNotNull(result);
+      assertEquals(result.size(), 1);
+      Volume volume = result.iterator().next();
+      assertEquals(volume.getId(), expected.getId());
+      assertEquals(volume.getVolumeType(), expected.getVolumeType());
+
+      client.deleteVolumeInRegion(volume.getRegion(), volume.getId());
+   }
+
+}