Issue 255, 256: lazy parsing of images so you can specify imageId() that isn't in the default owner list

aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java

@@ -22,12 +22,16 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static org.jclouds.aws.ec2.options.DescribeImagesOptions.Builder.ownedBy;
 import static org.jclouds.aws.ec2.reference.EC2Constants.PROPERTY_EC2_AMI_OWNERS;
 import static org.jclouds.compute.domain.OsFamily.UBUNTU;
+import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion;
 
 import java.net.URI;
 import java.security.SecureRandom;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
@@ -36,7 +40,7 @@ import javax.inject.Inject;
 import javax.inject.Named;
 import javax.inject.Singleton;
 
-import org.jclouds.aws.domain.Region;
+import org.jclouds.Constants;
 import org.jclouds.aws.ec2.EC2;
 import org.jclouds.aws.ec2.EC2AsyncClient;
 import org.jclouds.aws.ec2.EC2Client;
@@ -46,13 +50,16 @@ import org.jclouds.aws.ec2.compute.domain.RegionAndName;
 import org.jclouds.aws.ec2.compute.functions.CreateSecurityGroupIfNeeded;
 import org.jclouds.aws.ec2.compute.functions.CreateUniqueKeyPair;
 import org.jclouds.aws.ec2.compute.functions.ImageParser;
+import org.jclouds.aws.ec2.compute.functions.RegionAndIdToImage;
 import org.jclouds.aws.ec2.compute.functions.RunningInstanceToNodeMetadata;
+import org.jclouds.aws.ec2.compute.internal.EC2TemplateBuilderImpl;
 import org.jclouds.aws.ec2.compute.options.EC2TemplateOptions;
 import org.jclouds.aws.ec2.compute.strategy.EC2DestroyNodeStrategy;
 import org.jclouds.aws.ec2.compute.strategy.EC2RunNodesAndAddToSetStrategy;
 import org.jclouds.aws.ec2.config.EC2ContextModule;
 import org.jclouds.aws.ec2.domain.KeyPair;
 import org.jclouds.aws.ec2.domain.RunningInstance;
+import org.jclouds.aws.ec2.domain.Image.ImageType;
 import org.jclouds.aws.ec2.functions.RunningInstanceToStorageMappingUnix;
 import org.jclouds.aws.ec2.services.InstanceClient;
 import org.jclouds.compute.ComputeService;
@@ -74,6 +81,7 @@ import org.jclouds.compute.strategy.GetNodeMetadataStrategy;
 import org.jclouds.compute.strategy.ListNodesStrategy;
 import org.jclouds.compute.strategy.RebootNodeStrategy;
 import org.jclouds.compute.strategy.RunNodesAndAddToSetStrategy;
+import org.jclouds.concurrent.ConcurrentUtils;
 import org.jclouds.domain.Location;
 import org.jclouds.domain.LocationScope;
 import org.jclouds.domain.internal.LocationImpl;
@@ -89,8 +97,10 @@ import com.google.common.base.Splitter;
 import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.MapMaker;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ListenableFuture;
 import com.google.inject.Provides;
 import com.google.inject.Scopes;
 import com.google.inject.TypeLiteral;
@@ -111,6 +121,7 @@ public class EC2ComputeServiceContextModule extends EC2ContextModule {
    @Override
    protected void configure() {
       super.configure();
+      bind(TemplateBuilder.class).to(EC2TemplateBuilderImpl.class);
       bind(TemplateOptions.class).to(EC2TemplateOptions.class);
       bind(ComputeService.class).to(EC2ComputeService.class);
       bind(RunNodesAndAddToSetStrategy.class).to(EC2RunNodesAndAddToSetStrategy.class);
@@ -147,14 +158,23 @@ public class EC2ComputeServiceContextModule extends EC2ContextModule {
    // due to image parsing; consider using MapMaker. computing map
    @Singleton
    public static class EC2ListNodesStrategy implements ListNodesStrategy {
+      @Resource
+      @Named(ComputeServiceConstants.COMPUTE_LOGGER)
+      protected Logger logger = Logger.NULL;
+      
       private final InstanceClient client;
+      private final Map<String, URI> regionMap;
       private final RunningInstanceToNodeMetadata runningInstanceToNodeMetadata;
+      private final ExecutorService executor;
 
       @Inject
-      protected EC2ListNodesStrategy(InstanceClient client,
+      protected EC2ListNodesStrategy(InstanceClient client, @EC2 Map<String, URI> regionMap,
-               RunningInstanceToNodeMetadata runningInstanceToNodeMetadata) {
+               RunningInstanceToNodeMetadata runningInstanceToNodeMetadata,
+               @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) {
          this.client = client;
+         this.regionMap = regionMap;
          this.runningInstanceToNodeMetadata = runningInstanceToNodeMetadata;
+         this.executor = executor;
       }
 
       @Override
@@ -165,11 +185,28 @@ public class EC2ComputeServiceContextModule extends EC2ContextModule {
       @Override
       public Iterable<? extends NodeMetadata> listDetailsOnNodesMatching(
                Predicate<ComputeMetadata> filter) {
-         Set<NodeMetadata> nodes = Sets.newHashSet();
+         final Set<NodeMetadata> nodes = Sets.newHashSet();
-         for (String region : ImmutableSet.of(Region.US_EAST_1, Region.US_WEST_1, Region.EU_WEST_1)) {
+
-            Iterables.addAll(nodes, Iterables.transform(Iterables.concat(client
+         Map<String, ListenableFuture<?>> parallelResponses = Maps.newHashMap();
-                     .describeInstancesInRegion(region)), runningInstanceToNodeMetadata));
+
+         for (final String region : regionMap.keySet()) {
+            parallelResponses.put(region, ConcurrentUtils.makeListenable(executor
+                     .submit(new Callable<Void>() {
+                        @Override
+                        public Void call() throws Exception {
+                           Iterables.addAll(nodes, Iterables.transform(Iterables.concat(client
+                                    .describeInstancesInRegion(region)),
+                                    runningInstanceToNodeMetadata));
+                           return null;
+                        }
+                     }), executor));
          }
+         Map<String, Exception> exceptions = awaitCompletion(parallelResponses, executor, null,
+                  logger, "nodes");
+
+         if (exceptions.size() > 0)
+            throw new RuntimeException(String.format("error parsing nodes in regions: %s",
+                     exceptions));
          return Iterables.filter(nodes, filter);
       }
    }
@@ -322,30 +359,65 @@ public class EC2ComputeServiceContextModule extends EC2ContextModule {
    @Singleton
    @Named(PROPERTY_EC2_AMI_OWNERS)
    String[] amiOwners(@Named(PROPERTY_EC2_AMI_OWNERS) String amiOwners) {
+      if (amiOwners.trim().equals(""))
+         return new String[] {};
       return Iterables.toArray(Splitter.on(',').split(amiOwners), String.class);
    }
 
    @Provides
-   @Singleton
+   protected Set<? extends Image> provideImages(Map<RegionAndName, ? extends Image> map) {
-   protected Set<? extends Image> provideImages(final EC2Client sync,
+      return ImmutableSet.copyOf(map.values());
-            @EC2 Map<String, URI> regionMap, LogHolder holder,
+   }
-            Function<ComputeMetadata, String> indexer,
-            @Named(PROPERTY_EC2_AMI_OWNERS) String[] amiOwners, ImageParser parser)
-            throws InterruptedException, ExecutionException, TimeoutException {
-      final Set<Image> images = Sets.newHashSet();
-      holder.logger.debug(">> providing images");
 
-      for (final String region : regionMap.keySet()) {
+   @Provides
-         for (final org.jclouds.aws.ec2.domain.Image from : sync.getAMIServices()
+   @Singleton
-                  .describeImagesInRegion(region, ownedBy(amiOwners))) {
+   protected ConcurrentMap<RegionAndName, Image> provideImageMap(
-            Image image = parser.apply(from);
+            RegionAndIdToImage regionAndIdToImage) {
-            if (image != null)
+      return new MapMaker().makeComputingMap(regionAndIdToImage);
-               images.add(image);
+   }
-            else
+
-               holder.logger.trace("<< image(%s) didn't parse", from.getId());
+   @Provides
+   @Singleton
+   protected Map<RegionAndName, ? extends Image> provideImages(final EC2Client sync,
+            @EC2 Map<String, URI> regionMap, final LogHolder holder,
+            Function<ComputeMetadata, String> indexer,
+            @Named(PROPERTY_EC2_AMI_OWNERS) final String[] amiOwners, final ImageParser parser,
+            final ConcurrentMap<RegionAndName, Image> images,
+            @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor)
+            throws InterruptedException, ExecutionException, TimeoutException {
+      if (amiOwners.length == 0) {
+         holder.logger.debug(">> no owners specified, skipping image parsing");
+      } else {
+         holder.logger.debug(">> providing images");
+
+         Map<String, ListenableFuture<?>> parallelResponses = Maps.newHashMap();
+
+         for (final String region : regionMap.keySet()) {
+            parallelResponses.put(region, ConcurrentUtils.makeListenable(executor
+                     .submit(new Callable<Void>() {
+                        @Override
+                        public Void call() throws Exception {
+                           for (final org.jclouds.aws.ec2.domain.Image from : sync.getAMIServices()
+                                    .describeImagesInRegion(region, ownedBy(amiOwners))) {
+                              Image image = parser.apply(from);
+                              if (image != null)
+                                 images.put(new RegionAndName(region, image.getId()), image);
+                              else if (from.getImageType() == ImageType.MACHINE)
+                                 holder.logger.trace("<< image(%s) didn't parse", from.getId());
+                           }
+                           return null;
+                        }
+                     }), executor));
          }
+         Map<String, Exception> exceptions = awaitCompletion(parallelResponses, executor, null,
+                  holder.logger, "images");
+
+         if (exceptions.size() > 0)
+            throw new RuntimeException(String.format("error parsing images in regions: %s",
+                     exceptions));
+
+         holder.logger.debug("<< images(%d)", images.size());
       }
-      holder.logger.debug("<< images(%d)", images.size());
       return images;
    }
 }

aws/core/src/main/java/org/jclouds/aws/ec2/compute/functions/ImageParser.java

@@ -61,6 +61,14 @@ public class ImageParser implements Function<org.jclouds.aws.ec2.domain.Image, I
    public static final Pattern CANONICAL_PATTERN = Pattern
             .compile(".*/([^-]*)-([^-]*)-.*-(.*)(\\.manifest.xml)?");
 
+   // ex rightscale-us-east/CentOS_5.4_x64_v4.4.10.manifest.xml
+   public static final Pattern RIGHTSCALE_PATTERN = Pattern
+            .compile("[^/]*/([^_]*)_([^_]*)_[^vV]*[vV](.*)(\\.manifest.xml)?");
+
+   // ex 411009282317/RightImage_Ubuntu_9.10_x64_v4.5.3_EBS_Alpha
+   public static final Pattern RIGHTIMAGE_PATTERN = Pattern
+            .compile("[^/]*/RightImage_([^_]*)_([^_]*)_[^vV]*[vV](.*)(\\.manifest.xml)?");
+
    public static final Map<String, String> NAME_VERSION_MAP = ImmutableMap
             .<String, String> builder().put("hardy", "8.04").put("intrepid", "8.10").put("jaunty",
                      "9.04").put("karmic", "9.10").put("lucid", "10.04").put("maverick", "10.10")
@@ -96,19 +104,21 @@ public class ImageParser implements Function<org.jclouds.aws.ec2.domain.Image, I
                .getImageLocation();
       String osDescription = from.getImageLocation();
       String version = "";
-      Matcher matcher = CANONICAL_PATTERN.matcher(from.getImageLocation());
+
-      if (matcher.find()) {
+      try {
-         try {
+         Matcher matcher = getMatcherAndFind(from.getImageLocation());
-            os = OsFamily.fromValue(matcher.group(1));
+         os = OsFamily.fromValue(matcher.group(1));
-            name = matcher.group(2);// TODO no field for os version
+         name = matcher.group(2);// TODO no field for os version
-            // normalize versions across ubuntu from alestic and canonical
+         // normalize versions across ubuntu from alestic and canonical
-            if (NAME_VERSION_MAP.containsKey(name))
+         if (NAME_VERSION_MAP.containsKey(name))
-               name = NAME_VERSION_MAP.get(name);
+            name = NAME_VERSION_MAP.get(name);
-            version = matcher.group(3).replace(".manifest.xml", "");
+         version = matcher.group(3).replace(".manifest.xml", "");
-         } catch (IllegalArgumentException e) {
+      } catch (IllegalArgumentException e) {
-            logger.debug("<< didn't match os(%s)", matcher.group(1));
+         logger.debug("<< didn't match os(%s)", from.getImageLocation());
-         }
+      } catch (NoSuchElementException e) {
+         logger.debug("<< didn't match at all(%s)", from.getImageLocation());
       }
+
       Credentials defaultCredentials = credentialProvider.execute(from);
 
       Location location = null;
@@ -141,4 +151,19 @@ public class ImageParser implements Function<org.jclouds.aws.ec2.domain.Image, I
                         : Architecture.X86_64, defaultCredentials);
 
    }
+
+   /**
+    * 
+    * @throws NoSuchElementException
+    *            if no configured matcher matches the manifest.
+    */
+   private Matcher getMatcherAndFind(String manifest) {
+      for (Pattern pattern : new Pattern[] { CANONICAL_PATTERN, RIGHTIMAGE_PATTERN,
+               RIGHTSCALE_PATTERN }) {
+         Matcher matcher = pattern.matcher(manifest);
+         if (matcher.find())
+            return matcher;
+      }
+      throw new NoSuchElementException(manifest);
+   }
 }

aws/core/src/main/java/org/jclouds/aws/ec2/compute/functions/RegionAndIdToImage.java

@@ -0,0 +1,65 @@
+/**
+ *
+ * Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.aws.ec2.compute.functions;
+
+import static org.jclouds.aws.ec2.options.DescribeImagesOptions.Builder.imageIds;
+
+import javax.annotation.Resource;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.jclouds.aws.ec2.compute.domain.RegionAndName;
+import org.jclouds.aws.ec2.services.AMIClient;
+import org.jclouds.compute.domain.Image;
+import org.jclouds.logging.Logger;
+import org.jclouds.rest.ResourceNotFoundException;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Singleton
+public final class RegionAndIdToImage implements Function<RegionAndName, Image> {
+   @Resource
+   protected Logger logger = Logger.NULL;
+
+   private final ImageParser parser;
+   private final AMIClient sync;
+
+   @Inject
+   public RegionAndIdToImage(ImageParser parser, AMIClient sync) {
+      this.parser = parser;
+      this.sync = sync;
+   }
+
+   public Image apply(RegionAndName key) {
+      try {
+         org.jclouds.aws.ec2.domain.Image image = Iterables.getOnlyElement(sync
+                  .describeImagesInRegion(key.getRegion(), imageIds(key.getName())));
+         return parser.apply(image);
+      } catch (ResourceNotFoundException e) {
+         logger.warn(e, "no image found for %s/%s: %s", key.getRegion(), key.getName(), e
+                  .getMessage());
+         return null;
+      }
+   }
+}

aws/core/src/main/java/org/jclouds/aws/ec2/compute/functions/RunningInstanceToNodeMetadata.java

@@ -26,10 +26,12 @@ import java.net.URI;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
 
 import javax.annotation.Resource;
 import javax.inject.Inject;
 import javax.inject.Named;
+import javax.inject.Provider;
 import javax.inject.Singleton;
 
 import org.jclouds.aws.ec2.compute.domain.RegionAndName;
@@ -63,13 +65,14 @@ public class RunningInstanceToNodeMetadata implements Function<RunningInstance,
    @Resource
    protected Logger logger = Logger.NULL;
 
-   private static class FindImageForInstance implements Predicate<Image> {
+   @VisibleForTesting
+   static class FindImageForInstance implements Predicate<Image> {
       private final Location location;
       private final RunningInstance instance;
 
-      private FindImageForInstance(Location location, RunningInstance instance) {
+      FindImageForInstance(Location location, RunningInstance instance) {
-         this.location = location;
+         this.location = checkNotNull(location, "location");
-         this.instance = instance;
+         this.instance = checkNotNull(instance, "instance");
       }
 
       @Override
@@ -78,6 +81,37 @@ public class RunningInstanceToNodeMetadata implements Function<RunningInstance,
                   && (input.getLocation() == null || input.getLocation().equals(location) || input
                            .getLocation().equals(location.getParent()));
       }
+
+      @Override
+      public int hashCode() {
+         final int prime = 31;
+         int result = 1;
+         result = prime * result + ((instance.getId() == null) ? 0 : instance.getId().hashCode());
+         result = prime * result + ((location.getId() == null) ? 0 : location.getId().hashCode());
+         return result;
+      }
+
+      @Override
+      public boolean equals(Object obj) {
+         if (this == obj)
+            return true;
+         if (obj == null)
+            return false;
+         if (getClass() != obj.getClass())
+            return false;
+         FindImageForInstance other = (FindImageForInstance) obj;
+         if (instance.getId() == null) {
+            if (other.instance.getId() != null)
+               return false;
+         } else if (!instance.getId().equals(other.instance.getId()))
+            return false;
+         if (location.getId() == null) {
+            if (other.location.getId() != null)
+               return false;
+         } else if (!location.getId().equals(other.location.getId()))
+            return false;
+         return true;
+      }
    }
 
    private static final Map<InstanceState, NodeState> instanceToNodeState = ImmutableMap
@@ -88,16 +122,18 @@ public class RunningInstanceToNodeMetadata implements Function<RunningInstance,
    private final AMIClient amiClient;
    private final Map<RegionAndName, KeyPair> credentialsMap;
    private final PopulateDefaultLoginCredentialsForImageStrategy credentialProvider;
-   private final Set<? extends Image> images;
+   private final Provider<Set<? extends Image>> images;
    private final Set<? extends Location> locations;
    private final Function<RunningInstance, Map<String, String>> instanceToStorageMapping;
+   private final ConcurrentMap<RegionAndName, Image> imageMap;
 
    @Inject
    RunningInstanceToNodeMetadata(
             AMIClient amiClient,
             Map<RegionAndName, KeyPair> credentialsMap,
             PopulateDefaultLoginCredentialsForImageStrategy credentialProvider,
-            Set<? extends Image> images,
+            Provider<Set<? extends Image>> images, // to facilitate on-demand refresh of image list
+            ConcurrentMap<RegionAndName, Image> imageMap,
             Set<? extends Location> locations,
             @Named("volumeMapping") Function<RunningInstance, Map<String, String>> instanceToStorageMapping) {
       this.amiClient = checkNotNull(amiClient, "amiClient");
@@ -105,15 +141,50 @@ public class RunningInstanceToNodeMetadata implements Function<RunningInstance,
       this.credentialProvider = checkNotNull(credentialProvider, "credentialProvider");
       this.images = checkNotNull(images, "images");
       this.locations = checkNotNull(locations, "locations");
-      this.instanceToStorageMapping = checkNotNull(instanceToStorageMapping);
+      this.instanceToStorageMapping = checkNotNull(instanceToStorageMapping,
+               "instanceToStorageMapping");
+      this.imageMap = checkNotNull(imageMap, "imageMap");
    }
 
    @Override
    public NodeMetadata apply(final RunningInstance instance) {
       String id = checkNotNull(instance, "instance").getId();
+
       String name = null; // user doesn't determine a node name;
       URI uri = null; // no uri to get rest access to host info
 
+      String tag = getTagForInstace(instance);
+
+      Credentials credentials = getCredentialsForInstanceWithTag(instance, tag);
+
+      Map<String, String> userMetadata = ImmutableMap.<String, String> of();
+
+      NodeState state = instanceToNodeState.get(instance.getInstanceState());
+
+      Set<InetAddress> publicAddresses = nullSafeSet(instance.getIpAddress());
+      Set<InetAddress> privateAddresses = nullSafeSet(instance.getPrivateIpAddress());
+
+      Map<String, String> extra = getExtra(instance);
+
+      Location location = getLocationForAvailabilityZone(instance);
+
+      Image image = resolveImageForInstanceInLocation(instance, location);
+
+      return new NodeMetadataImpl(id, name, location, uri, userMetadata, tag, image, state,
+               publicAddresses, privateAddresses, extra, credentials);
+   }
+
+   private Credentials getCredentialsForInstanceWithTag(final RunningInstance instance, String tag) {
+      Credentials credentials = null;// default if no keypair exists
+
+      if (instance.getKeyName() != null) {
+         credentials = new Credentials(getLoginAccountFor(instance), getPrivateKeyOrNull(instance,
+                  tag));
+      }
+      return credentials;
+   }
+
+   private String getTagForInstace(final RunningInstance instance) {
       String tag = String.format("NOTAG-%s", instance.getId());// default
       try {
          tag = Iterables.getOnlyElement(
@@ -133,26 +204,13 @@ public class RunningInstanceToNodeMetadata implements Function<RunningInstance,
          logger.warn("too many groups match %s; %s's groups: %s", "jclouds#", instance.getId(),
                   instance.getGroupIds());
       }
+      return tag;
+   }
 
-      Credentials credentials = null;// default if no keypair exists
+   private Location getLocationForAvailabilityZone(final RunningInstance instance) {
-
-      if (instance.getKeyName() != null) {
-         credentials = new Credentials(getLoginAccountFor(instance), getPrivateKeyOrNull(instance,
-                  tag));
-      }
-
-      Map<String, String> userMetadata = ImmutableMap.<String, String> of();
-
-      NodeState state = instanceToNodeState.get(instance.getInstanceState());
-
-      Set<InetAddress> publicAddresses = nullSafeSet(instance.getIpAddress());
-      Set<InetAddress> privateAddresses = nullSafeSet(instance.getPrivateIpAddress());
-
       final String locationId = instance.getAvailabilityZone();
 
-      Map<String, String> extra = getExtra(instance);
+      Location location = Iterables.find(locations, new Predicate<Location>() {
-
-      final Location location = Iterables.find(locations, new Predicate<Location>() {
 
          @Override
          public boolean apply(Location input) {
@@ -160,16 +218,24 @@ public class RunningInstanceToNodeMetadata implements Function<RunningInstance,
          }
 
       });
+      return location;
+   }
 
+   @VisibleForTesting
+   Image resolveImageForInstanceInLocation(final RunningInstance instance, final Location location) {
       Image image = null;
       try {
-         image = Iterables.find(images, new FindImageForInstance(location, instance));
+         image = Iterables.find(images.get(), new FindImageForInstance(location, instance));
       } catch (NoSuchElementException e) {
-         logger.warn("could not find a matching image for instance %s in location %s", instance,
+         RegionAndName key = new RegionAndName(instance.getRegion(), instance.getImageId());
-                  location);
+         try {
+            image = imageMap.get(key);
+         } catch (NullPointerException nex) {
+            logger.warn("could not find a matching image for instance %s in location %s", instance,
+                     location);
+         }
       }
-      return new NodeMetadataImpl(id, name, location, uri, userMetadata, tag, image, state,
+      return image;
-               publicAddresses, privateAddresses, extra, credentials);
    }
 
    /**

aws/core/src/main/java/org/jclouds/aws/ec2/compute/internal/EC2TemplateBuilderImpl.java

@@ -0,0 +1,56 @@
+package org.jclouds.aws.ec2.compute.internal;
+
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Provider;
+
+import org.jclouds.aws.ec2.compute.domain.RegionAndName;
+import org.jclouds.compute.domain.Image;
+import org.jclouds.compute.domain.Size;
+import org.jclouds.compute.domain.TemplateBuilder;
+import org.jclouds.compute.internal.TemplateBuilderImpl;
+import org.jclouds.compute.options.TemplateOptions;
+import org.jclouds.domain.Location;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+public class EC2TemplateBuilderImpl extends TemplateBuilderImpl {
+
+   private final ConcurrentMap<RegionAndName, Image> imageMap;
+
+   @Inject
+   protected EC2TemplateBuilderImpl(Set<? extends Location> locations, Set<? extends Image> images,
+            Set<? extends Size> sizes, Location defaultLocation,
+            Provider<TemplateOptions> optionsProvider,
+            @Named("DEFAULT") Provider<TemplateBuilder> defaultTemplateProvider,
+            ConcurrentMap<RegionAndName, Image> imageMap) {
+      super(locations, images, sizes, defaultLocation, optionsProvider, defaultTemplateProvider);
+      this.imageMap = imageMap;
+   }
+
+   /**
+    * @throws NoSuchElementException
+    *            if the image is not found
+    */
+   @Override
+   protected Image resolveImage() {
+      try {
+         return super.resolveImage();
+      } catch (NoSuchElementException e) {
+         RegionAndName key = new RegionAndName(this.locationId, this.imageId);
+         try {
+            return imageMap.get(key);
+         } catch (NullPointerException nex) {
+            throw new NoSuchElementException(String.format("image %s/%s not found",
+                     key.getRegion(), key.getName()));
+         }
+      }
+   }
+
+}

aws/core/src/main/java/org/jclouds/aws/handlers/ParseAWSErrorFromXmlContent.java

@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import javax.annotation.Resource;
 import javax.inject.Inject;
+import javax.inject.Singleton;
 
 import org.jclouds.aws.AWSResponseException;
 import org.jclouds.aws.domain.AWSError;
@@ -34,6 +35,7 @@ import org.jclouds.http.HttpResponse;
 import org.jclouds.http.HttpResponseException;
 import org.jclouds.logging.Logger;
 import org.jclouds.rest.AuthorizationException;
+import org.jclouds.rest.ResourceNotFoundException;
 import org.jclouds.util.Utils;
 
 import com.google.common.io.Closeables;
@@ -45,6 +47,7 @@ import com.google.common.io.Closeables;
  * @author Adrian Cole
  * 
  */
+@Singleton
 public class ParseAWSErrorFromXmlContent implements HttpErrorHandler {
    @Resource
    protected Logger logger = Logger.NULL;
@@ -60,7 +63,13 @@ public class ParseAWSErrorFromXmlContent implements HttpErrorHandler {
       Exception exception = new HttpResponseException(command, response);
       try {
          AWSError error = parseErrorFromContentOrNull(command, response);
+         exception = error != null ? new AWSResponseException(command, response, error) : exception;
          switch (response.getStatusCode()) {
+            case 400:
+               if (error.getCode().equals("InvalidAMIID.NotFound")
+                        || error.getCode().equals("InvalidAMIID.Malformed"))
+                  exception = new ResourceNotFoundException(error.getMessage(), exception);
+               break;
             case 401:
                exception = new AuthorizationException(command.getRequest(), error != null ? error
                         .getMessage() : response.getStatusLine());
@@ -77,9 +86,6 @@ public class ParseAWSErrorFromXmlContent implements HttpErrorHandler {
                      exception = new KeyNotFoundException(container, key, message);
                }
                break;
-            default:
-               exception = error != null ? new AWSResponseException(command, response, error)
-                        : new HttpResponseException(command, response);
          }
       } finally {
          Closeables.closeQuietly(response.getContent());

aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2ComputeServiceLiveTest.java

@@ -32,10 +32,7 @@ import org.jclouds.aws.ec2.services.InstanceClient;
 import org.jclouds.aws.ec2.services.KeyPairClient;
 import org.jclouds.aws.ec2.services.SecurityGroupClient;
 import org.jclouds.compute.BaseComputeServiceLiveTest;
-import org.jclouds.compute.domain.Architecture;
 import org.jclouds.compute.domain.NodeMetadata;
-import org.jclouds.compute.domain.OsFamily;
-import org.jclouds.compute.domain.Template;
 import org.jclouds.compute.options.TemplateOptions;
 import org.jclouds.compute.predicates.NodePredicates;
 import org.jclouds.domain.Credentials;
@@ -58,26 +55,6 @@ public class EC2ComputeServiceLiveTest extends BaseComputeServiceLiveTest {
       service = "ec2";
    }
 
-   @Test
-   public void testTemplateBuilderCanUseImageId() {
-      client.templateBuilder().imageId(Iterables.get(client.listImages(), 0).getId()).build();
-   }
-
-   @Test
-   public void testTemplateBuilder() {
-      Template defaultTemplate = client.templateBuilder().build();
-      assert (defaultTemplate.getImage().getId().startsWith("ami-")) : defaultTemplate;
-      assertEquals(defaultTemplate.getImage().getName(), "9.10");
-      assertEquals(defaultTemplate.getImage().getArchitecture(), Architecture.X86_32);
-      assertEquals(defaultTemplate.getImage().getOsFamily(), OsFamily.UBUNTU);
-      assertEquals(defaultTemplate.getLocation().getId(), "us-east-1");
-      assertEquals(defaultTemplate.getSize().getCores(), 1.0d);
-      client.templateBuilder().osFamily(OsFamily.UBUNTU).smallest().architecture(
-               Architecture.X86_32).imageId("ami-7e28ca17").build();
-      client.templateBuilder().osFamily(OsFamily.UBUNTU).smallest().architecture(
-               Architecture.X86_32).imageId("ami-bb709dd2").build();
-   }
-
    @Override
    protected JschSshClientModule getSshModule() {
       return new JschSshClientModule();

aws/core/src/test/java/org/jclouds/aws/ec2/compute/EC2TemplateBuilderLiveTest.java

@@ -0,0 +1,121 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.aws.ec2.compute;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.testng.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+
+import org.jclouds.aws.ec2.reference.EC2Constants;
+import org.jclouds.compute.ComputeServiceContext;
+import org.jclouds.compute.ComputeServiceContextFactory;
+import org.jclouds.compute.domain.Architecture;
+import org.jclouds.compute.domain.OsFamily;
+import org.jclouds.compute.domain.Template;
+import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.testng.annotations.BeforeGroups;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Test(groups = "live", testName = "ec2.EC2TemplateBuilderLiveTest")
+public class EC2TemplateBuilderLiveTest {
+   private String password;
+   private String user;
+
+   @BeforeGroups(groups = { "live" })
+   public void setupClient() throws InterruptedException, ExecutionException, TimeoutException,
+            IOException {
+      user = checkNotNull(System.getProperty("jclouds.test.user"), "jclouds.test.user");
+      password = checkNotNull(System.getProperty("jclouds.test.key"), "jclouds.test.key");
+   }
+
+   @Test
+   public void testTemplateBuilderCanUseImageId() {
+   }
+
+   @Test
+   public void testTemplateBuilder() throws IOException {
+      ComputeServiceContext newContext = null;
+      try {
+         newContext = new ComputeServiceContextFactory().createContext("ec2", user, password,
+                  ImmutableSet.of(new Log4JLoggingModule()));
+
+         Template defaultTemplate = newContext.getComputeService().templateBuilder().build();
+         assert (defaultTemplate.getImage().getId().startsWith("ami-")) : defaultTemplate;
+         assertEquals(defaultTemplate.getImage().getName(), "9.10");
+         assertEquals(defaultTemplate.getImage().getArchitecture(), Architecture.X86_32);
+         assertEquals(defaultTemplate.getImage().getOsFamily(), OsFamily.UBUNTU);
+         assertEquals(defaultTemplate.getLocation().getId(), "us-east-1");
+         assertEquals(defaultTemplate.getSize().getCores(), 1.0d);
+         newContext.getComputeService().templateBuilder().imageId(
+                  Iterables.get(newContext.getComputeService().listImages(), 0).getId()).build();
+         newContext.getComputeService().templateBuilder().osFamily(OsFamily.UBUNTU).smallest()
+                  .architecture(Architecture.X86_32).imageId("ami-7e28ca17").build();
+         newContext.getComputeService().templateBuilder().osFamily(OsFamily.UBUNTU).smallest()
+                  .architecture(Architecture.X86_32).imageId("ami-bb709dd2").build();
+      } finally {
+         if (newContext != null)
+            newContext.close();
+      }
+   }
+
+   @Test
+   public void testTemplateBuilderWithNoOwnersParsesImageOnDemand() throws IOException {
+      ComputeServiceContext newContext = null;
+      try {
+         Properties overrides = new Properties();
+         // set owners to nothing
+         overrides.setProperty(EC2Constants.PROPERTY_EC2_AMI_OWNERS, "");
+
+         newContext = new ComputeServiceContextFactory().createContext("ec2", user, password,
+                  ImmutableSet.of(new Log4JLoggingModule()), overrides);
+
+         assertEquals(newContext.getComputeService().listImages().size(), 0);
+
+         Template template = newContext.getComputeService().templateBuilder().imageId(
+                  "ami-ccb35ea5").build();
+         System.out.println(template.getImage());
+         assert (template.getImage().getId().startsWith("ami-")) : template;
+         assertEquals(template.getImage().getName(), "5.4");
+         assertEquals(template.getImage().getArchitecture(), Architecture.X86_64);
+         assertEquals(template.getImage().getOsFamily(), OsFamily.CENTOS);
+         assertEquals(template.getImage().getVersion(), "4.4.10");
+         assertEquals(template.getLocation().getId(), "us-east-1");
+         assertEquals(template.getSize().getCores(), 4.0d); // because it is 64bit
+         
+         //ensure we cache the new image for next time
+         assertEquals(newContext.getComputeService().listImages().size(), 1);
+
+      } finally {
+         if (newContext != null)
+            newContext.close();
+      }
+   }
+
+}

aws/core/src/test/java/org/jclouds/aws/ec2/compute/functions/ImageParserTest.java

@@ -153,6 +153,45 @@ public class ImageParserTest extends BaseEC2HandlerTest {
 
    }
 
+   public void testParseRightScaleImage() {
+      InputStream is = getClass().getResourceAsStream("/ec2/rightscale_images.xml");
+
+      Set<Image> result = parseImages(is);
+
+      ImageParser parser = new ImageParser(
+               new EC2PopulateDefaultLoginCredentialsForImageStrategy(), ImmutableSet
+                        .<Location> of(defaultLocation), defaultLocation);
+
+      org.jclouds.compute.domain.Image image = parser.apply(Iterables.get(result, 0));
+
+      assertEquals(image.getArchitecture(), org.jclouds.compute.domain.Architecture.X86_64);
+      assertEquals(image.getDescription(), "rightscale-us-east/CentOS_5.4_x64_v4.4.10.manifest.xml");
+      assertEquals(image.getId(), "ami-ccb35ea5");
+      assertEquals(image.getLocation(), defaultLocation);
+      assertEquals(image.getName(), "5.4");
+      assertEquals(image.getOsDescription(),
+               "rightscale-us-east/CentOS_5.4_x64_v4.4.10.manifest.xml");
+      assertEquals(image.getOsFamily(), OsFamily.CENTOS);
+      assertEquals(image.getUserMetadata(), ImmutableMap.<String, String> of("owner",
+               "411009282317"));
+      assertEquals(image.getVersion(), "4.4.10");
+
+      image = parser.apply(Iterables.get(result, 1));
+
+      assertEquals(image.getArchitecture(), org.jclouds.compute.domain.Architecture.X86_64);
+      assertEquals(image.getDescription(), "RightImage_Ubuntu_9.10_x64_v4.5.3_EBS_Alpha");
+      assertEquals(image.getId(), "ami-c19db6b5");
+      assertEquals(image.getLocation(), defaultLocation);
+      assertEquals(image.getName(), "9.10");
+      assertEquals(image.getOsDescription(),
+               "411009282317/RightImage_Ubuntu_9.10_x64_v4.5.3_EBS_Alpha");
+      assertEquals(image.getOsFamily(), OsFamily.UBUNTU);
+      assertEquals(image.getUserMetadata(), ImmutableMap.<String, String> of("owner",
+               "411009282317"));
+      assertEquals(image.getVersion(), "4.5.3_EBS_Alpha");
+
+   }
+
    private Set<Image> parseImages(InputStream is) {
       DescribeImagesResponseHandler handler = injector
                .getInstance(DescribeImagesResponseHandler.class);

aws/core/src/test/java/org/jclouds/aws/ec2/compute/functions/RegionAndIdToImageTest.java

@@ -0,0 +1,98 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.aws.ec2.compute.functions;
+
+import static org.easymock.EasyMock.expect;
+import static org.easymock.classextension.EasyMock.createMock;
+import static org.easymock.classextension.EasyMock.createNiceMock;
+import static org.easymock.classextension.EasyMock.replay;
+import static org.easymock.classextension.EasyMock.verify;
+import static org.jclouds.aws.ec2.options.DescribeImagesOptions.Builder.imageIds;
+import static org.testng.Assert.assertEquals;
+
+import java.util.Set;
+
+import org.jclouds.aws.ec2.compute.domain.RegionAndName;
+import org.jclouds.aws.ec2.services.AMIClient;
+import org.jclouds.compute.domain.Image;
+import org.jclouds.rest.ResourceNotFoundException;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", testName = "ec2.RegionAndIdToImageTest")
+public class RegionAndIdToImageTest {
+
+   @Test
+   public void testApply() {
+
+      ImageParser parser = createMock(ImageParser.class);
+      AMIClient client = createMock(AMIClient.class);
+      org.jclouds.aws.ec2.domain.Image ec2Image = createMock(org.jclouds.aws.ec2.domain.Image.class);
+      Image image = createNiceMock(Image.class);
+      Set<org.jclouds.aws.ec2.domain.Image> images = ImmutableSet
+               .<org.jclouds.aws.ec2.domain.Image> of(ec2Image);
+      
+      expect(client.describeImagesInRegion("region", imageIds("ami"))).andReturn(images);
+      expect(parser.apply(ec2Image)).andReturn(image);
+
+      replay(image);
+      replay(parser);
+      replay(client);
+
+      RegionAndIdToImage function = new RegionAndIdToImage(parser, client);
+
+      assertEquals(function.apply(new RegionAndName("region", "ami")), image);
+
+      verify(image);
+      verify(parser);
+      verify(client);
+
+   }
+   
+   @Test
+   public void testApplyNotFound() {
+
+      ImageParser parser = createMock(ImageParser.class);
+      AMIClient client = createMock(AMIClient.class);
+      org.jclouds.aws.ec2.domain.Image ec2Image = createMock(org.jclouds.aws.ec2.domain.Image.class);
+      Image image = createNiceMock(Image.class);
+      Set<org.jclouds.aws.ec2.domain.Image> images = ImmutableSet
+               .<org.jclouds.aws.ec2.domain.Image> of(ec2Image);
+      
+      expect(client.describeImagesInRegion("region", imageIds("ami"))).andReturn(images);
+      expect(parser.apply(ec2Image)).andThrow(new ResourceNotFoundException());
+
+      replay(image);
+      replay(parser);
+      replay(client);
+
+      RegionAndIdToImage function = new RegionAndIdToImage(parser, client);
+
+      assertEquals(function.apply(new RegionAndName("region", "ami")), null);
+
+      verify(image);
+      verify(parser);
+      verify(client);
+
+   }
+}

aws/core/src/test/java/org/jclouds/aws/ec2/compute/functions/RunningInstanceToNodeMetadataTest.java

@@ -29,6 +29,9 @@ import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.inject.Provider;
 
 import org.jclouds.aws.domain.Region;
 import org.jclouds.aws.ec2.compute.domain.RegionAndName;
@@ -55,6 +58,203 @@ import com.google.common.collect.ImmutableSet;
  */
 @Test(groups = "unit", testName = "ec2.RunningInstanceToNodeMetadataTest")
 public class RunningInstanceToNodeMetadataTest {
+   private static class ImageProvider implements
+            Provider<Set<? extends org.jclouds.compute.domain.Image>> {
+      private final Set<? extends org.jclouds.compute.domain.Image> images;
+
+      private ImageProvider(org.jclouds.compute.domain.Image jcImage) {
+         this.images = ImmutableSet.<org.jclouds.compute.domain.Image> of(jcImage);
+      }
+
+      @Override
+      public Set<? extends org.jclouds.compute.domain.Image> get() {
+         return images;
+      }
+   }
+
+   @SuppressWarnings("unchecked")
+   @Test
+   public void testImageNotFoundAndLazyReturnsNull() throws UnknownHostException {
+      AMIClient amiClient = createMock(AMIClient.class);
+      Map<RegionAndName, KeyPair> credentialsMap = createMock(Map.class);
+      org.jclouds.compute.domain.Image jcImage = createMock(org.jclouds.compute.domain.Image.class);
+
+      ConcurrentMap<RegionAndName, org.jclouds.compute.domain.Image> imageMap = createMock(ConcurrentMap.class);
+
+      Location location = new LocationImpl(LocationScope.ZONE, "us-east-1a", "description", null);
+      Set<Location> locations = ImmutableSet.<Location> of(location);
+      PopulateDefaultLoginCredentialsForImageStrategy credentialProvider = createMock(PopulateDefaultLoginCredentialsForImageStrategy.class);
+      RunningInstance instance = createMock(RunningInstance.class);
+
+      expect(instance.getId()).andReturn("id").atLeastOnce();
+      expect(instance.getGroupIds()).andReturn(ImmutableSet.<String> of()).atLeastOnce();
+      expect(instance.getKeyName()).andReturn(null).atLeastOnce();
+      expect(instance.getInstanceState()).andReturn(InstanceState.RUNNING);
+
+      expect(instance.getIpAddress()).andReturn(
+               InetAddress.getByAddress(new byte[] { 12, 10, 10, 1 }));
+      expect(instance.getPrivateIpAddress()).andReturn(
+               InetAddress.getByAddress(new byte[] { 10, 10, 10, 1 }));
+
+      expect(instance.getAvailabilityZone()).andReturn(AvailabilityZone.US_EAST_1A).atLeastOnce();
+
+      expect(instance.getImageId()).andReturn("imageId").atLeastOnce();
+      expect(jcImage.getId()).andReturn("notImageId").atLeastOnce();
+      expect(instance.getRegion()).andReturn("us-east-1").atLeastOnce();
+
+      expect(imageMap.get(new RegionAndName("us-east-1", "imageId"))).andReturn(null);
+
+      expect(instance.getInstanceType()).andReturn(InstanceType.C1_XLARGE).atLeastOnce();
+
+      replay(imageMap);
+      replay(jcImage);
+      replay(amiClient);
+      replay(credentialsMap);
+      replay(credentialProvider);
+      replay(instance);
+
+      RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(amiClient,
+               credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations,
+               new RunningInstanceToStorageMappingUnix());
+
+      NodeMetadata metadata = parser.apply(instance);
+      assertEquals(metadata.getLocation(), locations.iterator().next());
+      assertEquals(metadata.getImage(), null);
+      assertEquals(metadata.getTag(), "NOTAG-id");
+      assertEquals(metadata.getCredentials(), null);
+
+      verify(imageMap);
+      verify(jcImage);
+      verify(amiClient);
+      verify(credentialsMap);
+      verify(credentialProvider);
+      verify(instance);
+   }
+
+   @SuppressWarnings("unchecked")
+   @Test
+   public void testImageNotFoundAndLazyFailsWithNPE() throws UnknownHostException {
+      AMIClient amiClient = createMock(AMIClient.class);
+      Map<RegionAndName, KeyPair> credentialsMap = createMock(Map.class);
+      org.jclouds.compute.domain.Image jcImage = createMock(org.jclouds.compute.domain.Image.class);
+
+      ConcurrentMap<RegionAndName, org.jclouds.compute.domain.Image> imageMap = createMock(ConcurrentMap.class);
+
+      Location location = new LocationImpl(LocationScope.ZONE, "us-east-1a", "description", null);
+      Set<Location> locations = ImmutableSet.<Location> of(location);
+      PopulateDefaultLoginCredentialsForImageStrategy credentialProvider = createMock(PopulateDefaultLoginCredentialsForImageStrategy.class);
+      RunningInstance instance = createMock(RunningInstance.class);
+
+      expect(instance.getId()).andReturn("id").atLeastOnce();
+      expect(instance.getGroupIds()).andReturn(ImmutableSet.<String> of()).atLeastOnce();
+      expect(instance.getKeyName()).andReturn(null).atLeastOnce();
+      expect(instance.getInstanceState()).andReturn(InstanceState.RUNNING);
+
+      expect(instance.getIpAddress()).andReturn(
+               InetAddress.getByAddress(new byte[] { 12, 10, 10, 1 }));
+      expect(instance.getPrivateIpAddress()).andReturn(
+               InetAddress.getByAddress(new byte[] { 10, 10, 10, 1 }));
+
+      expect(instance.getAvailabilityZone()).andReturn(AvailabilityZone.US_EAST_1A).atLeastOnce();
+
+      expect(instance.getImageId()).andReturn("imageId").atLeastOnce();
+      expect(jcImage.getId()).andReturn("notImageId").atLeastOnce();
+      expect(instance.getRegion()).andReturn("us-east-1").atLeastOnce();
+
+      expect(imageMap.get(new RegionAndName("us-east-1", "imageId"))).andThrow(
+               new NullPointerException()).atLeastOnce();
+
+      expect(instance.getInstanceType()).andReturn(InstanceType.C1_XLARGE).atLeastOnce();
+
+      replay(imageMap);
+      replay(jcImage);
+      replay(amiClient);
+      replay(credentialsMap);
+      replay(credentialProvider);
+      replay(instance);
+
+      RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(amiClient,
+               credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations,
+               new RunningInstanceToStorageMappingUnix());
+
+      NodeMetadata metadata = parser.apply(instance);
+      assertEquals(metadata.getLocation(), locations.iterator().next());
+      assertEquals(metadata.getImage(), null);
+      assertEquals(metadata.getTag(), "NOTAG-id");
+      assertEquals(metadata.getCredentials(), null);
+
+      verify(imageMap);
+      verify(jcImage);
+      verify(amiClient);
+      verify(credentialsMap);
+      verify(credentialProvider);
+      verify(instance);
+   }
+
+   @SuppressWarnings("unchecked")
+   @Test
+   public void testImageNotFoundAndLazySucceeds() throws UnknownHostException {
+      AMIClient amiClient = createMock(AMIClient.class);
+      Map<RegionAndName, KeyPair> credentialsMap = createMock(Map.class);
+      org.jclouds.compute.domain.Image jcImage = createMock(org.jclouds.compute.domain.Image.class);
+
+      ConcurrentMap<RegionAndName, org.jclouds.compute.domain.Image> imageMap = createMock(ConcurrentMap.class);
+
+      Location location = new LocationImpl(LocationScope.ZONE, "us-east-1a", "description", null);
+      Set<Location> locations = ImmutableSet.<Location> of(location);
+      PopulateDefaultLoginCredentialsForImageStrategy credentialProvider = createMock(PopulateDefaultLoginCredentialsForImageStrategy.class);
+      RunningInstance instance = createMock(RunningInstance.class);
+
+      expect(instance.getId()).andReturn("id").atLeastOnce();
+      expect(instance.getGroupIds()).andReturn(ImmutableSet.<String> of()).atLeastOnce();
+      expect(instance.getKeyName()).andReturn(null).atLeastOnce();
+      expect(instance.getInstanceState()).andReturn(InstanceState.RUNNING);
+
+      expect(instance.getIpAddress()).andReturn(
+               InetAddress.getByAddress(new byte[] { 12, 10, 10, 1 }));
+      expect(instance.getPrivateIpAddress()).andReturn(
+               InetAddress.getByAddress(new byte[] { 10, 10, 10, 1 }));
+
+      expect(instance.getAvailabilityZone()).andReturn(AvailabilityZone.US_EAST_1A).atLeastOnce();
+
+      expect(instance.getImageId()).andReturn("imageId").atLeastOnce();
+      expect(jcImage.getId()).andReturn("notImageId").atLeastOnce();
+      expect(instance.getRegion()).andReturn("us-east-1").atLeastOnce();
+
+      org.jclouds.compute.domain.Image lateImage = createMock(org.jclouds.compute.domain.Image.class);
+
+      expect(imageMap.get(new RegionAndName("us-east-1", "imageId"))).andReturn(lateImage)
+               .atLeastOnce();
+
+      expect(instance.getInstanceType()).andReturn(InstanceType.C1_XLARGE).atLeastOnce();
+
+      replay(lateImage);
+      replay(imageMap);
+      replay(jcImage);
+      replay(amiClient);
+      replay(credentialsMap);
+      replay(credentialProvider);
+      replay(instance);
+
+      RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(amiClient,
+               credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations,
+               new RunningInstanceToStorageMappingUnix());
+
+      NodeMetadata metadata = parser.apply(instance);
+      assertEquals(metadata.getLocation(), locations.iterator().next());
+      assertEquals(metadata.getImage(), lateImage);
+      assertEquals(metadata.getTag(), "NOTAG-id");
+      assertEquals(metadata.getCredentials(), null);
+
+      verify(lateImage);
+      verify(imageMap);
+      verify(jcImage);
+      verify(amiClient);
+      verify(credentialsMap);
+      verify(credentialProvider);
+      verify(instance);
+   }
+
    @SuppressWarnings("unchecked")
    @Test
    public void testApplyWithNoSecurityGroupCreatesTagOfIdPrefixedByTagAndNullCredentials()
@@ -63,8 +263,7 @@ public class RunningInstanceToNodeMetadataTest {
       Map<RegionAndName, KeyPair> credentialsMap = createMock(Map.class);
       org.jclouds.compute.domain.Image jcImage = createMock(org.jclouds.compute.domain.Image.class);
 
-      Set<org.jclouds.compute.domain.Image> images = ImmutableSet
+      ConcurrentMap<RegionAndName, org.jclouds.compute.domain.Image> imageMap = createMock(ConcurrentMap.class);
-               .<org.jclouds.compute.domain.Image> of(jcImage);
 
       Location location = new LocationImpl(LocationScope.ZONE, "us-east-1a", "description", null);
       Set<Location> locations = ImmutableSet.<Location> of(location);
@@ -89,6 +288,7 @@ public class RunningInstanceToNodeMetadataTest {
 
       expect(instance.getInstanceType()).andReturn(InstanceType.C1_XLARGE).atLeastOnce();
 
+      replay(imageMap);
       replay(jcImage);
       replay(amiClient);
       replay(credentialsMap);
@@ -96,7 +296,7 @@ public class RunningInstanceToNodeMetadataTest {
       replay(instance);
 
       RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(amiClient,
-               credentialsMap, credentialProvider, images, locations,
+               credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations,
                new RunningInstanceToStorageMappingUnix());
 
       NodeMetadata metadata = parser.apply(instance);
@@ -105,6 +305,7 @@ public class RunningInstanceToNodeMetadataTest {
       assertEquals(metadata.getTag(), "NOTAG-id");
       assertEquals(metadata.getCredentials(), null);
 
+      verify(imageMap);
       verify(jcImage);
       verify(amiClient);
       verify(credentialsMap);
@@ -119,9 +320,7 @@ public class RunningInstanceToNodeMetadataTest {
       AMIClient amiClient = createMock(AMIClient.class);
       Map<RegionAndName, KeyPair> credentialsMap = createMock(Map.class);
       org.jclouds.compute.domain.Image jcImage = createMock(org.jclouds.compute.domain.Image.class);
-
+      ConcurrentMap<RegionAndName, org.jclouds.compute.domain.Image> imageMap = createMock(ConcurrentMap.class);
-      Set<org.jclouds.compute.domain.Image> images = ImmutableSet
-               .<org.jclouds.compute.domain.Image> of(jcImage);
 
       Location location = new LocationImpl(LocationScope.ZONE, "us-east-1a", "description", null);
       Set<Location> locations = ImmutableSet.<Location> of(location);
@@ -146,6 +345,7 @@ public class RunningInstanceToNodeMetadataTest {
 
       expect(instance.getInstanceType()).andReturn(InstanceType.C1_XLARGE).atLeastOnce();
 
+      replay(imageMap);
       replay(jcImage);
       replay(amiClient);
       replay(credentialsMap);
@@ -153,7 +353,7 @@ public class RunningInstanceToNodeMetadataTest {
       replay(instance);
 
       RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(amiClient,
-               credentialsMap, credentialProvider, images, locations,
+               credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations,
                new RunningInstanceToStorageMappingUnix());
 
       NodeMetadata metadata = parser.apply(instance);
@@ -162,6 +362,7 @@ public class RunningInstanceToNodeMetadataTest {
       assertEquals(metadata.getTag(), "tag");
       assertEquals(metadata.getCredentials(), null);
 
+      verify(imageMap);
       verify(jcImage);
       verify(amiClient);
       verify(credentialsMap);
@@ -175,6 +376,7 @@ public class RunningInstanceToNodeMetadataTest {
             throws UnknownHostException {
       AMIClient amiClient = createMock(AMIClient.class);
       Map<RegionAndName, KeyPair> credentialsMap = createMock(Map.class);
+      ConcurrentMap<RegionAndName, org.jclouds.compute.domain.Image> imageMap = createMock(ConcurrentMap.class);
 
       PopulateDefaultLoginCredentialsForImageStrategy credentialProvider = createMock(PopulateDefaultLoginCredentialsForImageStrategy.class);
       RunningInstance instance = createMock(RunningInstance.class);
@@ -188,8 +390,6 @@ public class RunningInstanceToNodeMetadataTest {
       Location location = new LocationImpl(LocationScope.ZONE, "us-east-1a", "description", null);
       Set<Location> locations = ImmutableSet.<Location> of(location);
       org.jclouds.compute.domain.Image jcImage = createMock(org.jclouds.compute.domain.Image.class);
-      Set<org.jclouds.compute.domain.Image> images = ImmutableSet
-               .<org.jclouds.compute.domain.Image> of(jcImage);
 
       expect(instance.getIpAddress()).andReturn(
                InetAddress.getByAddress(new byte[] { 12, 10, 10, 1 }));
@@ -214,6 +414,7 @@ public class RunningInstanceToNodeMetadataTest {
 
       expect(instance.getInstanceType()).andReturn(InstanceType.C1_XLARGE).atLeastOnce();
 
+      replay(imageMap);
       replay(amiClient);
       replay(credentialsMap);
       replay(credentialProvider);
@@ -221,9 +422,8 @@ public class RunningInstanceToNodeMetadataTest {
       replay(jcImage);
 
       RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(amiClient,
-               credentialsMap, credentialProvider, images, locations,
+               credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations,
                new RunningInstanceToStorageMappingUnix());
-
       NodeMetadata metadata = parser.apply(instance);
 
       assertEquals(metadata.getTag(), "tag");
@@ -232,6 +432,7 @@ public class RunningInstanceToNodeMetadataTest {
 
       assertEquals(metadata.getCredentials(), new Credentials("user", "pass"));
 
+      verify(imageMap);
       verify(jcImage);
       verify(amiClient);
       verify(credentialsMap);
@@ -242,25 +443,24 @@ public class RunningInstanceToNodeMetadataTest {
 
    @SuppressWarnings("unchecked")
    @Test
-   public void testApplyWithTwoSecurityGroups()
+   public void testApplyWithTwoSecurityGroups() throws UnknownHostException {
-            throws UnknownHostException {
       AMIClient amiClient = createMock(AMIClient.class);
       Map<RegionAndName, KeyPair> credentialsMap = createMock(Map.class);
+      ConcurrentMap<RegionAndName, org.jclouds.compute.domain.Image> imageMap = createMock(ConcurrentMap.class);
 
       PopulateDefaultLoginCredentialsForImageStrategy credentialProvider = createMock(PopulateDefaultLoginCredentialsForImageStrategy.class);
       RunningInstance instance = createMock(RunningInstance.class);
       Image image = createMock(Image.class);
 
       expect(instance.getId()).andReturn("id").atLeastOnce();
-      expect(instance.getGroupIds()).andReturn(ImmutableSet.of("jclouds#tag","jclouds#tag2")).atLeastOnce();
+      expect(instance.getGroupIds()).andReturn(ImmutableSet.of("jclouds#tag", "jclouds#tag2"))
+               .atLeastOnce();
       expect(instance.getKeyName()).andReturn("jclouds#keyName").atLeastOnce();
       expect(instance.getInstanceState()).andReturn(InstanceState.RUNNING);
 
       Location location = new LocationImpl(LocationScope.ZONE, "us-east-1a", "description", null);
       Set<Location> locations = ImmutableSet.<Location> of(location);
       org.jclouds.compute.domain.Image jcImage = createMock(org.jclouds.compute.domain.Image.class);
-      Set<org.jclouds.compute.domain.Image> images = ImmutableSet
-               .<org.jclouds.compute.domain.Image> of(jcImage);
 
       expect(instance.getIpAddress()).andReturn(
                InetAddress.getByAddress(new byte[] { 12, 10, 10, 1 }));
@@ -285,6 +485,7 @@ public class RunningInstanceToNodeMetadataTest {
 
       expect(instance.getInstanceType()).andReturn(InstanceType.C1_XLARGE).atLeastOnce();
 
+      replay(imageMap);
       replay(amiClient);
       replay(credentialsMap);
       replay(credentialProvider);
@@ -292,7 +493,7 @@ public class RunningInstanceToNodeMetadataTest {
       replay(jcImage);
 
       RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(amiClient,
-               credentialsMap, credentialProvider, images, locations,
+               credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations,
                new RunningInstanceToStorageMappingUnix());
 
       NodeMetadata metadata = parser.apply(instance);
@@ -303,6 +504,7 @@ public class RunningInstanceToNodeMetadataTest {
 
       assertEquals(metadata.getCredentials(), new Credentials("user", "pass"));
 
+      verify(imageMap);
       verify(jcImage);
       verify(amiClient);
       verify(credentialsMap);

aws/core/src/test/java/org/jclouds/aws/ec2/compute/internal/EC2TemplateBuilderImplTest.java

@@ -0,0 +1,160 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.aws.ec2.compute.internal;
+
+import static org.easymock.EasyMock.expect;
+import static org.easymock.classextension.EasyMock.createMock;
+import static org.easymock.classextension.EasyMock.createNiceMock;
+import static org.easymock.classextension.EasyMock.replay;
+import static org.easymock.classextension.EasyMock.verify;
+import static org.testng.Assert.assertEquals;
+
+import java.util.EnumSet;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.inject.Provider;
+
+import org.jclouds.aws.ec2.compute.domain.RegionAndName;
+import org.jclouds.compute.domain.Architecture;
+import org.jclouds.compute.domain.Image;
+import org.jclouds.compute.domain.Size;
+import org.jclouds.compute.domain.TemplateBuilder;
+import org.jclouds.compute.domain.internal.SizeImpl;
+import org.jclouds.compute.internal.TemplateBuilderImpl;
+import org.jclouds.compute.internal.TemplateBuilderImplTest;
+import org.jclouds.compute.options.TemplateOptions;
+import org.jclouds.domain.Location;
+import org.jclouds.domain.LocationScope;
+import org.jclouds.domain.internal.LocationImpl;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+@Test(groups = "unit", sequential = true)
+public class EC2TemplateBuilderImplTest extends TemplateBuilderImplTest {
+
+   RegionAndName knownRegionAndName = new RegionAndName("region", "ami");
+   Image knownImage = createNiceMock(Image.class);
+
+   ConcurrentMap<RegionAndName, Image> imageMap = new MapMaker()
+            .makeComputingMap(new Function<RegionAndName, Image>() {
+               @Override
+               public Image apply(RegionAndName from) {
+                  return from.equals(knownRegionAndName) ? knownImage : null;
+               }
+
+            });
+
+   @BeforeTest
+   void setup() {
+      knownImage = createNiceMock(Image.class);
+   }
+
+   @Override
+   protected EC2TemplateBuilderImpl createTemplateBuilder(Set<Location> locations,
+            Set<Image> images, Set<Size> sizes, Location defaultLocation,
+            Provider<TemplateOptions> optionsProvider,
+            Provider<TemplateBuilder> templateBuilderProvider) {
+      return new EC2TemplateBuilderImpl(locations, images, sizes, defaultLocation, optionsProvider,
+               templateBuilderProvider, imageMap);
+   }
+
+   @SuppressWarnings("unchecked")
+   @Test
+   public void testParseOnDemand() {
+      Location location = new LocationImpl(LocationScope.REGION, "region", "region", null);
+      Set<Location> locations = ImmutableSet.<Location> of(location);
+      Set<Image> images = ImmutableSet.<Image> of();
+      Set<Size> sizes = ImmutableSet.<Size> of(new SizeImpl(null, null, location, null,
+               ImmutableMap.<String, String> of(), 1, 1, 1, EnumSet.allOf(Architecture.class)));
+      Location defaultLocation = createMock(Location.class);
+      Provider<TemplateOptions> optionsProvider = createMock(Provider.class);
+      Provider<TemplateBuilder> templateBuilderProvider = createMock(Provider.class);
+      TemplateOptions defaultOptions = createMock(TemplateOptions.class);
+      knownImage = createMock(Image.class);
+
+      expect(defaultLocation.getId()).andReturn("region");
+      expect(optionsProvider.get()).andReturn(defaultOptions);
+      expect(knownImage.getArchitecture()).andReturn(Architecture.X86_32);
+
+      replay(knownImage);
+      replay(defaultOptions);
+      replay(defaultLocation);
+      replay(optionsProvider);
+      replay(templateBuilderProvider);
+
+      TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes,
+               defaultLocation, optionsProvider, templateBuilderProvider);
+
+      assertEquals(template.imageId("ami").build().getImage(), knownImage);
+
+      verify(knownImage);
+      verify(defaultOptions);
+      verify(defaultLocation);
+      verify(optionsProvider);
+      verify(templateBuilderProvider);
+   }
+
+   @SuppressWarnings("unchecked")
+   @Test(expectedExceptions = NoSuchElementException.class)
+   public void testParseOnDemandNotFound() {
+      Location location = new LocationImpl(LocationScope.REGION, "region", "region", null);
+      Set<Location> locations = ImmutableSet.<Location> of(location);
+      Set<Image> images = ImmutableSet.<Image> of();
+      Set<Size> sizes = ImmutableSet.<Size> of(new SizeImpl(null, null, location, null,
+               ImmutableMap.<String, String> of(), 1, 1, 1, EnumSet.allOf(Architecture.class)));
+      Location defaultLocation = createMock(Location.class);
+      Provider<TemplateOptions> optionsProvider = createMock(Provider.class);
+      Provider<TemplateBuilder> templateBuilderProvider = createMock(Provider.class);
+      TemplateOptions defaultOptions = createMock(TemplateOptions.class);
+      knownImage = createMock(Image.class);
+
+      expect(defaultLocation.getId()).andReturn("region");
+      expect(optionsProvider.get()).andReturn(defaultOptions);
+      expect(knownImage.getArchitecture()).andReturn(Architecture.X86_32).atLeastOnce();
+
+      replay(knownImage);
+      replay(defaultOptions);
+      replay(defaultLocation);
+      replay(optionsProvider);
+      replay(templateBuilderProvider);
+
+      TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes,
+               defaultLocation, optionsProvider, templateBuilderProvider);
+
+      assertEquals(template.imageId("bad").build().getImage(), knownImage);
+
+      verify(knownImage);
+      verify(defaultOptions);
+      verify(defaultLocation);
+      verify(optionsProvider);
+      verify(templateBuilderProvider);
+   }
+
+}

aws/core/src/test/java/org/jclouds/aws/ec2/services/AMIClientLiveTest.java

@@ -37,6 +37,7 @@ import org.jclouds.aws.ec2.domain.Image;
 import org.jclouds.aws.ec2.domain.RootDeviceType;
 import org.jclouds.aws.ec2.domain.Image.ImageType;
 import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.rest.ResourceNotFoundException;
 import org.jclouds.rest.RestContext;
 import org.testng.annotations.AfterTest;
 import org.testng.annotations.BeforeGroups;
@@ -74,9 +75,19 @@ public class AMIClientLiveTest {
       client = context.getApi().getAMIServices();
    }
 
+   @Test(expectedExceptions = ResourceNotFoundException.class)
+   public void testDescribeImageNotExists() {
+      client.describeImagesInRegion(null, imageIds("ami-cdf819a3"));
+   }
+
+   @Test(expectedExceptions = ResourceNotFoundException.class)
+   public void testDescribeImageBadId() {
+      client.describeImagesInRegion(null, imageIds("asdaasdsa"));
+   }
+
+   @Test(enabled = false)
    public void testDescribeImages() {
-      for (String region : ImmutableSet.of(Region.EU_WEST_1, Region.US_EAST_1,
+      for (String region : ImmutableSet.of(Region.EU_WEST_1, Region.US_EAST_1, Region.US_WEST_1)) {
-               Region.US_WEST_1)) {
          SortedSet<Image> allResults = Sets.newTreeSet(client.describeImagesInRegion(region));
          assertNotNull(allResults);
          assert allResults.size() >= 2 : allResults.size();
@@ -95,8 +106,8 @@ public class AMIClientLiveTest {
 
    @Test(enabled = false)
    public void testRegisterImageFromManifest() {
-      String imageRegisteredId = client.registerImageFromManifestInRegion(null,
+      String imageRegisteredId = client.registerImageFromManifestInRegion(null, "jcloudstest1",
-               "jcloudstest1", DEFAULT_MANIFEST);
+               DEFAULT_MANIFEST);
       imagesToDeregister.add(imageRegisteredId);
       Image imageRegisteredFromManifest = Iterables.getOnlyElement(client.describeImagesInRegion(
                null, imageIds(imageRegisteredId)));
@@ -109,8 +120,8 @@ public class AMIClientLiveTest {
 
    @Test(enabled = false)
    public void testRegisterImageFromManifestOptions() {
-      String imageRegisteredWithOptionsId = client.registerImageFromManifestInRegion(
+      String imageRegisteredWithOptionsId = client.registerImageFromManifestInRegion(null,
-               null, "jcloudstest2", DEFAULT_MANIFEST, withDescription("adrian"));
+               "jcloudstest2", DEFAULT_MANIFEST, withDescription("adrian"));
       imagesToDeregister.add(imageRegisteredWithOptionsId);
       Image imageRegisteredFromManifestWithOptions = Iterables.getOnlyElement(client
                .describeImagesInRegion(null, imageIds(imageRegisteredWithOptionsId)));
@@ -126,11 +137,11 @@ public class AMIClientLiveTest {
    @Test(enabled = false)
    // awaiting EBS functionality to be added to jclouds
    public void testRegisterImageBackedByEBS() {
-      String imageRegisteredId = client.registerUnixImageBackedByEbsInRegion(null,
+      String imageRegisteredId = client.registerUnixImageBackedByEbsInRegion(null, "jcloudstest1",
-               "jcloudstest1", DEFAULT_MANIFEST);
+               DEFAULT_MANIFEST);
       imagesToDeregister.add(imageRegisteredId);
-      Image imageRegistered = Iterables.getOnlyElement(client.describeImagesInRegion(
+      Image imageRegistered = Iterables.getOnlyElement(client.describeImagesInRegion(null,
-               null, imageIds(imageRegisteredId)));
+               imageIds(imageRegisteredId)));
       assertEquals(imageRegistered.getName(), "jcloudstest1");
       assertEquals(imageRegistered.getImageType(), ImageType.MACHINE);
       assertEquals(imageRegistered.getRootDeviceType(), RootDeviceType.EBS);
@@ -140,9 +151,9 @@ public class AMIClientLiveTest {
    @Test(enabled = false)
    // awaiting EBS functionality to be added to jclouds
    public void testRegisterImageBackedByEBSOptions() {
-      String imageRegisteredWithOptionsId = client.registerUnixImageBackedByEbsInRegion(
+      String imageRegisteredWithOptionsId = client.registerUnixImageBackedByEbsInRegion(null,
-               null, "jcloudstest2", DEFAULT_SNAPSHOT, addNewBlockDevice("/dev/sda2",
+               "jcloudstest2", DEFAULT_SNAPSHOT, addNewBlockDevice("/dev/sda2", "myvirtual", 1)
-                        "myvirtual", 1).withDescription("adrian"));
+                        .withDescription("adrian"));
       imagesToDeregister.add(imageRegisteredWithOptionsId);
       Image imageRegisteredWithOptions = Iterables.getOnlyElement(client.describeImagesInRegion(
                null, imageIds(imageRegisteredWithOptionsId)));

aws/core/src/test/resources/ec2/rightscale_images.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<DescribeImagesResponse xmlns="http://ec2.amazonaws.com/doc/2009-11-30/">
+    <requestId>50c73a72-cf38-462f-aba2-f59116380d36</requestId>
+    <imagesSet>
+        <item>
+            <imageId>ami-ccb35ea5</imageId>
+            <imageLocation>rightscale-us-east/CentOS_5.4_x64_v4.4.10.manifest.xml</imageLocation>
+            <imageState>available</imageState>
+            <imageOwnerId>411009282317</imageOwnerId>
+            <isPublic>true</isPublic>
+            <architecture>x86_64</architecture>
+            <imageType>machine</imageType>
+            <kernelId>aki-b51cf9dc</kernelId>
+            <ramdiskId>ari-b31cf9da</ramdiskId>
+            <rootDeviceType>instance-store</rootDeviceType>
+            <rootDeviceName>/dev/sda1</rootDeviceName>
+            <blockDeviceMapping/>
+        </item>
+        <item>
+            <imageId>ami-c19db6b5</imageId>
+            <imageLocation>411009282317/RightImage_Ubuntu_9.10_x64_v4.5.3_EBS_Alpha</imageLocation>
+            <imageState>available</imageState>
+            <imageOwnerId>411009282317</imageOwnerId>
+            <isPublic>true</isPublic>
+            <architecture>x86_64</architecture>
+            <imageType>machine</imageType>
+            <kernelId>aki-a22a01d6</kernelId>
+            <ramdiskId>ari-ac2a01d8</ramdiskId>
+            <name>RightImage_Ubuntu_9.10_x64_v4.5.3_EBS_Alpha</name>
+            <description>RightImage_Ubuntu_9.10_x64_v4.5.3_EBS_Alpha</description>
+            <rootDeviceType>ebs</rootDeviceType>
+            <rootDeviceName>/dev/sda1</rootDeviceName>
+            <blockDeviceMapping>
+                <item>
+                    <deviceName>/dev/sda1</deviceName>
+                    <ebs>
+                        <snapshotId>snap-a222ddcb</snapshotId>
+                        <volumeSize>10</volumeSize>
+                        <deleteOnTermination>true</deleteOnTermination>
+                    </ebs>
+                </item>
+            </blockDeviceMapping>
+        </item>
+    </imagesSet>
+</DescribeImagesResponse>

aws/core/src/test/resources/log4j.xml

@@ -122,7 +122,7 @@
     <!-- Limit categories -->
     <!-- ================ -->
     <category name="jclouds.blobstore">
-        <priority value="TRACE" />
+        <priority value="DEBUG" />
         <appender-ref ref="ASYNCBLOBSTORE" />
     </category>
 
@@ -136,15 +136,16 @@
         <appender-ref ref="ASYNCWIRE" />
     </category>
     <category name="jclouds.compute">
-        <priority value="TRACE" />
+        <priority value="DEBUG" />
         <appender-ref ref="ASYNCCOMPUTE" />
-    </category><!--
+    </category>
 
     <category name="jclouds.wire">
         <priority value="DEBUG" />
         <appender-ref ref="ASYNCWIRE" />
     </category>
-    --><!--  ======================= -->
+    
+    <!--  ======================= -->
     <!-- Setup the Root category -->
     <!-- ======================= -->
 

compute/src/main/java/org/jclouds/compute/internal/TemplateBuilderImpl.java

@@ -69,33 +69,33 @@ public class TemplateBuilderImpl implements TemplateBuilder {
    private final Location defaultLocation;
 
    @VisibleForTesting
-   OsFamily os;
+   protected OsFamily os;
    @VisibleForTesting
-   Architecture arch;
+   protected Architecture arch;
    @VisibleForTesting
-   String locationId;
+   protected String locationId;
    @VisibleForTesting
-   String imageId;
+   protected String imageId;
    @VisibleForTesting
-   String sizeId;
+   protected String sizeId;
    @VisibleForTesting
-   String osDescription;
+   protected String osDescription;
    @VisibleForTesting
-   String imageVersion;
+   protected String imageVersion;
    @VisibleForTesting
-   String imageName;
+   protected String imageName;
    @VisibleForTesting
-   String imageDescription;
+   protected String imageDescription;
    @VisibleForTesting
-   double minCores;
+   protected double minCores;
    @VisibleForTesting
-   int minRam;
+   protected int minRam;
    @VisibleForTesting
-   boolean biggest;
+   protected boolean biggest;
    @VisibleForTesting
-   boolean fastest;
+   protected boolean fastest;
    @VisibleForTesting
-   TemplateOptions options;
+   protected TemplateOptions options;
 
    @Inject
    protected TemplateBuilderImpl(Set<? extends Location> locations, Set<? extends Image> images,
@@ -401,28 +401,19 @@ public class TemplateBuilderImpl implements TemplateBuilder {
       if (options == null)
          options = optionsProvider.get();
       logger.debug(">> searching params(%s)", this);
-      Image image;
+      Location location = resolveLocation();
-      try {
+      Image image = resolveImage();
-         image = DEFAULT_IMAGE_ORDERING.max(Iterables.filter(images, imagePredicate));
+
-      } catch (NoSuchElementException exception) {
-         throw new NoSuchElementException("image didn't match: " + toString() + "\n" + images);
-      }
-      logger.debug("<<   matched image(%s)", image);
       // ensure we have an architecture matching
       this.arch = image.getArchitecture();
 
-      Ordering<Size> sizeOrdering = DEFAULT_SIZE_ORDERING;
+      Ordering<Size> sizeOrdering = sizeSorter();
-      if (!biggest)
+      Size size = resolveSize(sizeOrdering);
-         sizeOrdering = sizeOrdering.reverse();
+
-      if (fastest)
+      return new TemplateImpl(image, size, location, options);
-         sizeOrdering = Ordering.compound(ImmutableList.of(BY_CORES_ORDERING, sizeOrdering));
+   }
-      Size size;
+
-      try {
+   protected Location resolveLocation() {
-         size = sizeOrdering.max(Iterables.filter(sizes, sizePredicate));
-      } catch (NoSuchElementException exception) {
-         throw new NoSuchElementException("size didn't match: " + toString() + "\n" + sizes);
-      }
-      logger.debug("<<   matched size(%s)", size);
       Location location = Iterables.find(locations, new Predicate<Location>() {
 
          @Override
@@ -432,7 +423,43 @@ public class TemplateBuilderImpl implements TemplateBuilder {
 
       });
       logger.debug("<<   matched location(%s)", location);
-      return new TemplateImpl(image, size, location, options);
+      return location;
+   }
+
+   protected Size resolveSize(Ordering<Size> sizeOrdering) {
+      Size size;
+      try {
+         size = sizeOrdering.max(Iterables.filter(sizes, sizePredicate));
+      } catch (NoSuchElementException exception) {
+         throw new NoSuchElementException("size didn't match: " + toString() + "\n" + sizes);
+      }
+      logger.debug("<<   matched size(%s)", size);
+      return size;
+   }
+
+   protected Ordering<Size> sizeSorter() {
+      Ordering<Size> sizeOrdering = DEFAULT_SIZE_ORDERING;
+      if (!biggest)
+         sizeOrdering = sizeOrdering.reverse();
+      if (fastest)
+         sizeOrdering = Ordering.compound(ImmutableList.of(BY_CORES_ORDERING, sizeOrdering));
+      return sizeOrdering;
+   }
+
+   /**
+    * 
+    * @throws NoSuchElementException
+    *            if there's no image that matches the predicate
+    */
+   protected Image resolveImage() {
+      Image image;
+      try {
+         image = DEFAULT_IMAGE_ORDERING.max(Iterables.filter(images, imagePredicate));
+      } catch (NoSuchElementException exception) {
+         throw new NoSuchElementException("image didn't match: " + toString() + "\n" + images);
+      }
+      logger.debug("<<   matched image(%s)", image);
+      return image;
    }
 
    /**

compute/src/main/java/org/jclouds/compute/options/RunScriptOptions.java

@@ -18,6 +18,8 @@
  */
 package org.jclouds.compute.options;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import org.jclouds.domain.Credentials;
 
 /**
@@ -77,6 +79,9 @@ public class RunScriptOptions {
    private boolean runAsRoot = true;
 
    public RunScriptOptions withOverridingCredentials(Credentials overridingCredentials) {
+      checkNotNull(overridingCredentials, "overridingCredentials");
+      checkNotNull(overridingCredentials.account, "overridingCredentials.account");
+      checkNotNull(overridingCredentials.key, "overridingCredentials.key");
       this.overridingCredentials = overridingCredentials;
       return this;
    }

compute/src/test/java/org/jclouds/compute/BaseComputeServiceLiveTest.java

@@ -231,6 +231,7 @@ public abstract class BaseComputeServiceLiveTest {
          Set<? extends NodeMetadata> nodes = client.runNodesWithTag(tag, 1, options);
          Credentials good = nodes.iterator().next().getCredentials();
          assert good.account != null;
+         assert good.key != null;
 
          Image image = Iterables.get(nodes, 0).getImage();
          try {

compute/src/test/java/org/jclouds/compute/internal/TemplateBuilderImplTest.java

@@ -25,6 +25,7 @@ import static org.easymock.classextension.EasyMock.verify;
 import static org.testng.Assert.assertEquals;
 
 import java.util.NoSuchElementException;
+import java.util.Set;
 
 import javax.inject.Provider;
 
@@ -49,7 +50,9 @@ public class TemplateBuilderImplTest {
    @SuppressWarnings("unchecked")
    @Test
    public void testNothingUsesDefaultTemplateBuilder() {
-
+      Set<Location> locations = ImmutableSet.<Location> of();
+      Set<Image> images = ImmutableSet.<Image> of();
+      Set<Size> sizes = ImmutableSet.<Size> of();
       Location defaultLocation = createMock(Location.class);
       Provider<TemplateOptions> optionsProvider = createMock(Provider.class);
       Provider<TemplateBuilder> templateBuilderProvider = createMock(Provider.class);
@@ -63,9 +66,8 @@ public class TemplateBuilderImplTest {
       replay(optionsProvider);
       replay(templateBuilderProvider);
 
-      TemplateBuilderImpl template = new TemplateBuilderImpl(ImmutableSet.<Location> of(),
+      TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes,
-               ImmutableSet.<Image> of(), ImmutableSet.<Size> of(), defaultLocation,
+               defaultLocation, optionsProvider, templateBuilderProvider);
-               optionsProvider, templateBuilderProvider);
 
       template.build();
 
@@ -75,10 +77,20 @@ public class TemplateBuilderImplTest {
       verify(templateBuilderProvider);
    }
 
+   protected TemplateBuilderImpl createTemplateBuilder(Set<Location> locations, Set<Image> images,
+            Set<Size> sizes, Location defaultLocation, Provider<TemplateOptions> optionsProvider,
+            Provider<TemplateBuilder> templateBuilderProvider) {
+      TemplateBuilderImpl template = new TemplateBuilderImpl(locations, images, sizes,
+               defaultLocation, optionsProvider, templateBuilderProvider);
+      return template;
+   }
+
    @SuppressWarnings("unchecked")
    @Test
    public void testSuppliedLocationWithNoOptions() {
-
+      Set<Location> locations = ImmutableSet.<Location> of();
+      Set<Image> images = ImmutableSet.<Image> of();
+      Set<Size> sizes = ImmutableSet.<Size> of();
       Location defaultLocation = createMock(Location.class);
       Provider<TemplateOptions> optionsProvider = createMock(Provider.class);
       Provider<TemplateBuilder> templateBuilderProvider = createMock(Provider.class);
@@ -91,9 +103,8 @@ public class TemplateBuilderImplTest {
       replay(optionsProvider);
       replay(templateBuilderProvider);
 
-      TemplateBuilderImpl template = new TemplateBuilderImpl(ImmutableSet.<Location> of(),
+      TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes,
-               ImmutableSet.<Image> of(), ImmutableSet.<Size> of(), defaultLocation,
+               defaultLocation, optionsProvider, templateBuilderProvider);
-               optionsProvider, templateBuilderProvider);
 
       try {
          template.imageId("foo").locationId("location").build();
@@ -111,7 +122,9 @@ public class TemplateBuilderImplTest {
    @SuppressWarnings("unchecked")
    @Test
    public void testSuppliedLocationAndOptions() {
-
+      Set<Location> locations = ImmutableSet.<Location> of();
+      Set<Image> images = ImmutableSet.<Image> of();
+      Set<Size> sizes = ImmutableSet.<Size> of();
       Location defaultLocation = createMock(Location.class);
       Provider<TemplateOptions> optionsProvider = createMock(Provider.class);
       Provider<TemplateBuilder> templateBuilderProvider = createMock(Provider.class);
@@ -120,9 +133,8 @@ public class TemplateBuilderImplTest {
       replay(optionsProvider);
       replay(templateBuilderProvider);
 
-      TemplateBuilderImpl template = new TemplateBuilderImpl(ImmutableSet.<Location> of(),
+      TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes,
-               ImmutableSet.<Image> of(), ImmutableSet.<Size> of(), defaultLocation,
+               defaultLocation, optionsProvider, templateBuilderProvider);
-               optionsProvider, templateBuilderProvider);
 
       try {
          template.imageId("foo").options(TemplateOptions.NONE).locationId("location").build();
@@ -138,8 +150,10 @@ public class TemplateBuilderImplTest {
 
    @SuppressWarnings("unchecked")
    @Test
-   public void testDefaultLocationWithNoOptions() {
+   public void testDefaultLocationWithNoOptionsNoSuchElement() {
-
+      Set<Location> locations = ImmutableSet.<Location> of();
+      Set<Image> images = ImmutableSet.<Image> of();
+      Set<Size> sizes = ImmutableSet.<Size> of();
       Location defaultLocation = createMock(Location.class);
       Provider<TemplateOptions> optionsProvider = createMock(Provider.class);
       Provider<TemplateBuilder> templateBuilderProvider = createMock(Provider.class);
@@ -153,9 +167,8 @@ public class TemplateBuilderImplTest {
       replay(optionsProvider);
       replay(templateBuilderProvider);
 
-      TemplateBuilderImpl template = new TemplateBuilderImpl(ImmutableSet.<Location> of(),
+      TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes,
-               ImmutableSet.<Image> of(), ImmutableSet.<Size> of(), defaultLocation,
+               defaultLocation, optionsProvider, templateBuilderProvider);
-               optionsProvider, templateBuilderProvider);
 
       try {
          template.imageId("foo").build();
@@ -173,7 +186,9 @@ public class TemplateBuilderImplTest {
    @SuppressWarnings("unchecked")
    @Test
    public void testDefaultLocationWithOptions() {
-
+      Set<Location> locations = ImmutableSet.<Location> of();
+      Set<Image> images = ImmutableSet.<Image> of();
+      Set<Size> sizes = ImmutableSet.<Size> of();
       Location defaultLocation = createMock(Location.class);
       Provider<TemplateOptions> optionsProvider = createMock(Provider.class);
       Provider<TemplateBuilder> templateBuilderProvider = createMock(Provider.class);
@@ -184,9 +199,8 @@ public class TemplateBuilderImplTest {
       replay(optionsProvider);
       replay(templateBuilderProvider);
 
-      TemplateBuilderImpl template = new TemplateBuilderImpl(ImmutableSet.<Location> of(),
+      TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes,
-               ImmutableSet.<Image> of(), ImmutableSet.<Size> of(), defaultLocation,
+               defaultLocation, optionsProvider, templateBuilderProvider);
-               optionsProvider, templateBuilderProvider);
 
       try {
          template.imageId("foo").options(TemplateOptions.NONE).build();
@@ -203,6 +217,9 @@ public class TemplateBuilderImplTest {
    @SuppressWarnings("unchecked")
    @Test
    public void testImageIdNullsEverythingElse() {
+      Set<Location> locations = ImmutableSet.<Location> of();
+      Set<Image> images = ImmutableSet.<Image> of();
+      Set<Size> sizes = ImmutableSet.<Size> of();
       Location defaultLocation = createMock(Location.class);
       Provider<TemplateOptions> optionsProvider = createMock(Provider.class);
       Provider<TemplateBuilder> templateBuilderProvider = createMock(Provider.class);
@@ -211,9 +228,8 @@ public class TemplateBuilderImplTest {
       replay(optionsProvider);
       replay(templateBuilderProvider);
 
-      TemplateBuilderImpl template = new TemplateBuilderImpl(ImmutableSet.<Location> of(),
+      TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes,
-               ImmutableSet.<Image> of(), ImmutableSet.<Size> of(), defaultLocation,
+               defaultLocation, optionsProvider, templateBuilderProvider);
-               optionsProvider, templateBuilderProvider);
 
       template.architecture(Architecture.X86_32);
       template.imageDescriptionMatches("imageDescriptionMatches");

core/src/main/java/org/jclouds/concurrent/internal/SyncProxy.java

@@ -24,6 +24,7 @@ import java.lang.reflect.Proxy;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
 import javax.annotation.Resource;
@@ -120,12 +121,23 @@ public class SyncProxy implements InvocationHandler {
          try {
             return ((ListenableFuture<?>) methodMap.get(method).invoke(delegate, args)).get(
                      timeoutMap.get(method), TimeUnit.NANOSECONDS);
+         } catch (ExecutionException e) {
+            throw typedExceptionOrPropagate(method.getExceptionTypes(), e.getCause());
          } catch (Exception e) {
-            throw Throwables.getRootCause(e);
+            throw typedExceptionOrPropagate(method.getExceptionTypes(), e);
          }
       }
    }
 
+   public static Throwable typedExceptionOrPropagate(Class<?>[] exceptionTypes, Throwable throwable) {
+      for (Class<?> type : exceptionTypes) {
+         if (type.isInstance(throwable)) {
+            return throwable;
+         }
+      }
+      return Throwables.propagate(throwable);
+   }
+
    @Override
    public boolean equals(Object obj) {
       if (obj == null || !(obj instanceof SyncProxy))

core/src/test/java/org/jclouds/concurrent/internal/SyncProxyTest.java

@@ -24,7 +24,6 @@ import static org.testng.Assert.assertEquals;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
-import java.lang.reflect.UndeclaredThrowableException;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -168,7 +167,7 @@ public class SyncProxyTest {
 
    }
 
-   @Test(expectedExceptions = UndeclaredThrowableException.class)
+   @Test(expectedExceptions = RuntimeException.class)
    public void testTake100MillisecondsAndTimeout() {
       assertEquals(sync.take100MillisecondsAndTimeout(), "foo");