Issue 412: parallel image fetches

elasticstack/src/main/java/org/jclouds/elasticstack/ElasticStackAsyncClient.java

@@ -81,7 +81,7 @@ public interface ElasticStackAsyncClient {
    @GET
    @Path("/servers/info")
    @ResponseParser(ListOfKeyValuesDelimitedByBlankLinesToServerInfoSet.class)
-   ListenableFuture<Set<? extends ServerInfo>> listServerInfo();
+   ListenableFuture<Set<ServerInfo>> listServerInfo();
 
    /**
     * @see ElasticStackClient#getServerInfo
@@ -90,7 +90,7 @@ public interface ElasticStackAsyncClient {
    @ExceptionParser(ReturnNullOnNotFoundOr404.class)
    @ResponseParser(KeyValuesDelimitedByBlankLinesToServerInfo.class)
    @Path("/servers/{uuid}/info")
-   ListenableFuture<? extends ServerInfo> getServerInfo(@PathParam("uuid") String uuid);
+   ListenableFuture<ServerInfo> getServerInfo(@PathParam("uuid") String uuid);
 
    /**
     * @see ElasticStackClient#createServer
@@ -99,7 +99,7 @@ public interface ElasticStackAsyncClient {
    @ExceptionParser(ReturnNullOnNotFoundOr404.class)
    @ResponseParser(KeyValuesDelimitedByBlankLinesToServerInfo.class)
    @Path("/servers/create/stopped")
-   ListenableFuture<? extends ServerInfo> createServer(
+   ListenableFuture<ServerInfo> createServer(
          @BinderParam(BindServerToPlainTextString.class) Server createServer);
 
    /**
@@ -109,7 +109,7 @@ public interface ElasticStackAsyncClient {
    @ExceptionParser(ReturnNullOnNotFoundOr404.class)
    @ResponseParser(KeyValuesDelimitedByBlankLinesToServerInfo.class)
    @Path("/servers/{uuid}/set")
-   ListenableFuture<? extends ServerInfo> setServerConfiguration(@PathParam("uuid") String uuid,
+   ListenableFuture<ServerInfo> setServerConfiguration(@PathParam("uuid") String uuid,
          @BinderParam(BindServerToPlainTextString.class) Server setServer);
 
    /**
@@ -162,7 +162,7 @@ public interface ElasticStackAsyncClient {
    @GET
    @Path("/drives/info")
    @ResponseParser(ListOfKeyValuesDelimitedByBlankLinesToDriveInfoSet.class)
-   ListenableFuture<Set<? extends DriveInfo>> listDriveInfo();
+   ListenableFuture<Set<DriveInfo>> listDriveInfo();
 
    /**
     * @see ElasticStackClient#getDriveInfo
@@ -171,7 +171,7 @@ public interface ElasticStackAsyncClient {
    @ExceptionParser(ReturnNullOnNotFoundOr404.class)
    @ResponseParser(KeyValuesDelimitedByBlankLinesToDriveInfo.class)
    @Path("/drives/{uuid}/info")
-   ListenableFuture<? extends DriveInfo> getDriveInfo(@PathParam("uuid") String uuid);
+   ListenableFuture<DriveInfo> getDriveInfo(@PathParam("uuid") String uuid);
 
    /**
     * @see ElasticStackClient#createDrive
@@ -180,7 +180,7 @@ public interface ElasticStackAsyncClient {
    @ExceptionParser(ReturnNullOnNotFoundOr404.class)
    @ResponseParser(KeyValuesDelimitedByBlankLinesToDriveInfo.class)
    @Path("/drives/create")
-   ListenableFuture<? extends DriveInfo> createDrive(@BinderParam(BindDriveToPlainTextString.class) Drive createDrive);
+   ListenableFuture<DriveInfo> createDrive(@BinderParam(BindDriveToPlainTextString.class) Drive createDrive);
 
    /**
     * @see ElasticStackClient#setDriveData
@@ -189,7 +189,7 @@ public interface ElasticStackAsyncClient {
    @ExceptionParser(ReturnNullOnNotFoundOr404.class)
    @ResponseParser(KeyValuesDelimitedByBlankLinesToDriveInfo.class)
    @Path("/drives/{uuid}/set")
-   ListenableFuture<? extends DriveInfo> setDriveData(@PathParam("uuid") String uuid,
+   ListenableFuture<DriveInfo> setDriveData(@PathParam("uuid") String uuid,
          @BinderParam(BindDriveDataToPlainTextString.class) DriveData setDrive);
 
    /**
@@ -207,7 +207,7 @@ public interface ElasticStackAsyncClient {
    @ExceptionParser(ReturnNullOnNotFoundOr404.class)
    @ResponseParser(KeyValuesDelimitedByBlankLinesToServerInfo.class)
    @Path("/servers/create")
-   ListenableFuture<? extends ServerInfo> createAndStartServer(
+   ListenableFuture<ServerInfo> createAndStartServer(
          @BinderParam(BindServerToPlainTextString.class) Server createServer);
 
    /**

elasticstack/src/main/java/org/jclouds/elasticstack/compute/ElasticStackComputeServiceAdapter.java

@@ -3,25 +3,25 @@ package org.jclouds.elasticstack.compute;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Predicates.notNull;
 import static com.google.common.collect.Iterables.filter;
-import static com.google.common.collect.Iterables.transform;
+import static org.jclouds.concurrent.FutureIterables.transformParallel;
 import static org.jclouds.elasticstack.util.Servers.small;
 
 import java.net.URI;
-import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
 
 import javax.annotation.Resource;
 import javax.inject.Inject;
 import javax.inject.Named;
 import javax.inject.Singleton;
 
+import org.jclouds.Constants;
 import org.jclouds.compute.ComputeService;
 import org.jclouds.compute.ComputeServiceAdapter;
 import org.jclouds.compute.domain.Hardware;
 import org.jclouds.compute.domain.HardwareBuilder;
 import org.jclouds.compute.domain.Image;
-import org.jclouds.compute.domain.ImageBuilder;
-import org.jclouds.compute.domain.OperatingSystemBuilder;
 import org.jclouds.compute.domain.Processor;
 import org.jclouds.compute.domain.Template;
 import org.jclouds.compute.domain.Volume;
@@ -31,6 +31,7 @@ import org.jclouds.domain.Credentials;
 import org.jclouds.domain.Location;
 import org.jclouds.domain.LocationScope;
 import org.jclouds.domain.internal.LocationImpl;
+import org.jclouds.elasticstack.ElasticStackAsyncClient;
 import org.jclouds.elasticstack.ElasticStackClient;
 import org.jclouds.elasticstack.domain.Device;
 import org.jclouds.elasticstack.domain.Drive;
@@ -45,9 +46,7 @@ import org.jclouds.rest.annotations.Provider;
 
 import com.google.common.base.Function;
 import com.google.common.base.Predicate;
-import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSet.Builder;
 
@@ -57,31 +56,37 @@ import com.google.common.collect.ImmutableSet.Builder;
  * 
  */
 @Singleton
-public class ElasticStackComputeServiceAdapter implements ComputeServiceAdapter<ServerInfo, Hardware, Image, Location> {
+public class ElasticStackComputeServiceAdapter implements
+      ComputeServiceAdapter<ServerInfo, Hardware, DriveInfo, Location> {
    private final ElasticStackClient client;
+   private final ElasticStackAsyncClient aclient;
    private final Predicate<DriveInfo> driveNotClaimed;
-   private final Supplier<Location> locationSupplier;
+   private final Map<String, WellKnownImage> preinstalledImages;
-   private final List<WellKnownImage> preinstalledImages;
+   private final Map<String, DriveInfo> cache;
    private final String providerName;
    private final URI providerURI;
    private final String defaultVncPassword;
+   private final ExecutorService executor;
 
    @Resource
    @Named(ComputeServiceConstants.COMPUTE_LOGGER)
    protected Logger logger = Logger.NULL;
 
    @Inject
-   public ElasticStackComputeServiceAdapter(ElasticStackClient client, Predicate<DriveInfo> driveNotClaimed,
+   public ElasticStackComputeServiceAdapter(ElasticStackClient client, ElasticStackAsyncClient aclient,
-         Supplier<Location> locationSupplier, @Provider String providerName, @Provider URI providerURI,
+         Predicate<DriveInfo> driveNotClaimed, @Provider String providerName, @Provider URI providerURI,
-         List<WellKnownImage> preinstalledImages,
+         Map<String, WellKnownImage> preinstalledImages, Map<String, DriveInfo> cache,
-         @Named(ElasticStackConstants.PROPERTY_VNC_PASSWORD) String defaultVncPassword) {
+         @Named(ElasticStackConstants.PROPERTY_VNC_PASSWORD) String defaultVncPassword,
+         @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) {
       this.client = checkNotNull(client, "client");
+      this.aclient = checkNotNull(aclient, "aclient");
       this.driveNotClaimed = checkNotNull(driveNotClaimed, "driveNotClaimed");
-      this.locationSupplier = checkNotNull(locationSupplier, "locationSupplier");
       this.providerName = checkNotNull(providerName, "providerName");
       this.providerURI = checkNotNull(providerURI, "providerURI");
       this.preinstalledImages = checkNotNull(preinstalledImages, "preinstalledImages");
+      this.cache = checkNotNull(cache, "cache");
       this.defaultVncPassword = checkNotNull(defaultVncPassword, "defaultVncPassword");
+      this.executor = checkNotNull(executor, "executor");
    }
 
    @Override
@@ -125,6 +130,11 @@ public class ElasticStackComputeServiceAdapter implements ComputeServiceAdapter<
                   return (toParse != null && new Float(toParse) <= size);
                }
 
+               @Override
+               public String toString() {
+                  return "sizeLessThanOrEqual(" + size + ")";
+               }
+
             }).ids(id).ram(ram).processors(ImmutableList.of(new Processor(1, cpu)))
                   .volumes(ImmutableList.<Volume> of(new VolumeImpl(size, true, true))).build());
          }
@@ -135,37 +145,20 @@ public class ElasticStackComputeServiceAdapter implements ComputeServiceAdapter<
     * look up the current standard images and do not error out, if they are not found.
     */
    @Override
-   public Iterable<Image> listImages() {
+   public Iterable<DriveInfo> listImages() {
-      return filter(transform(preinstalledImages, new Function<WellKnownImage, Image>() {
+      Iterable<DriveInfo> drives = transformParallel(preinstalledImages.keySet(),
+            new Function<String, Future<DriveInfo>>() {
 
-         @Override
+               @Override
-         public Image apply(WellKnownImage input) {
+               public Future<DriveInfo> apply(String input) {
-            DriveInfo drive = null;
+                  return aclient.getDriveInfo(input);
-            try {
+               }
-               drive = client.getDriveInfo(input.getUuid());
-            } catch (Exception e) {
-               logger.warn(e, "could not find image: %s", input);
-            }
-            if (drive == null) {
-               logger.warn("could not find image: %s", input);
-               return null;
-            }
-            return new ImageBuilder()
-                  .ids(drive.getUuid())
-                  .userMetadata(
-                        ImmutableMap.<String, String> builder().putAll(drive.getUserMetadata())
-                              .put("size", input.getSize() + "").build())
-                  .defaultCredentials(new Credentials("toor", null))
-                  .location(locationSupplier.get())
-                  .name(input.getDescription())
-                  .description(drive.getName())
-                  .operatingSystem(
-                        new OperatingSystemBuilder().family(input.getOsFamily()).version(input.getOsVersion())
-                              .name(input.getDescription()).description(drive.getName()).is64Bit(true).build())
-                  .version("").build();
-         }
 
-      }), notNull());
+            }, executor, null, logger, "drives");
+      Iterable<DriveInfo> returnVal = filter(drives, notNull());
+      for (DriveInfo drive : returnVal)
+         cache.put(drive.getUuid(), drive);
+      return returnVal;
    }
 
    @SuppressWarnings("unchecked")

elasticstack/src/main/java/org/jclouds/elasticstack/compute/config/ElasticStackComputeServiceContextModule.java

@@ -44,6 +44,7 @@ import org.jclouds.elasticstack.compute.functions.ServerInfoToNodeMetadata;
 import org.jclouds.elasticstack.compute.functions.ServerInfoToNodeMetadata.DeviceToVolume;
 import org.jclouds.elasticstack.compute.functions.ServerInfoToNodeMetadata.FindImageForId;
 import org.jclouds.elasticstack.compute.functions.ServerInfoToNodeMetadata.GetImageIdFromServer;
+import org.jclouds.elasticstack.compute.functions.WellKnownImageToImage;
 import org.jclouds.elasticstack.domain.Device;
 import org.jclouds.elasticstack.domain.DriveInfo;
 import org.jclouds.elasticstack.domain.Server;
@@ -59,6 +60,7 @@ import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
 import com.google.common.base.Supplier;
 import com.google.common.collect.MapMaker;
+import com.google.common.collect.Maps;
 import com.google.inject.Provides;
 import com.google.inject.TypeLiteral;
 
@@ -68,7 +70,7 @@ import com.google.inject.TypeLiteral;
  */
 public class ElasticStackComputeServiceContextModule
       extends
-      ComputeServiceAdapterContextModule<ElasticStackClient, ElasticStackAsyncClient, ServerInfo, Hardware, Image, Location> {
+      ComputeServiceAdapterContextModule<ElasticStackClient, ElasticStackAsyncClient, ServerInfo, Hardware, DriveInfo, Location> {
 
    public ElasticStackComputeServiceContextModule() {
       super(ElasticStackClient.class, ElasticStackAsyncClient.class);
@@ -78,7 +80,7 @@ public class ElasticStackComputeServiceContextModule
    @Override
    protected void configure() {
       super.configure();
-      bind(new TypeLiteral<ComputeServiceAdapter<ServerInfo, Hardware, Image, Location>>() {
+      bind(new TypeLiteral<ComputeServiceAdapter<ServerInfo, Hardware, DriveInfo, Location>>() {
       }).to(ElasticStackComputeServiceAdapter.class);
       bind(IdentityFunction.class).toInstance(IdentityFunction.INSTANCE);
       bind(new TypeLiteral<Supplier<Location>>() {
@@ -97,12 +99,14 @@ public class ElasticStackComputeServiceContextModule
       }).to(GetImageIdFromServer.class);
       bind(new TypeLiteral<Function<String, Image>>() {
       }).to(FindImageForId.class);
+      bind(new TypeLiteral<Function<DriveInfo, Image>>() {
+      }).to(WellKnownImageToImage.class);
    }
 
    @Provides
    @Singleton
    protected Map<String, DriveInfo> cache(GetDrive getDrive) {
-      return new MapMaker().expiration(30, TimeUnit.SECONDS).makeComputingMap(getDrive);
+      return new MapMaker().makeComputingMap(getDrive);
    }
 
    @Singleton
@@ -122,10 +126,19 @@ public class ElasticStackComputeServiceContextModule
 
    @Singleton
    @Provides
-   protected List<WellKnownImage> provideImages(Json json) throws IOException {
+   protected Map<String, WellKnownImage> provideImages(Json json) throws IOException {
-      return json.fromJson(Utils.toStringAndClose(getClass().getResourceAsStream("/preinstalled_images.json")),
+      List<WellKnownImage> wellKnowns = json.fromJson(
+            Utils.toStringAndClose(getClass().getResourceAsStream("/preinstalled_images.json")),
             new TypeLiteral<List<WellKnownImage>>() {
             }.getType());
+      return Maps.uniqueIndex(wellKnowns, new Function<WellKnownImage, String>() {
+
+         @Override
+         public String apply(WellKnownImage input) {
+            return input.getUuid();
+         }
+
+      });
    }
 
    @Provides

elasticstack/src/main/java/org/jclouds/elasticstack/compute/functions/ServerInfoToNodeMetadata.java

@@ -132,6 +132,13 @@ public class ServerInfoToNodeMetadata implements Function<ServerInfo, NodeMetada
       }
    }
 
+   /**
+    * When we create the boot drive of the server, by convention we set the name to the image it
+    * came from.
+    * 
+    * @author Adrian Cole
+    * 
+    */
    @Singleton
    public static class GetImageIdFromServer implements Function<Server, String> {
       private final Map<String, DriveInfo> cache;
@@ -148,7 +155,7 @@ public class ServerInfoToNodeMetadata implements Function<ServerInfo, NodeMetada
          Device bootDevice = from.getDevices().get(bootDeviceId);
          if (bootDevice != null) {
             try {
-               imageId = cache.get(bootDevice.getDriveUuid()).getUuid();
+               imageId = cache.get(bootDevice.getDriveUuid()).getName();
             } catch (NullPointerException e) {
 
             }