Issue 412: parallel image fetches

This commit is contained in:
Adrian Cole 2010-12-20 14:37:24 +01:00
parent 2760345bd3
commit 5587fc6635
4 changed files with 71 additions and 58 deletions

View File

@ -81,7 +81,7 @@ public interface ElasticStackAsyncClient {
@GET
@Path("/servers/info")
@ResponseParser(ListOfKeyValuesDelimitedByBlankLinesToServerInfoSet.class)
ListenableFuture<Set<? extends ServerInfo>> listServerInfo();
ListenableFuture<Set<ServerInfo>> listServerInfo();
/**
* @see ElasticStackClient#getServerInfo
@ -90,7 +90,7 @@ public interface ElasticStackAsyncClient {
@ExceptionParser(ReturnNullOnNotFoundOr404.class)
@ResponseParser(KeyValuesDelimitedByBlankLinesToServerInfo.class)
@Path("/servers/{uuid}/info")
ListenableFuture<? extends ServerInfo> getServerInfo(@PathParam("uuid") String uuid);
ListenableFuture<ServerInfo> getServerInfo(@PathParam("uuid") String uuid);
/**
* @see ElasticStackClient#createServer
@ -99,7 +99,7 @@ public interface ElasticStackAsyncClient {
@ExceptionParser(ReturnNullOnNotFoundOr404.class)
@ResponseParser(KeyValuesDelimitedByBlankLinesToServerInfo.class)
@Path("/servers/create/stopped")
ListenableFuture<? extends ServerInfo> createServer(
ListenableFuture<ServerInfo> createServer(
@BinderParam(BindServerToPlainTextString.class) Server createServer);
/**
@ -109,7 +109,7 @@ public interface ElasticStackAsyncClient {
@ExceptionParser(ReturnNullOnNotFoundOr404.class)
@ResponseParser(KeyValuesDelimitedByBlankLinesToServerInfo.class)
@Path("/servers/{uuid}/set")
ListenableFuture<? extends ServerInfo> setServerConfiguration(@PathParam("uuid") String uuid,
ListenableFuture<ServerInfo> setServerConfiguration(@PathParam("uuid") String uuid,
@BinderParam(BindServerToPlainTextString.class) Server setServer);
/**
@ -162,7 +162,7 @@ public interface ElasticStackAsyncClient {
@GET
@Path("/drives/info")
@ResponseParser(ListOfKeyValuesDelimitedByBlankLinesToDriveInfoSet.class)
ListenableFuture<Set<? extends DriveInfo>> listDriveInfo();
ListenableFuture<Set<DriveInfo>> listDriveInfo();
/**
* @see ElasticStackClient#getDriveInfo
@ -171,7 +171,7 @@ public interface ElasticStackAsyncClient {
@ExceptionParser(ReturnNullOnNotFoundOr404.class)
@ResponseParser(KeyValuesDelimitedByBlankLinesToDriveInfo.class)
@Path("/drives/{uuid}/info")
ListenableFuture<? extends DriveInfo> getDriveInfo(@PathParam("uuid") String uuid);
ListenableFuture<DriveInfo> getDriveInfo(@PathParam("uuid") String uuid);
/**
* @see ElasticStackClient#createDrive
@ -180,7 +180,7 @@ public interface ElasticStackAsyncClient {
@ExceptionParser(ReturnNullOnNotFoundOr404.class)
@ResponseParser(KeyValuesDelimitedByBlankLinesToDriveInfo.class)
@Path("/drives/create")
ListenableFuture<? extends DriveInfo> createDrive(@BinderParam(BindDriveToPlainTextString.class) Drive createDrive);
ListenableFuture<DriveInfo> createDrive(@BinderParam(BindDriveToPlainTextString.class) Drive createDrive);
/**
* @see ElasticStackClient#setDriveData
@ -189,7 +189,7 @@ public interface ElasticStackAsyncClient {
@ExceptionParser(ReturnNullOnNotFoundOr404.class)
@ResponseParser(KeyValuesDelimitedByBlankLinesToDriveInfo.class)
@Path("/drives/{uuid}/set")
ListenableFuture<? extends DriveInfo> setDriveData(@PathParam("uuid") String uuid,
ListenableFuture<DriveInfo> setDriveData(@PathParam("uuid") String uuid,
@BinderParam(BindDriveDataToPlainTextString.class) DriveData setDrive);
/**
@ -207,7 +207,7 @@ public interface ElasticStackAsyncClient {
@ExceptionParser(ReturnNullOnNotFoundOr404.class)
@ResponseParser(KeyValuesDelimitedByBlankLinesToServerInfo.class)
@Path("/servers/create")
ListenableFuture<? extends ServerInfo> createAndStartServer(
ListenableFuture<ServerInfo> createAndStartServer(
@BinderParam(BindServerToPlainTextString.class) Server createServer);
/**

View File

@ -3,25 +3,25 @@ package org.jclouds.elasticstack.compute;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Predicates.notNull;
import static com.google.common.collect.Iterables.filter;
import static com.google.common.collect.Iterables.transform;
import static org.jclouds.concurrent.FutureIterables.transformParallel;
import static org.jclouds.elasticstack.util.Servers.small;
import java.net.URI;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import javax.annotation.Resource;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import org.jclouds.Constants;
import org.jclouds.compute.ComputeService;
import org.jclouds.compute.ComputeServiceAdapter;
import org.jclouds.compute.domain.Hardware;
import org.jclouds.compute.domain.HardwareBuilder;
import org.jclouds.compute.domain.Image;
import org.jclouds.compute.domain.ImageBuilder;
import org.jclouds.compute.domain.OperatingSystemBuilder;
import org.jclouds.compute.domain.Processor;
import org.jclouds.compute.domain.Template;
import org.jclouds.compute.domain.Volume;
@ -31,6 +31,7 @@ import org.jclouds.domain.Credentials;
import org.jclouds.domain.Location;
import org.jclouds.domain.LocationScope;
import org.jclouds.domain.internal.LocationImpl;
import org.jclouds.elasticstack.ElasticStackAsyncClient;
import org.jclouds.elasticstack.ElasticStackClient;
import org.jclouds.elasticstack.domain.Device;
import org.jclouds.elasticstack.domain.Drive;
@ -45,9 +46,7 @@ import org.jclouds.rest.annotations.Provider;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSet.Builder;
@ -57,31 +56,37 @@ import com.google.common.collect.ImmutableSet.Builder;
*
*/
@Singleton
public class ElasticStackComputeServiceAdapter implements ComputeServiceAdapter<ServerInfo, Hardware, Image, Location> {
public class ElasticStackComputeServiceAdapter implements
ComputeServiceAdapter<ServerInfo, Hardware, DriveInfo, Location> {
private final ElasticStackClient client;
private final ElasticStackAsyncClient aclient;
private final Predicate<DriveInfo> driveNotClaimed;
private final Supplier<Location> locationSupplier;
private final List<WellKnownImage> preinstalledImages;
private final Map<String, WellKnownImage> preinstalledImages;
private final Map<String, DriveInfo> cache;
private final String providerName;
private final URI providerURI;
private final String defaultVncPassword;
private final ExecutorService executor;
@Resource
@Named(ComputeServiceConstants.COMPUTE_LOGGER)
protected Logger logger = Logger.NULL;
@Inject
public ElasticStackComputeServiceAdapter(ElasticStackClient client, Predicate<DriveInfo> driveNotClaimed,
Supplier<Location> locationSupplier, @Provider String providerName, @Provider URI providerURI,
List<WellKnownImage> preinstalledImages,
@Named(ElasticStackConstants.PROPERTY_VNC_PASSWORD) String defaultVncPassword) {
public ElasticStackComputeServiceAdapter(ElasticStackClient client, ElasticStackAsyncClient aclient,
Predicate<DriveInfo> driveNotClaimed, @Provider String providerName, @Provider URI providerURI,
Map<String, WellKnownImage> preinstalledImages, Map<String, DriveInfo> cache,
@Named(ElasticStackConstants.PROPERTY_VNC_PASSWORD) String defaultVncPassword,
@Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) {
this.client = checkNotNull(client, "client");
this.aclient = checkNotNull(aclient, "aclient");
this.driveNotClaimed = checkNotNull(driveNotClaimed, "driveNotClaimed");
this.locationSupplier = checkNotNull(locationSupplier, "locationSupplier");
this.providerName = checkNotNull(providerName, "providerName");
this.providerURI = checkNotNull(providerURI, "providerURI");
this.preinstalledImages = checkNotNull(preinstalledImages, "preinstalledImages");
this.cache = checkNotNull(cache, "cache");
this.defaultVncPassword = checkNotNull(defaultVncPassword, "defaultVncPassword");
this.executor = checkNotNull(executor, "executor");
}
@Override
@ -125,6 +130,11 @@ public class ElasticStackComputeServiceAdapter implements ComputeServiceAdapter<
return (toParse != null && new Float(toParse) <= size);
}
@Override
public String toString() {
return "sizeLessThanOrEqual(" + size + ")";
}
}).ids(id).ram(ram).processors(ImmutableList.of(new Processor(1, cpu)))
.volumes(ImmutableList.<Volume> of(new VolumeImpl(size, true, true))).build());
}
@ -135,37 +145,20 @@ public class ElasticStackComputeServiceAdapter implements ComputeServiceAdapter<
* look up the current standard images and do not error out, if they are not found.
*/
@Override
public Iterable<Image> listImages() {
return filter(transform(preinstalledImages, new Function<WellKnownImage, Image>() {
public Iterable<DriveInfo> listImages() {
Iterable<DriveInfo> drives = transformParallel(preinstalledImages.keySet(),
new Function<String, Future<DriveInfo>>() {
@Override
public Image apply(WellKnownImage input) {
DriveInfo drive = null;
try {
drive = client.getDriveInfo(input.getUuid());
} catch (Exception e) {
logger.warn(e, "could not find image: %s", input);
}
if (drive == null) {
logger.warn("could not find image: %s", input);
return null;
}
return new ImageBuilder()
.ids(drive.getUuid())
.userMetadata(
ImmutableMap.<String, String> builder().putAll(drive.getUserMetadata())
.put("size", input.getSize() + "").build())
.defaultCredentials(new Credentials("toor", null))
.location(locationSupplier.get())
.name(input.getDescription())
.description(drive.getName())
.operatingSystem(
new OperatingSystemBuilder().family(input.getOsFamily()).version(input.getOsVersion())
.name(input.getDescription()).description(drive.getName()).is64Bit(true).build())
.version("").build();
}
@Override
public Future<DriveInfo> apply(String input) {
return aclient.getDriveInfo(input);
}
}), notNull());
}, executor, null, logger, "drives");
Iterable<DriveInfo> returnVal = filter(drives, notNull());
for (DriveInfo drive : returnVal)
cache.put(drive.getUuid(), drive);
return returnVal;
}
@SuppressWarnings("unchecked")

View File

@ -44,6 +44,7 @@ import org.jclouds.elasticstack.compute.functions.ServerInfoToNodeMetadata;
import org.jclouds.elasticstack.compute.functions.ServerInfoToNodeMetadata.DeviceToVolume;
import org.jclouds.elasticstack.compute.functions.ServerInfoToNodeMetadata.FindImageForId;
import org.jclouds.elasticstack.compute.functions.ServerInfoToNodeMetadata.GetImageIdFromServer;
import org.jclouds.elasticstack.compute.functions.WellKnownImageToImage;
import org.jclouds.elasticstack.domain.Device;
import org.jclouds.elasticstack.domain.DriveInfo;
import org.jclouds.elasticstack.domain.Server;
@ -59,6 +60,7 @@ import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Supplier;
import com.google.common.collect.MapMaker;
import com.google.common.collect.Maps;
import com.google.inject.Provides;
import com.google.inject.TypeLiteral;
@ -68,7 +70,7 @@ import com.google.inject.TypeLiteral;
*/
public class ElasticStackComputeServiceContextModule
extends
ComputeServiceAdapterContextModule<ElasticStackClient, ElasticStackAsyncClient, ServerInfo, Hardware, Image, Location> {
ComputeServiceAdapterContextModule<ElasticStackClient, ElasticStackAsyncClient, ServerInfo, Hardware, DriveInfo, Location> {
public ElasticStackComputeServiceContextModule() {
super(ElasticStackClient.class, ElasticStackAsyncClient.class);
@ -78,7 +80,7 @@ public class ElasticStackComputeServiceContextModule
@Override
protected void configure() {
super.configure();
bind(new TypeLiteral<ComputeServiceAdapter<ServerInfo, Hardware, Image, Location>>() {
bind(new TypeLiteral<ComputeServiceAdapter<ServerInfo, Hardware, DriveInfo, Location>>() {
}).to(ElasticStackComputeServiceAdapter.class);
bind(IdentityFunction.class).toInstance(IdentityFunction.INSTANCE);
bind(new TypeLiteral<Supplier<Location>>() {
@ -97,12 +99,14 @@ public class ElasticStackComputeServiceContextModule
}).to(GetImageIdFromServer.class);
bind(new TypeLiteral<Function<String, Image>>() {
}).to(FindImageForId.class);
bind(new TypeLiteral<Function<DriveInfo, Image>>() {
}).to(WellKnownImageToImage.class);
}
@Provides
@Singleton
protected Map<String, DriveInfo> cache(GetDrive getDrive) {
return new MapMaker().expiration(30, TimeUnit.SECONDS).makeComputingMap(getDrive);
return new MapMaker().makeComputingMap(getDrive);
}
@Singleton
@ -122,10 +126,19 @@ public class ElasticStackComputeServiceContextModule
@Singleton
@Provides
protected List<WellKnownImage> provideImages(Json json) throws IOException {
return json.fromJson(Utils.toStringAndClose(getClass().getResourceAsStream("/preinstalled_images.json")),
protected Map<String, WellKnownImage> provideImages(Json json) throws IOException {
List<WellKnownImage> wellKnowns = json.fromJson(
Utils.toStringAndClose(getClass().getResourceAsStream("/preinstalled_images.json")),
new TypeLiteral<List<WellKnownImage>>() {
}.getType());
return Maps.uniqueIndex(wellKnowns, new Function<WellKnownImage, String>() {
@Override
public String apply(WellKnownImage input) {
return input.getUuid();
}
});
}
@Provides

View File

@ -132,6 +132,13 @@ public class ServerInfoToNodeMetadata implements Function<ServerInfo, NodeMetada
}
}
/**
* When we create the boot drive of the server, by convention we set the name to the image it
* came from.
*
* @author Adrian Cole
*
*/
@Singleton
public static class GetImageIdFromServer implements Function<Server, String> {
private final Map<String, DriveInfo> cache;
@ -148,7 +155,7 @@ public class ServerInfoToNodeMetadata implements Function<ServerInfo, NodeMetada
Device bootDevice = from.getDevices().get(bootDeviceId);
if (bootDevice != null) {
try {
imageId = cache.get(bootDevice.getDriveUuid()).getUuid();
imageId = cache.get(bootDevice.getDriveUuid()).getName();
} catch (NullPointerException e) {
}