From 34518f54badeafe4c250bc102bb5f936ab1f51f4 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Fri, 30 Jul 2010 01:26:55 -0700 Subject: [PATCH] Issue 325: massive refactor in order to stop serializing native async http client requests --- allcompute/pom.xml | 10 + .../src/main/java/__providerName__Client.java | 2 +- .../saas/AtmosStorageAsyncClient.java | 4 +- .../saas/blobstore/AtmosAsyncBlobStore.java | 56 +-- .../strategy/FindMD5InUserMetadata.java | 30 +- .../internal/StubAtmosStorageAsyncClient.java | 76 ++- .../EC2ComputeServiceContextModule.java | 166 ++++--- .../internal/EC2TemplateBuilderImpl.java | 9 +- .../strategy/DescribeImagesParallel.java | 79 ++++ .../EC2RunNodesAndAddToSetStrategy.java | 46 +- .../jclouds/aws/ec2/domain/Reservation.java | 13 +- .../aws/ec2/options/CreateImageOptions.java | 2 +- .../DescribeAvailabilityZonesOptions.java | 2 +- .../ec2/options/DescribeImagesOptions.java | 2 +- .../ec2/options/DescribeRegionsOptions.java | 2 +- .../aws/ec2/options/RunInstancesOptions.java | 2 +- .../aws/ec2/services/AMIAsyncClient.java | 20 +- .../jclouds/aws/ec2/services/AMIClient.java | 2 +- .../aws/ec2/services/InstanceAsyncClient.java | 32 +- .../aws/ec2/services/InstanceClient.java | 10 +- .../aws/ec2/xml/BaseReservationHandler.java | 5 +- .../xml/DescribeInstancesResponseHandler.java | 7 +- .../ec2/xml/RunInstancesResponseHandler.java | 5 +- .../java/org/jclouds/aws/s3/S3Client.java | 10 +- .../aws/s3/blobstore/S3AsyncBlobStore.java | 58 +-- .../aws/s3/options/CopyObjectOptions.java | 2 +- .../aws/s3/options/ListBucketOptions.java | 2 +- .../aws/s3/options/PutBucketOptions.java | 2 +- .../aws/s3/options/PutObjectOptions.java | 2 +- ...icationArchitecturesEC2ClientLiveTest.java | 256 ++++------ .../aws/ec2/EBSBootEC2ClientLiveTest.java | 144 +++--- .../functions/RegionAndIdToImageTest.java | 10 +- .../RunningInstanceToNodeMetadataTest.java | 54 +-- .../EC2RunNodesAndAddToSetStrategyTest.java | 64 +-- .../ec2/services/InstanceClientLiveTest.java | 7 +- .../DescribeInstancesResponseHandlerTest.java | 83 ++-- .../xml/RunInstancesResponseHandlerTest.java | 35 +- .../aws/s3/internal/StubS3AsyncClient.java | 76 ++- .../aws/ec2/demos/createlamp/MainApp.java | 52 +-- .../java/org/jclouds/aws/sqs/SpeedTest.java | 6 +- .../aws/s3/jets3t/JCloudsS3Service.java | 2 +- .../azure/storage/blob/AzureBlobClient.java | 6 +- .../blob/blobstore/AzureAsyncBlobStore.java | 48 +- .../azure/storage/queue/AzureQueueClient.java | 6 +- .../org/jclouds/blobstore/AsyncBlobStore.java | 16 +- .../blobstore/TransientAsyncBlobStore.java | 93 ++-- .../internal/BaseAsyncBlobStore.java | 44 +- .../blobstore/internal/BaseBlobMap.java | 4 +- .../blobstore/internal/BlobMapImpl.java | 11 +- .../jclouds/blobstore/options/GetOptions.java | 2 +- .../options/ListContainerOptions.java | 2 +- .../blobstore/options/ListOptions.java | 2 +- .../strategy/GetBlobsInListStrategy.java | 4 +- .../strategy/ListBlobsInContainer.java | 4 +- .../internal/DeleteAllKeysInList.java | 4 +- .../strategy/internal/FetchBlobMetadata.java | 68 +-- .../GetAllBlobsInListAndRetryOnFailure.java | 65 +-- ...ListContainerAndRecurseThroughFolders.java | 24 +- .../MarkersDeleteDirectoryStrategy.java | 4 +- .../internal/PutBlobsStrategyImpl.java | 4 +- .../internal/BaseBlobIntegrationTest.java | 54 +-- .../internal/BaseMapIntegrationTest.java | 96 ++-- .../jclouds/boxdotnet/BoxDotNetClient.java | 2 +- .../java/org/jclouds/chef/ChefService.java | 8 +- .../chef/internal/BaseChefService.java | 8 +- .../org/jclouds/chef/strategy/GetNodes.java | 8 +- .../CleanupStaleNodesAndClientsImpl.java | 2 +- .../DeleteAllClientsAndNodesInListImpl.java | 4 +- .../chef/strategy/internal/GetNodesImpl.java | 53 +-- .../internal/GetNodesImplLiveTest.java | 9 +- .../servlet/ChefRegistrationListener.java | 28 +- .../functions/InitParamsToProperties.java | 47 ++ .../compute/domain/TemplateBuilder.java | 5 + .../compute/internal/BaseComputeService.java | 121 ++--- .../compute/internal/TemplateBuilderImpl.java | 86 ++-- .../strategy/RunNodesAndAddToSetStrategy.java | 4 +- ...agIntoNameRunNodesAndAddToSetStrategy.java | 78 ++-- .../StubComputeServiceContextModule.java | 68 ++- .../jclouds/compute/util/ComputeUtils.java | 63 +-- .../compute/BaseComputeServiceLiveTest.java | 119 ++--- .../StubComputeServiceIntegrationTest.java | 45 +- .../internal/TemplateBuilderImplTest.java | 81 ++-- .../jclouds/concurrent/ConcurrentUtils.java | 438 ++++++++++++++++-- ... => ExceptionParsingListenableFuture.java} | 35 +- .../RetryOnTimeOutExceptionSupplier.java | 13 +- .../config/ExecutorServiceModule.java | 41 +- .../http/HttpCommandExecutorService.java | 6 +- .../jclouds/http/HttpCommandRendezvous.java | 10 +- ...ransformingHttpCommandExecutorService.java | 4 +- ...formingHttpCommandExecutorServiceImpl.java | 20 +- .../handlers/BackoffLimitedRetryHandler.java | 17 +- .../BaseHttpCommandExecutorService.java | 20 +- core/src/main/java/org/jclouds/json/Json.java | 13 +- .../jclouds/json/internal/GsonWrapper.java | 5 + .../java/org/jclouds/rest/RestContext.java | 4 +- .../org/jclouds/rest/RestContextBuilder.java | 2 +- .../org/jclouds/rest/RestContextFactory.java | 177 +++---- .../rest/internal/AsyncRestClientProxy.java | 18 +- .../internal/RestAnnotationProcessor.java | 104 ++--- .../src/main/java/org/jclouds/util/Utils.java | 88 ++-- .../concurrent/ConcurrentUtilsTest.java | 8 +- .../concurrent/FutureExceptionParserTest.java | 28 +- .../FuturesComposePerformanceTest.java | 161 ++++++- .../concurrent/FuturesTestingUtils.java | 66 ++- .../RetryOnTimeOutExceptionSupplierTest.java | 2 +- .../concurrent/internal/SyncProxyTest.java | 21 +- .../java/org/jclouds/http/BaseJettyTest.java | 9 + .../BackoffLimitedRetryHandlerTest.java | 34 +- .../jclouds/http/internal/WireLiveTest.java | 5 +- .../org/jclouds/rest/BaseRestClientTest.java | 2 +- .../jclouds/rest/RestContextFactoryTest.java | 4 +- .../internal/RestAnnotationProcessorTest.java | 272 +++++------ .../jclouds/azure/azurequeue/SpeedTest.java | 6 +- .../AsyncGaeHttpCommandExecutorService.java | 111 +++-- .../GoogleAppEngineConfigurationModule.java | 2 +- ...CommandExecutorServiceIntegrationTest.java | 206 ++++++-- ...loperCloudComputeServiceContextModule.java | 4 +- .../mezeo/pcs2/options/PutBlockOptions.java | 2 +- .../functions/AddMetadataAndReturnIdTest.java | 2 +- .../sdn/SDNAuthenticationLiveTest.java | 2 +- .../filters/AddSessionTokenToRequestTest.java | 2 +- .../InsertUserContextIntoPathTest.java | 2 +- .../OpscodePlatformAsyncClientTest.java | 2 +- .../cloudfiles/CloudFilesClient.java | 6 +- .../blobstore/CloudFilesAsyncBlobStore.java | 53 +-- .../cloudservers/CloudServersClient.java | 6 +- .../RackspaceAuthenticationRestModule.java | 4 +- .../RackspaceAuthenticationLiveTest.java | 2 +- .../internal/StubCloudFilesAsyncClient.java | 40 +- ...LockVCloudComputeServiceContextModule.java | 30 +- ...ParseVAppTemplatesInVDCToSizeProvider.java | 24 +- ...eVAppTemplatesInVDCToSizeProviderTest.java | 25 +- .../VCloudComputeServiceContextModule.java | 14 +- .../config/providers/VAppTemplatesInVDCs.java | 72 +++ .../config/providers/VCloudImageProvider.java | 125 ----- ...nVDC.java => FindLocationForResource.java} | 41 +- .../functions/ImageForVAppTemplate.java | 72 +++ .../functions/ImagesInOrganization.java | 51 ++ .../vcloud/compute/functions/ImagesInVDC.java | 55 +++ .../functions/VCloudGetNodeMetadata.java | 27 +- .../strategy/VCloudListNodesStrategy.java | 43 +- .../functions/AllCatalogItemsInCatalog.java | 84 ++++ .../AllCatalogItemsInOrganization.java | 67 +++ .../functions/AllCatalogsInOrganization.java | 75 +++ .../functions/AllVDCsInOrganization.java | 74 +++ .../functions/OrganizatonsForLocations.java | 85 ++++ .../VAppTemplatesForCatalogItems.java | 81 ++++ .../VAppTemplatesForResourceEntities.java | 81 ++++ .../jclouds/vcloud/VCloudLoginLiveTest.java | 2 +- .../vcloud/VCloudVersionsLiveTest.java | 2 +- ...markVCloudComputeServiceContextModule.java | 4 +- ...ppTemplatesAndConvertToImagesProvider.java | 132 ------ .../config/providers/VAppTemplatesInOrgs.java | 72 +++ .../functions/ImagesInOrganization.java | 62 +++ ...ewKeyPairUnlessUserSpecifiedOtherwise.java | 14 +- ...scriptionToGetDefaultLoginCredentials.java | 14 +- ...IdIntoNameRunNodesAndAddToSetStrategy.java | 33 +- .../VAppTemplatesInOrgsLiveTest.java | 92 ++++ 158 files changed, 3866 insertions(+), 2613 deletions(-) create mode 100644 aws/core/src/main/java/org/jclouds/aws/ec2/compute/strategy/DescribeImagesParallel.java create mode 100644 chef/servlet/src/main/java/org/jclouds/chef/servlet/functions/InitParamsToProperties.java rename core/src/main/java/org/jclouds/concurrent/{FutureExceptionParser.java => ExceptionParsingListenableFuture.java} (68%) create mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/compute/config/providers/VAppTemplatesInVDCs.java delete mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/compute/config/providers/VCloudImageProvider.java rename vcloud/core/src/main/java/org/jclouds/vcloud/compute/functions/{FindLocationForResourceInVDC.java => FindLocationForResource.java} (59%) create mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/compute/functions/ImageForVAppTemplate.java create mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/compute/functions/ImagesInOrganization.java create mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/compute/functions/ImagesInVDC.java create mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/functions/AllCatalogItemsInCatalog.java create mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/functions/AllCatalogItemsInOrganization.java create mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/functions/AllCatalogsInOrganization.java create mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/functions/AllVDCsInOrganization.java create mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/functions/OrganizatonsForLocations.java create mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/functions/VAppTemplatesForCatalogItems.java create mode 100644 vcloud/core/src/main/java/org/jclouds/vcloud/functions/VAppTemplatesForResourceEntities.java delete mode 100644 vcloud/terremark/src/main/java/org/jclouds/vcloud/terremark/compute/config/providers/QueryCatalogForVAppTemplatesAndConvertToImagesProvider.java create mode 100644 vcloud/terremark/src/main/java/org/jclouds/vcloud/terremark/compute/config/providers/VAppTemplatesInOrgs.java create mode 100644 vcloud/terremark/src/main/java/org/jclouds/vcloud/terremark/compute/functions/ImagesInOrganization.java create mode 100644 vcloud/terremark/src/test/java/org/jclouds/vcloud/terremark/providers/VAppTemplatesInOrgsLiveTest.java diff --git a/allcompute/pom.xml b/allcompute/pom.xml index f878476495..760cb00133 100644 --- a/allcompute/pom.xml +++ b/allcompute/pom.xml @@ -49,6 +49,16 @@ jclouds-vcloud ${project.version} + + ${project.groupId} + jclouds-bluelock + ${project.version} + + + ${project.groupId} + jclouds-terremark + ${project.version} + ${project.groupId} jclouds-gogrid diff --git a/archetypes/json-client-archetype/src/main/resources/archetype-resources/src/main/java/__providerName__Client.java b/archetypes/json-client-archetype/src/main/resources/archetype-resources/src/main/java/__providerName__Client.java index f739311897..58fbf97ee8 100644 --- a/archetypes/json-client-archetype/src/main/resources/archetype-resources/src/main/java/__providerName__Client.java +++ b/archetypes/json-client-archetype/src/main/resources/archetype-resources/src/main/java/__providerName__Client.java @@ -43,7 +43,7 @@ public interface ${providerName}Client { /* * Note all these delegate to methods in ${providerName}AsyncClient with a specified or inherited timeout. * The singatures should match those of ${providerName}AsyncClient, except the returnvals should not be - * wrapped in a ListenableFuture + * wrapped in a Future */ String list(); diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageAsyncClient.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageAsyncClient.java index 181526c74f..7fcd486c12 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageAsyncClient.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/AtmosStorageAsyncClient.java @@ -82,7 +82,7 @@ public interface AtmosStorageAsyncClient { @Path("/rest/namespace") @ResponseParser(ParseDirectoryListFromContentAndHeaders.class) @Consumes(MediaType.TEXT_XML) - ListenableFuture> listDirectories( + ListenableFuture> listDirectories( ListOptions... options); /** @@ -93,7 +93,7 @@ public interface AtmosStorageAsyncClient { @ResponseParser(ParseDirectoryListFromContentAndHeaders.class) @ExceptionParser(ThrowContainerNotFoundOn404.class) @Consumes(MediaType.TEXT_XML) - ListenableFuture> listDirectory( + ListenableFuture> listDirectory( @PathParam("directoryName") String directoryName, ListOptions... options); /** diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java index 70312be621..85096fbc03 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/AtmosAsyncBlobStore.java @@ -19,7 +19,8 @@ package org.jclouds.atmosonline.saas.blobstore; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.util.concurrent.Futures.compose; +import static org.jclouds.concurrent.ConcurrentUtils.compose; +import static org.jclouds.concurrent.ConcurrentUtils.makeListenable; import java.net.URI; import java.util.Set; @@ -53,7 +54,6 @@ import org.jclouds.blobstore.functions.BlobToHttpGetOptions; import org.jclouds.blobstore.internal.BaseAsyncBlobStore; import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata; import org.jclouds.blobstore.util.BlobUtils; -import org.jclouds.concurrent.ConcurrentUtils; import org.jclouds.domain.Location; import org.jclouds.encryption.EncryptionService; import org.jclouds.http.options.GetOptions; @@ -79,15 +79,12 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { @Inject AtmosAsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, - Location defaultLocation, Set locations, - - AtmosStorageAsyncClient async, AtmosStorageClient sync, ObjectToBlob object2Blob, - ObjectToBlobMetadata object2BlobMd, BlobToObject blob2Object, + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Location defaultLocation, + Set locations, AtmosStorageAsyncClient async, AtmosStorageClient sync, + ObjectToBlob object2Blob, ObjectToBlobMetadata object2BlobMd, BlobToObject blob2Object, BlobStoreListOptionsToListOptions container2ContainerListOptions, - DirectoryEntryListToResourceMetadataList container2ResourceList, - EncryptionService encryptionService, BlobToHttpGetOptions blob2ObjectGetOptions, - Provider fetchBlobMetadataProvider) { + DirectoryEntryListToResourceMetadataList container2ResourceList, EncryptionService encryptionService, + BlobToHttpGetOptions blob2ObjectGetOptions, Provider fetchBlobMetadataProvider) { super(context, blobUtils, service, defaultLocation, locations); this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions"); this.sync = checkNotNull(sync, "sync"); @@ -99,8 +96,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { this.blob2Object = checkNotNull(blob2Object, "blob2Object"); this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd"); this.encryptionService = checkNotNull(encryptionService, "encryptionService"); - this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, - "fetchBlobMetadataProvider"); + this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider"); } /** @@ -108,13 +104,12 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { */ @Override public ListenableFuture blobMetadata(String container, String key) { - return compose(async.headFile(container + "/" + key), - new Function() { - @Override - public BlobMetadata apply(AtmosObject from) { - return object2BlobMd.apply(from); - } - }, service); + return compose(async.headFile(container + "/" + key), new Function() { + @Override + public BlobMetadata apply(AtmosObject from) { + return object2BlobMd.apply(from); + } + }, service); } /** @@ -130,7 +125,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { return true; } - }); + }, service); } /** @@ -144,7 +139,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { return null;// no etag } - }); + }, service); } /** @@ -197,8 +192,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { * This implementation invokes {@link AtmosStorageAsyncClient#readFile} */ @Override - public ListenableFuture getBlob(String container, String key, - org.jclouds.blobstore.options.GetOptions options) { + public ListenableFuture getBlob(String container, String key, org.jclouds.blobstore.options.GetOptions options) { GetOptions httpOptions = blob2ObjectGetOptions.apply(options); ListenableFuture returnVal = async.readFile(container + "/" + key, httpOptions); return compose(returnVal, object2Blob, service); @@ -208,7 +202,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { * This implementation invokes {@link AtmosStorageAsyncClient#listDirectories} */ @Override - public ListenableFuture> list() { + public ListenableFuture> list() { return compose(async.listDirectories(), container2ResourceList, service); } @@ -216,16 +210,14 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { * This implementation invokes {@link AtmosStorageAsyncClient#listDirectory} */ @Override - public ListenableFuture> list(String container, + public ListenableFuture> list(String container, org.jclouds.blobstore.options.ListContainerOptions options) { container = AtmosStorageUtils.adjustContainerIfDirOptionPresent(container, options); ListOptions nativeOptions = container2ContainerListOptions.apply(options); - ListenableFuture> returnVal = async - .listDirectory(container, nativeOptions); - ListenableFuture> list = compose(returnVal, - container2ResourceList, service); - return options.isDetailed() ? compose(list, fetchBlobMetadataProvider.get().setContainerName( - container), service) : list; + ListenableFuture> returnVal = async.listDirectory(container, nativeOptions); + ListenableFuture> list = compose(returnVal, container2ResourceList, service); + return (ListenableFuture>) (options.isDetailed() ? compose(list, + fetchBlobMetadataProvider.get().setContainerName(container), service) : list); } /** @@ -235,7 +227,7 @@ public class AtmosAsyncBlobStore extends BaseAsyncBlobStore { */ @Override public ListenableFuture putBlob(final String container, final Blob blob) { - return ConcurrentUtils.makeListenable(service.submit(new Callable() { + return makeListenable(service.submit(new Callable() { @Override public String call() throws Exception { diff --git a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java index 614595b7ed..d9e764ce5e 100644 --- a/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java +++ b/atmos/src/main/java/org/jclouds/atmosonline/saas/blobstore/strategy/FindMD5InUserMetadata.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; @@ -42,6 +43,7 @@ import org.jclouds.blobstore.internal.BlobRuntimeException; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.strategy.ContainsValueInListStrategy; import org.jclouds.blobstore.strategy.ListBlobsInContainer; +import static org.jclouds.concurrent.ConcurrentUtils.*; import org.jclouds.logging.Logger; import com.google.common.base.Throwables; @@ -70,10 +72,8 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy { protected Long maxTime; @Inject - private FindMD5InUserMetadata( - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, - ObjectMD5 objectMD5, ListBlobsInContainer getAllBlobMetadata, - AtmosStorageAsyncClient client) { + private FindMD5InUserMetadata(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, + ObjectMD5 objectMD5, ListBlobsInContainer getAllBlobMetadata, AtmosStorageAsyncClient client) { this.objectMD5 = objectMD5; this.getAllBlobMetadata = getAllBlobMetadata; this.client = client; @@ -83,10 +83,10 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy { public boolean execute(final String containerName, Object value, ListContainerOptions options) { final byte[] toSearch = objectMD5.apply(value); final BlockingQueue queue = new SynchronousQueue(); - Map> responses = Maps.newHashMap(); + Map> responses = Maps.newHashMap(); for (BlobMetadata md : getAllBlobMetadata.execute(containerName, options)) { - final ListenableFuture future = client.headFile(containerName + "/" - + md.getName()); + final ListenableFuture future = makeListenable(client.headFile(containerName + + "/" + md.getName()), userExecutor); future.addListener(new Runnable() { public void run() { try { @@ -97,8 +97,7 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy { queue.put(true); } } else { - logger.debug("object %s has no content md5", object.getSystemMetadata() - .getObjectID()); + logger.debug("object %s has no content md5", object.getSystemMetadata().getObjectID()); } } catch (InterruptedException e) { Throwables.propagate(e); @@ -109,11 +108,11 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy { }, userExecutor); responses.put(md.getName(), future); } - Map exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, - String.format("searching for md5 in container %s", containerName)); + Map exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, String.format( + "searching for md5 in container %s", containerName)); if (exceptions.size() > 0) - throw new BlobRuntimeException(String.format("searching for md5 in container %s: %s", - containerName, exceptions)); + throw new BlobRuntimeException(String.format("searching for md5 in container %s: %s", containerName, + exceptions)); try { return queue.poll(1, TimeUnit.MICROSECONDS) != null; } catch (InterruptedException e) { @@ -121,9 +120,8 @@ public class FindMD5InUserMetadata implements ContainsValueInListStrategy { return false; } catch (Exception e) { Throwables.propagateIfPossible(e, BlobRuntimeException.class); - throw new BlobRuntimeException(String.format( - "Error searching for ETAG of value: [%s] in container:%s", value, containerName), - e); + throw new BlobRuntimeException(String.format("Error searching for ETAG of value: [%s] in container:%s", value, + containerName), e); } } } diff --git a/atmos/src/test/java/org/jclouds/atmosonline/saas/internal/StubAtmosStorageAsyncClient.java b/atmos/src/test/java/org/jclouds/atmosonline/saas/internal/StubAtmosStorageAsyncClient.java index a08f3b31c0..14dbe07d89 100644 --- a/atmos/src/test/java/org/jclouds/atmosonline/saas/internal/StubAtmosStorageAsyncClient.java +++ b/atmos/src/test/java/org/jclouds/atmosonline/saas/internal/StubAtmosStorageAsyncClient.java @@ -19,15 +19,18 @@ package org.jclouds.atmosonline.saas.internal; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.util.concurrent.Futures.compose; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; +import static org.jclouds.concurrent.ConcurrentUtils.compose; import java.net.URI; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import javax.inject.Inject; +import javax.inject.Named; +import org.jclouds.Constants; import org.jclouds.atmosonline.saas.AtmosStorageAsyncClient; import org.jclouds.atmosonline.saas.blobstore.functions.BlobMetadataToObject; import org.jclouds.atmosonline.saas.blobstore.functions.BlobToObject; @@ -48,7 +51,6 @@ import org.jclouds.http.options.GetOptions; import com.google.common.base.Function; import com.google.common.base.Throwables; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; /** @@ -65,13 +67,13 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient { private final BlobMetadataToObject blob2ObjectInfo; private final ListOptionsToBlobStoreListOptions container2ContainerListOptions; private final ResourceMetadataListToDirectoryEntryList resource2ObjectList; + private final ExecutorService service; @Inject - private StubAtmosStorageAsyncClient(TransientAsyncBlobStore blobStore, - AtmosObject.Factory objectProvider, - HttpGetOptionsListToGetOptions httpGetOptionsConverter, ObjectToBlob object2Blob, - BlobToObject blob2Object, BlobMetadataToObject blob2ObjectInfo, - ListOptionsToBlobStoreListOptions container2ContainerListOptions, + private StubAtmosStorageAsyncClient(TransientAsyncBlobStore blobStore, AtmosObject.Factory objectProvider, + HttpGetOptionsListToGetOptions httpGetOptionsConverter, ObjectToBlob object2Blob, BlobToObject blob2Object, + BlobMetadataToObject blob2ObjectInfo, ListOptionsToBlobStoreListOptions container2ContainerListOptions, + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, ResourceMetadataListToDirectoryEntryList resource2ContainerList) { this.blobStore = blobStore; this.objectProvider = objectProvider; @@ -82,6 +84,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient { this.container2ContainerListOptions = checkNotNull(container2ContainerListOptions, "container2ContainerListOptions"); this.resource2ObjectList = checkNotNull(resource2ContainerList, "resource2ContainerList"); + this.service = service; } public ListenableFuture createDirectory(String directoryName) { @@ -94,25 +97,23 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient { container = directoryName; path = null; } - return Futures.compose(blobStore.createContainerInLocation(null, container), - new Function() { + return compose(blobStore.createContainerInLocation(null, container), new Function() { - public URI apply(Boolean from) { - if (path != null) { - Blob blob = blobStore.newBlob(path + "/"); - blob.getMetadata().setContentType("application/directory"); - blob.setPayload(""); - blobStore.putBlob(container, blob); - } - return URI.create("http://stub/containers/" + container); - } + public URI apply(Boolean from) { + if (path != null) { + Blob blob = blobStore.newBlob(path + "/"); + blob.getMetadata().setContentType("application/directory"); + blob.setPayload(""); + blobStore.putBlob(container, blob); + } + return URI.create("http://stub/containers/" + container); + } - }); + }, service); } public ListenableFuture createFile(String parent, AtmosObject object) { - final String uri = "http://stub/containers/" + parent + "/" - + object.getContentMetadata().getName(); + final String uri = "http://stub/containers/" + parent + "/" + object.getContentMetadata().getName(); String file = object.getContentMetadata().getName(); String container = parent; if (parent.indexOf('/') != -1) { @@ -128,7 +129,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient { return URI.create(uri); } - }); + }, service); } public ListenableFuture deletePath(String path) { @@ -141,7 +142,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient { return null; } - }); + }, service); } else { String container = path.substring(0, path.indexOf('/')); path = path.substring(path.indexOf('/') + 1); @@ -159,12 +160,11 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient { else { String container = path.substring(0, path.indexOf('/')); path = path.substring(path.indexOf('/') + 1); - return compose(blobStore.blobMetadata(container, path), - new Function() { - public UserMetadata apply(BlobMetadata from) { - return blob2ObjectInfo.apply(from).getUserMetadata(); - } - }); + return compose(blobStore.blobMetadata(container, path), new Function() { + public UserMetadata apply(BlobMetadata from) { + return blob2ObjectInfo.apply(from).getUserMetadata(); + } + }, service); } } @@ -172,23 +172,21 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient { String container = path.substring(0, path.indexOf('/')); path = path.substring(path.indexOf('/') + 1); try { - return Futures.compose(blobStore.getBlob(container, path), blob2Object); + return compose(blobStore.getBlob(container, path), blob2Object, service); } catch (Exception e) { return immediateFailedFuture(Throwables.getRootCause(e)); } } - public ListenableFuture> listDirectories( - ListOptions... optionsList) { + public ListenableFuture> listDirectories(ListOptions... optionsList) { // org.jclouds.blobstore.options.ListOptions options = container2ContainerListOptions // .apply(optionsList); - return Futures.compose(blobStore.list(), resource2ObjectList); + return compose(blobStore.list(), resource2ObjectList, service); } - public ListenableFuture> listDirectory( - String directoryName, ListOptions... optionsList) { - org.jclouds.blobstore.options.ListContainerOptions options = container2ContainerListOptions - .apply(optionsList); + public ListenableFuture> listDirectory(String directoryName, + ListOptions... optionsList) { + org.jclouds.blobstore.options.ListContainerOptions options = container2ContainerListOptions.apply(optionsList); String container = directoryName; if (directoryName.indexOf('/') != -1) { container = directoryName.substring(0, directoryName.indexOf('/')); @@ -196,7 +194,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient { if (!path.equals("")) options.inDirectory(path); } - return compose(blobStore.list(container, options), resource2ObjectList); + return compose(blobStore.list(container, options), resource2ObjectList, service); } public AtmosObject newObject() { @@ -224,7 +222,7 @@ public class StubAtmosStorageAsyncClient implements AtmosStorageAsyncClient { String container = path.substring(0, path.indexOf('/')); String blobName = path.substring(path.indexOf('/') + 1); org.jclouds.blobstore.options.GetOptions getOptions = httpGetOptionsConverter.apply(options); - return Futures.compose(blobStore.getBlob(container, blobName, getOptions), blob2Object); + return compose(blobStore.getBlob(container, blobName, getOptions), blob2Object, service); } public ListenableFuture updateFile(String parent, AtmosObject object) { diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java index 14427df55e..4a6c4e209b 100755 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/config/EC2ComputeServiceContextModule.java @@ -18,6 +18,16 @@ */ package org.jclouds.aws.ec2.compute.config; +import static com.google.common.collect.Iterables.concat; +import static com.google.common.collect.Iterables.filter; +import static com.google.common.collect.Iterables.find; +import static com.google.common.collect.Iterables.getOnlyElement; +import static com.google.common.collect.Iterables.toArray; +import static com.google.common.collect.Iterables.transform; +import static com.google.common.collect.Maps.newLinkedHashMap; +import static com.google.common.collect.Maps.uniqueIndex; +import static com.google.common.collect.Sets.newHashSet; +import static com.google.common.collect.Sets.newLinkedHashSet; import static org.jclouds.aws.ec2.options.DescribeImagesOptions.Builder.imageIds; import static org.jclouds.aws.ec2.options.DescribeImagesOptions.Builder.ownedBy; import static org.jclouds.aws.ec2.reference.EC2Constants.PROPERTY_EC2_AMI_OWNERS; @@ -26,17 +36,18 @@ import static org.jclouds.aws.ec2.util.EC2Utils.getAllRunningInstancesInRegion; import static org.jclouds.aws.ec2.util.EC2Utils.parseHandle; import static org.jclouds.compute.domain.OsFamily.CENTOS; import static org.jclouds.compute.domain.OsFamily.UBUNTU; -import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion; +import static org.jclouds.concurrent.ConcurrentUtils.transformParallel; import java.net.URI; import java.security.SecureRandom; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; -import java.util.concurrent.Callable; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -61,6 +72,7 @@ import org.jclouds.aws.ec2.compute.functions.RegionAndIdToImage; import org.jclouds.aws.ec2.compute.functions.RunningInstanceToNodeMetadata; import org.jclouds.aws.ec2.compute.internal.EC2TemplateBuilderImpl; import org.jclouds.aws.ec2.compute.options.EC2TemplateOptions; +import org.jclouds.aws.ec2.compute.strategy.DescribeImagesParallel; import org.jclouds.aws.ec2.compute.strategy.EC2DestroyLoadBalancerStrategy; import org.jclouds.aws.ec2.compute.strategy.EC2DestroyNodeStrategy; import org.jclouds.aws.ec2.compute.strategy.EC2LoadBalanceNodesStrategy; @@ -68,8 +80,8 @@ import org.jclouds.aws.ec2.compute.strategy.EC2RunNodesAndAddToSetStrategy; import org.jclouds.aws.ec2.domain.InstanceType; import org.jclouds.aws.ec2.domain.KeyPair; import org.jclouds.aws.ec2.domain.PlacementGroup; +import org.jclouds.aws.ec2.domain.Reservation; import org.jclouds.aws.ec2.domain.RunningInstance; -import org.jclouds.aws.ec2.domain.Image.ImageType; import org.jclouds.aws.ec2.functions.RunningInstanceToStorageMappingUnix; import org.jclouds.aws.ec2.options.DescribeImagesOptions; import org.jclouds.aws.ec2.predicates.InstancePresent; @@ -96,7 +108,6 @@ import org.jclouds.compute.strategy.ListNodesStrategy; import org.jclouds.compute.strategy.LoadBalanceNodesStrategy; import org.jclouds.compute.strategy.RebootNodeStrategy; import org.jclouds.compute.strategy.RunNodesAndAddToSetStrategy; -import org.jclouds.concurrent.ConcurrentUtils; import org.jclouds.domain.Location; import org.jclouds.domain.LocationScope; import org.jclouds.domain.internal.LocationImpl; @@ -108,6 +119,7 @@ import org.jclouds.rest.internal.RestContextImpl; import com.google.common.base.Function; import com.google.common.base.Predicate; +import com.google.common.base.Predicates; import com.google.common.base.Splitter; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; @@ -115,8 +127,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.MapMaker; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Scopes; @@ -206,13 +216,13 @@ public class EC2ComputeServiceContextModule extends AbstractModule { @Named(ComputeServiceConstants.COMPUTE_LOGGER) protected Logger logger = Logger.NULL; - private final EC2Client client; + private final EC2AsyncClient client; private final Map regionMap; private final RunningInstanceToNodeMetadata runningInstanceToNodeMetadata; private final ExecutorService executor; @Inject - protected EC2ListNodesStrategy(EC2Client client, @Region Map regionMap, + protected EC2ListNodesStrategy(EC2AsyncClient client, @Region Map regionMap, RunningInstanceToNodeMetadata runningInstanceToNodeMetadata, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) { this.client = client; @@ -222,31 +232,25 @@ public class EC2ComputeServiceContextModule extends AbstractModule { } @Override - public Iterable list() { + public Set list() { return listDetailsOnNodesMatching(NodePredicates.all()); } @Override - public Iterable listDetailsOnNodesMatching(Predicate filter) { - final Set nodes = Sets.newHashSet(); + public Set listDetailsOnNodesMatching(Predicate filter) { + Iterable>> reservations = transformParallel(regionMap + .keySet(), new Function>>>() { - Map> parallelResponses = Maps.newHashMap(); + @Override + public Future>> apply(String from) { + return client.getInstanceServices().describeInstancesInRegion(from); + } - for (final String region : regionMap.keySet()) { - parallelResponses.put(region, ConcurrentUtils.makeListenable(executor.submit(new Callable() { - @Override - public Void call() throws Exception { - Iterables.addAll(nodes, Iterables.transform(Iterables.concat(client.getInstanceServices() - .describeInstancesInRegion(region)), runningInstanceToNodeMetadata)); - return null; - } - }), executor)); - } - Map exceptions = awaitCompletion(parallelResponses, executor, null, logger, "nodes"); + }, executor, null, logger, "reservations"); - if (exceptions.size() > 0) - throw new RuntimeException(String.format("error parsing nodes in regions: %s", exceptions)); - return Iterables.filter(nodes, filter); + Iterable instances = concat(concat(reservations)); + Iterable nodes = filter(transform(instances, runningInstanceToNodeMetadata), filter); + return newLinkedHashSet(nodes); } } @@ -268,7 +272,7 @@ public class EC2ComputeServiceContextModule extends AbstractModule { String region = parts[0]; String instanceId = parts[1]; try { - RunningInstance runningInstance = Iterables.getOnlyElement(getAllRunningInstancesInRegion(client + RunningInstance runningInstance = getOnlyElement(getAllRunningInstancesInRegion(client .getInstanceServices(), region, instanceId)); return runningInstanceToNodeMetadata.apply(runningInstance); } catch (NoSuchElementException e) { @@ -305,7 +309,7 @@ public class EC2ComputeServiceContextModule extends AbstractModule { protected final Map credentialsMap(CreateUniqueKeyPair in) { // doesn't seem to clear when someone issues remove(key) // return new MapMaker().makeComputingMap(in); - return Maps.newLinkedHashMap(); + return newLinkedHashMap(); } @Provides @@ -314,7 +318,7 @@ public class EC2ComputeServiceContextModule extends AbstractModule { protected final Map securityGroupMap(CreateSecurityGroupIfNeeded in) { // doesn't seem to clear when someone issues remove(key) // return new MapMaker().makeComputingMap(in); - return Maps.newLinkedHashMap(); + return newLinkedHashMap(); } @Provides @@ -323,7 +327,7 @@ public class EC2ComputeServiceContextModule extends AbstractModule { protected final Map placementGroupMap(CreatePlacementGroupIfNeeded in) { // doesn't seem to clear when someone issues remove(key) // return new MapMaker().makeComputingMap(in); - return Maps.newLinkedHashMap(); + return newLinkedHashMap(); } @Provides @@ -340,10 +344,10 @@ public class EC2ComputeServiceContextModule extends AbstractModule { @Provides @Singleton Set provideSizes(Set locations, @Named(PROPERTY_EC2_CC_AMIs) String[] ccAmis) { - Set sizes = Sets.newHashSet(); + Set sizes = newHashSet(); for (String ccAmi : ccAmis) { final String region = ccAmi.split("/")[0]; - Location location = Iterables.find(locations, new Predicate() { + Location location = find(locations, new Predicate() { @Override public boolean apply(Location input) { @@ -362,11 +366,11 @@ public class EC2ComputeServiceContextModule extends AbstractModule { Set provideLocations(Map availabilityZoneToRegionMap, @Provider String providerName) { Location ec2 = new LocationImpl(LocationScope.PROVIDER, providerName, providerName, null); - Set locations = Sets.newLinkedHashSet(); - for (String region : Sets.newLinkedHashSet(availabilityZoneToRegionMap.values())) { + Set locations = newLinkedHashSet(); + for (String region : newLinkedHashSet(availabilityZoneToRegionMap.values())) { locations.add(new LocationImpl(LocationScope.REGION, region, region, ec2)); } - ImmutableMap idToLocation = Maps.uniqueIndex(locations, new Function() { + ImmutableMap idToLocation = uniqueIndex(locations, new Function() { @Override public String apply(Location from) { return from.getId(); @@ -391,7 +395,7 @@ public class EC2ComputeServiceContextModule extends AbstractModule { String[] amiOwners(@Named(PROPERTY_EC2_AMI_OWNERS) String amiOwners) { if (amiOwners.trim().equals("")) return new String[] {}; - return Iterables.toArray(Splitter.on(',').split(amiOwners), String.class); + return toArray(Splitter.on(',').split(amiOwners), String.class); } @Provides @@ -400,7 +404,7 @@ public class EC2ComputeServiceContextModule extends AbstractModule { String[] ccAmis(@Named(PROPERTY_EC2_CC_AMIs) String ccAmis) { if (ccAmis.trim().equals("")) return new String[] {}; - return Iterables.toArray(Splitter.on(',').split(ccAmis), String.class); + return toArray(Splitter.on(',').split(ccAmis), String.class); } @Provides @@ -416,54 +420,62 @@ public class EC2ComputeServiceContextModule extends AbstractModule { @Provides @Singleton - protected Map provideImages(final EC2Client sync, - @Region Map regionMap, final LogHolder holder, Function indexer, - @Named(PROPERTY_EC2_CC_AMIs) String[] ccAmis, @Named(PROPERTY_EC2_AMI_OWNERS) final String[] amiOwners, - final ImageParser parser, final ConcurrentMap images, - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) throws InterruptedException, - ExecutionException, TimeoutException { + protected Map provideImages(@Region Map regionMap, + DescribeImagesParallel describer, LogHolder holder, @Named(PROPERTY_EC2_CC_AMIs) String[] ccAmis, + @Named(PROPERTY_EC2_AMI_OWNERS) final String[] amiOwners, final ImageParser parser, + final ConcurrentMap images) throws InterruptedException, ExecutionException, + TimeoutException { if (amiOwners.length == 0) { holder.logger.debug(">> no owners specified, skipping image parsing"); } else { holder.logger.debug(">> providing images"); - Map> parallelResponses = Maps.newHashMap(); - final DescribeImagesOptions options; - if (amiOwners.length == 1 && amiOwners[0].equals("*")) - options = new DescribeImagesOptions(); - else - options = ownedBy(amiOwners); - for (final String region : regionMap.keySet()) { - parallelResponses.put(region, ConcurrentUtils.makeListenable(executor.submit(new Callable() { - @Override - public Void call() throws Exception { - Set matchingImages = sync.getAMIServices().describeImagesInRegion( - region, options); - for (final org.jclouds.aws.ec2.domain.Image from : matchingImages) { - Image image = parser.apply(from); - if (image != null) - images.put(new RegionAndName(region, image.getProviderId()), image); - else if (from.getImageType() == ImageType.MACHINE) - holder.logger.trace("<< image(%s) didn't parse", from.getId()); - } - return null; - } - }), executor)); - } - Map exceptions = awaitCompletion(parallelResponses, executor, null, holder.logger, "images"); - for (String ccAmi : ccAmis) { - String region = ccAmi.split("/")[0]; - org.jclouds.aws.ec2.domain.Image from = Iterables.getOnlyElement(sync.getAMIServices() - .describeImagesInRegion(region, imageIds(ccAmi.split("/")[1]))); - Image image = parser.apply(from); - if (image != null) - images.put(new RegionAndName(region, image.getProviderId()), image); - } - if (exceptions.size() > 0) - throw new RuntimeException(String.format("error parsing images in regions: %s", exceptions)); + Iterable> queries = concat(getDescribeQueriesForOwnersInRegions( + regionMap, amiOwners).entrySet(), ccAmisToDescribeQueries(ccAmis).entrySet()); + + Iterable parsedImages = filter(transform(describer.apply(queries), parser), Predicates + .notNull()); + + images.putAll(Maps.uniqueIndex(parsedImages, new Function() { + + @Override + public RegionAndName apply(Image from) { + return new RegionAndName(from.getLocation().getId(), from.getProviderId()); + } + + })); holder.logger.debug("<< images(%d)", images.size()); } return images; } + + private Map ccAmisToDescribeQueries(String[] ccAmis) { + Map queries = Maps.newLinkedHashMap(); + for (String from : ccAmis) { + queries.put(from.split("/")[0], imageIds(from.split("/")[1])); + } + return queries; + } + + private Map getDescribeQueriesForOwnersInRegions(Map regionMap, + final String[] amiOwners) { + final DescribeImagesOptions options = getOptionsForOwners(amiOwners); + + return Maps.transformValues(regionMap, new Function() { + @Override + public DescribeImagesOptions apply(URI from) { + return options; + } + }); + } + + private DescribeImagesOptions getOptionsForOwners(final String[] amiOwners) { + final DescribeImagesOptions options; + if (amiOwners.length == 1 && amiOwners[0].equals("*")) + options = new DescribeImagesOptions(); + else + options = ownedBy(amiOwners); + return options; + } } diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/internal/EC2TemplateBuilderImpl.java b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/internal/EC2TemplateBuilderImpl.java index 3129e47b92..02a3e3bf9f 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/internal/EC2TemplateBuilderImpl.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/internal/EC2TemplateBuilderImpl.java @@ -49,9 +49,9 @@ public class EC2TemplateBuilderImpl extends TemplateBuilderImpl { @Inject protected EC2TemplateBuilderImpl(Provider> locations, Provider> images, - Provider> sizes, Location defaultLocation, Provider optionsProvider, - @Named("DEFAULT") Provider defaultTemplateProvider, - ConcurrentMap imageMap) { + Provider> sizes, Location defaultLocation, Provider optionsProvider, + @Named("DEFAULT") Provider defaultTemplateProvider, + ConcurrentMap imageMap) { super(locations, images, sizes, defaultLocation, optionsProvider, defaultTemplateProvider); this.imageMap = imageMap; } @@ -80,7 +80,8 @@ public class EC2TemplateBuilderImpl extends TemplateBuilderImpl { if (imageId != null) { String[] regionName = imageId.split("/"); checkArgument(regionName.length == 2, - "amazon image ids must include the region. ex. us-east-1/ami-7ea24a17"); + "amazon image ids must include the region ( ex. us-east-1/ami-7ea24a17 ) you specified: " + + imageId); RegionAndName key = new RegionAndName(regionName[0], regionName[1]); try { return imageMap.get(key); diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/strategy/DescribeImagesParallel.java b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/strategy/DescribeImagesParallel.java new file mode 100644 index 0000000000..f963d9cd32 --- /dev/null +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/strategy/DescribeImagesParallel.java @@ -0,0 +1,79 @@ +/** + * + * Copyright (C) 2009 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ +package org.jclouds.aws.ec2.compute.strategy; + +import static com.google.common.collect.Iterables.concat; +import static org.jclouds.concurrent.ConcurrentUtils.transformParallel; + +import java.util.Set; +import java.util.Map.Entry; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import javax.annotation.Resource; +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Singleton; + +import org.jclouds.Constants; +import org.jclouds.aws.ec2.EC2AsyncClient; +import org.jclouds.aws.ec2.options.DescribeImagesOptions; +import org.jclouds.compute.reference.ComputeServiceConstants; +import org.jclouds.logging.Logger; + +import com.google.common.base.Function; + +/** + * + * @author Adrian Cole + */ +@Singleton +public class DescribeImagesParallel implements + Function>, Iterable> { + @Resource + @Named(ComputeServiceConstants.COMPUTE_LOGGER) + protected Logger logger = Logger.NULL; + + protected final EC2AsyncClient sync; + final ExecutorService executor; + + @Inject + public DescribeImagesParallel(EC2AsyncClient sync, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) { + super(); + this.sync = sync; + this.executor = executor; + } + + @Override + public Iterable apply( + Iterable> queries) { + return concat(transformParallel( + queries, + new Function, Future>>() { + + @Override + public Future> apply( + Entry from) { + return sync.getAMIServices().describeImagesInRegion(from.getKey(), from.getValue()); + } + + }, executor, null, logger, "amis")); + } + +} \ No newline at end of file diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/strategy/EC2RunNodesAndAddToSetStrategy.java b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/strategy/EC2RunNodesAndAddToSetStrategy.java index 4f51efa8cf..fe133fdf28 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/strategy/EC2RunNodesAndAddToSetStrategy.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/strategy/EC2RunNodesAndAddToSetStrategy.java @@ -27,6 +27,7 @@ import static org.jclouds.aws.ec2.compute.util.EC2ComputeUtils.instanceToId; import java.util.Map; import java.util.Set; +import java.util.concurrent.Future; import javax.annotation.Resource; import javax.inject.Inject; @@ -48,7 +49,6 @@ import org.jclouds.logging.Logger; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Predicate; -import com.google.common.util.concurrent.ListenableFuture; /** * creates futures that correlate to @@ -56,8 +56,7 @@ import com.google.common.util.concurrent.ListenableFuture; * @author Adrian Cole */ @Singleton -public class EC2RunNodesAndAddToSetStrategy implements - RunNodesAndAddToSetStrategy { +public class EC2RunNodesAndAddToSetStrategy implements RunNodesAndAddToSetStrategy { @Resource @Named(ComputeServiceConstants.COMPUTE_LOGGER) @@ -76,11 +75,10 @@ public class EC2RunNodesAndAddToSetStrategy implements @Inject EC2RunNodesAndAddToSetStrategy( - EC2Client client, - CreateKeyPairPlacementAndSecurityGroupsAsNeededAndReturnRunOptions createKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions, - @Named("PRESENT") Predicate instancePresent, - RunningInstanceToNodeMetadata runningInstanceToNodeMetadata, - ComputeUtils utils) { + EC2Client client, + CreateKeyPairPlacementAndSecurityGroupsAsNeededAndReturnRunOptions createKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions, + @Named("PRESENT") Predicate instancePresent, + RunningInstanceToNodeMetadata runningInstanceToNodeMetadata, ComputeUtils utils) { this.client = client; this.instancePresent = instancePresent; this.createKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions = createKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions; @@ -89,12 +87,11 @@ public class EC2RunNodesAndAddToSetStrategy implements } @Override - public Map> execute(String tag, int count, - Template template, Set goodNodes, - Map badNodes) { + public Map> execute(String tag, int count, Template template, Set goodNodes, + Map badNodes) { - Reservation reservation = createKeyPairAndSecurityGroupsAsNeededThenRunInstances( - tag, count, template); + Reservation reservation = createKeyPairAndSecurityGroupsAsNeededThenRunInstances(tag, + count, template); Iterable ids = transform(reservation, instanceToId); @@ -104,28 +101,25 @@ public class EC2RunNodesAndAddToSetStrategy implements all(reservation, instancePresent); logger.debug("<< present instances(%s)", idsString); - return utils.runOptionsOnNodesAndAddToGoodSetOrPutExceptionIntoBadMap( - template.getOptions(), transform(reservation, - runningInstanceToNodeMetadata), goodNodes, badNodes); + return utils.runOptionsOnNodesAndAddToGoodSetOrPutExceptionIntoBadMap(template.getOptions(), transform( + reservation, runningInstanceToNodeMetadata), goodNodes, badNodes); } @VisibleForTesting - Reservation createKeyPairAndSecurityGroupsAsNeededThenRunInstances( - String tag, int count, Template template) { + Reservation createKeyPairAndSecurityGroupsAsNeededThenRunInstances(String tag, int count, + Template template) { String region = getRegionFromLocationOrNull(template.getLocation()); String zone = getZoneFromLocationOrNull(template.getLocation()); - RunInstancesOptions instanceOptions = createKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions - .execute(region, tag, template); + RunInstancesOptions instanceOptions = createKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions.execute(region, + tag, template); if (logger.isDebugEnabled()) - logger.debug( - ">> running %d instance region(%s) zone(%s) ami(%s) params(%s)", - count, region, zone, template.getImage().getProviderId(), - instanceOptions.buildFormParameters()); + logger.debug(">> running %d instance region(%s) zone(%s) ami(%s) params(%s)", count, region, zone, template + .getImage().getProviderId(), instanceOptions.buildFormParameters()); - return client.getInstanceServices().runInstancesInRegion(region, zone, - template.getImage().getProviderId(), 1, count, instanceOptions); + return client.getInstanceServices().runInstancesInRegion(region, zone, template.getImage().getProviderId(), 1, + count, instanceOptions); } } \ No newline at end of file diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/domain/Reservation.java b/aws/core/src/main/java/org/jclouds/aws/ec2/domain/Reservation.java index 70d19361a1..a24bcd2eec 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/domain/Reservation.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/domain/Reservation.java @@ -33,8 +33,8 @@ import com.google.inject.internal.Nullable; * /> * @author Adrian Cole */ -public class Reservation extends LinkedHashSet implements Comparable, - Set { +public class Reservation extends LinkedHashSet implements Comparable>, + Set { /** The serialVersionUID */ private static final long serialVersionUID = -9051777593518861395L; @@ -47,9 +47,8 @@ public class Reservation extends LinkedHashSet implements Compa private final @Nullable String reservationId; - public Reservation(String region, Iterable groupIds, - Iterable instances, @Nullable String ownerId, - @Nullable String requesterId, @Nullable String reservationId) { + public Reservation(String region, Iterable groupIds, Iterable instances, + @Nullable String ownerId, @Nullable String requesterId, @Nullable String reservationId) { this.region = checkNotNull(region, "region"); Iterables.addAll(this.groupIds, checkNotNull(groupIds, "groupIds")); Iterables.addAll(this, checkNotNull(instances, "instances")); @@ -65,7 +64,7 @@ public class Reservation extends LinkedHashSet implements Compa return region; } - public int compareTo(Reservation o) { + public int compareTo(Reservation o) { return (this == o) ? 0 : getReservationId().compareTo(o.getReservationId()); } @@ -117,7 +116,7 @@ public class Reservation extends LinkedHashSet implements Compa return false; if (getClass() != obj.getClass()) return false; - Reservation other = (Reservation) obj; + Reservation other = (Reservation) obj; if (groupIds == null) { if (other.groupIds != null) return false; diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/options/CreateImageOptions.java b/aws/core/src/main/java/org/jclouds/aws/ec2/options/CreateImageOptions.java index 186a0b1728..b533c42f3c 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/options/CreateImageOptions.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/options/CreateImageOptions.java @@ -32,7 +32,7 @@ import org.jclouds.aws.ec2.options.internal.BaseEC2RequestOptions; * import static org.jclouds.aws.ec2.options.CreateImageOptions.Builder.* *

* EC2Client connection = // get connection - * ListenableFuture> images = connection.getAMIServices().createImage(withDescription("123125").noReboot()); + * Future> images = connection.getAMIServices().createImage(withDescription("123125").noReboot()); * * * @author Adrian Cole diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeAvailabilityZonesOptions.java b/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeAvailabilityZonesOptions.java index f121600b6f..9955839581 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeAvailabilityZonesOptions.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeAvailabilityZonesOptions.java @@ -33,7 +33,7 @@ import org.jclouds.aws.ec2.options.internal.BaseEC2RequestOptions; * import static org.jclouds.aws.ec2.options.DescribeAvailabilityZonesOptions.Builder.* *

* EC2Client connection = // get connection - * ListenableFuture> images = connection.getAvailabilityZoneAndRegionServices().describeAvailabilityZones(zones("us-east-1a", "us-east-1b")); + * Future> images = connection.getAvailabilityZoneAndRegionServices().describeAvailabilityZones(zones("us-east-1a", "us-east-1b")); * * * @author Adrian Cole diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeImagesOptions.java b/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeImagesOptions.java index bcd143de30..d53e004b01 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeImagesOptions.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeImagesOptions.java @@ -34,7 +34,7 @@ import org.jclouds.aws.ec2.options.internal.BaseEC2RequestOptions; * import static org.jclouds.aws.ec2.options.DescribeImagesOptions.Builder.* *

* EC2Client connection = // get connection - * ListenableFuture> images = connection.getAMIServices().describeImages(executableBy("123125").imageIds(1000, 1004)); + * Future> images = connection.getAMIServices().describeImages(executableBy("123125").imageIds(1000, 1004)); * * * @author Adrian Cole diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeRegionsOptions.java b/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeRegionsOptions.java index 9107d8f201..9626a61240 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeRegionsOptions.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/options/DescribeRegionsOptions.java @@ -34,7 +34,7 @@ import org.jclouds.aws.ec2.options.internal.BaseEC2RequestOptions; * import static org.jclouds.aws.ec2.options.DescribeRegionsOptions.Builder.* *

* EC2Client connection = // get connection - * ListenableFuture> images = connection.getRegionsAndRegionsServices().describeRegions(regions("us-east-1a", "us-east-1b")); + * Future> images = connection.getRegionsAndRegionsServices().describeRegions(regions("us-east-1a", "us-east-1b")); * * * @author Adrian Cole diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/options/RunInstancesOptions.java b/aws/core/src/main/java/org/jclouds/aws/ec2/options/RunInstancesOptions.java index a388ce36dc..5d03c59bb8 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/options/RunInstancesOptions.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/options/RunInstancesOptions.java @@ -35,7 +35,7 @@ import org.jclouds.encryption.internal.Base64; * import static org.jclouds.aws.ec2.options.RunInstancesOptions.Builder.* *

* EC2Client connection = // get connection - * ListenableFuture instances = connection.runInstances(executableBy("123125").imageIds(1000, 1004)); + * Future instances = connection.runInstances(executableBy("123125").imageIds(1000, 1004)); * * * @author Adrian Cole diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/services/AMIAsyncClient.java b/aws/core/src/main/java/org/jclouds/aws/ec2/services/AMIAsyncClient.java index b0c2e0c03f..9f2e51fc06 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/services/AMIAsyncClient.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/services/AMIAsyncClient.java @@ -23,6 +23,7 @@ import static org.jclouds.aws.ec2.reference.EC2Parameters.VERSION; import java.util.Map; import java.util.Set; +import com.google.common.util.concurrent.ListenableFuture; import javax.annotation.Nullable; import javax.ws.rs.FormParam; @@ -56,8 +57,6 @@ import org.jclouds.rest.annotations.VirtualHost; import org.jclouds.rest.annotations.XMLResponseParser; import org.jclouds.rest.functions.ReturnEmptySetOnNotFoundOr404; -import com.google.common.util.concurrent.ListenableFuture; - /** * Provides access to AMI Services. *

@@ -77,7 +76,7 @@ public interface AMIAsyncClient { @FormParams(keys = ACTION, values = "DescribeImages") @XMLResponseParser(DescribeImagesResponseHandler.class) @ExceptionParser(ReturnEmptySetOnNotFoundOr404.class) - ListenableFuture> describeImagesInRegion( + ListenableFuture> describeImagesInRegion( @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, DescribeImagesOptions... options); /** @@ -87,9 +86,8 @@ public interface AMIAsyncClient { @Path("/") @FormParams(keys = ACTION, values = "CreateImage") @XMLResponseParser(ImageIdHandler.class) - ListenableFuture createImageInRegion( - @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @FormParam("Name") String name, - @FormParam("InstanceId") String instanceId, CreateImageOptions... options); + ListenableFuture createImageInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, + @FormParam("Name") String name, @FormParam("InstanceId") String instanceId, CreateImageOptions... options); /** * @see AMIClient#deregisterImageInRegion @@ -97,8 +95,7 @@ public interface AMIAsyncClient { @POST @Path("/") @FormParams(keys = ACTION, values = "DeregisterImage") - ListenableFuture deregisterImageInRegion( - @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, + ListenableFuture deregisterImageInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @FormParam("ImageId") String imageId); /** @@ -181,7 +178,7 @@ public interface AMIAsyncClient { @Path("/") @FormParams(keys = { ACTION, "Attribute" }, values = { "DescribeImageAttribute", "productCodes" }) @XMLResponseParser(ProductCodesHandler.class) - ListenableFuture> getProductCodesForImageInRegion( + ListenableFuture> getProductCodesForImageInRegion( @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @FormParam("ImageId") String imageId); @@ -192,7 +189,7 @@ public interface AMIAsyncClient { @Path("/") @FormParams(keys = { ACTION, "Attribute" }, values = { "DescribeImageAttribute", "blockDeviceMapping" }) @XMLResponseParser(BlockDeviceMappingHandler.class) - ListenableFuture> getBlockDeviceMappingsForImageInRegion( + ListenableFuture> getBlockDeviceMappingsForImageInRegion( @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @FormParam("ImageId") String imageId); @@ -203,8 +200,7 @@ public interface AMIAsyncClient { @Path("/") @FormParams(keys = { ACTION, "OperationType", "Attribute" }, values = { "ModifyImageAttribute", "add", "productCodes" }) - ListenableFuture addProductCodesToImageInRegion( - @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, + ListenableFuture addProductCodesToImageInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @BinderParam(BindProductCodesToIndexedFormParams.class) Iterable productCodes, @FormParam("ImageId") String imageId); diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/services/AMIClient.java b/aws/core/src/main/java/org/jclouds/aws/ec2/services/AMIClient.java index 44b752051a..4d4037784a 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/services/AMIClient.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/services/AMIClient.java @@ -57,7 +57,7 @@ public interface AMIClient { * @see DescribeImagesOptions */ @Timeout(duration = 300, timeUnit = TimeUnit.SECONDS) - Set describeImagesInRegion(@Nullable String region, DescribeImagesOptions... options); + Set describeImagesInRegion(@Nullable String region, DescribeImagesOptions... options); /** * Returns the Product Codes of an image. diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/services/InstanceAsyncClient.java b/aws/core/src/main/java/org/jclouds/aws/ec2/services/InstanceAsyncClient.java index b9595c6057..84cbbf8c5e 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/services/InstanceAsyncClient.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/services/InstanceAsyncClient.java @@ -23,6 +23,7 @@ import static org.jclouds.aws.ec2.reference.EC2Parameters.VERSION; import java.util.Map; import java.util.Set; +import com.google.common.util.concurrent.ListenableFuture; import javax.annotation.Nullable; import javax.ws.rs.FormParam; @@ -61,8 +62,6 @@ import org.jclouds.rest.annotations.VirtualHost; import org.jclouds.rest.annotations.XMLResponseParser; import org.jclouds.rest.functions.ReturnEmptySetOnNotFoundOr404; -import com.google.common.util.concurrent.ListenableFuture; - /** * Provides access to EC2 Instance Services via their REST API. *

@@ -82,7 +81,7 @@ public interface InstanceAsyncClient { @FormParams(keys = ACTION, values = "DescribeInstances") @XMLResponseParser(DescribeInstancesResponseHandler.class) @ExceptionParser(ReturnEmptySetOnNotFoundOr404.class) - ListenableFuture> describeInstancesInRegion( + ListenableFuture>> describeInstancesInRegion( @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @BinderParam(BindInstanceIdsToIndexedFormParams.class) String... instanceIds); @@ -93,8 +92,7 @@ public interface InstanceAsyncClient { @Path("/") @FormParams(keys = ACTION, values = "RunInstances") @XMLResponseParser(RunInstancesResponseHandler.class) - ListenableFuture runInstancesInRegion( - @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, + ListenableFuture> runInstancesInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @Nullable @BinderParam(IfNotNullBindAvailabilityZoneToFormParam.class) String nullableAvailabilityZone, @FormParam("ImageId") String imageId, @FormParam("MinCount") int minCount, @FormParam("MaxCount") int maxCount, RunInstancesOptions... options); @@ -105,8 +103,7 @@ public interface InstanceAsyncClient { @POST @Path("/") @FormParams(keys = ACTION, values = "RebootInstances") - ListenableFuture rebootInstancesInRegion( - @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, + ListenableFuture rebootInstancesInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @BinderParam(BindInstanceIdsToIndexedFormParams.class) String... instanceIds); /** @@ -116,7 +113,7 @@ public interface InstanceAsyncClient { @Path("/") @FormParams(keys = ACTION, values = "TerminateInstances") @XMLResponseParser(InstanceStateChangeHandler.class) - ListenableFuture> terminateInstancesInRegion( + ListenableFuture> terminateInstancesInRegion( @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @BinderParam(BindInstanceIdsToIndexedFormParams.class) String... instanceIds); @@ -127,7 +124,7 @@ public interface InstanceAsyncClient { @Path("/") @FormParams(keys = ACTION, values = "StopInstances") @XMLResponseParser(InstanceStateChangeHandler.class) - ListenableFuture> stopInstancesInRegion( + ListenableFuture> stopInstancesInRegion( @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @FormParam("Force") boolean force, @BinderParam(BindInstanceIdsToIndexedFormParams.class) String... instanceIds); @@ -138,7 +135,7 @@ public interface InstanceAsyncClient { @Path("/") @FormParams(keys = ACTION, values = "StartInstances") @XMLResponseParser(InstanceStateChangeHandler.class) - ListenableFuture> startInstancesInRegion( + ListenableFuture> startInstancesInRegion( @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @BinderParam(BindInstanceIdsToIndexedFormParams.class) String... instanceIds); @@ -182,8 +179,7 @@ public interface InstanceAsyncClient { @Path("/") @FormParams(keys = { ACTION, "Attribute" }, values = { "DescribeInstanceAttribute", "kernel" }) @XMLResponseParser(StringValueHandler.class) - ListenableFuture getKernelForInstanceInRegion( - @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, + ListenableFuture getKernelForInstanceInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @FormParam("InstanceId") String instanceId); /** @@ -247,8 +243,7 @@ public interface InstanceAsyncClient { @POST @Path("/") @FormParams(keys = { ACTION, "Attribute" }, values = { "ResetInstanceAttribute", "kernel" }) - ListenableFuture resetKernelForInstanceInRegion( - @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, + ListenableFuture resetKernelForInstanceInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @FormParam("InstanceId") String instanceId); /** @@ -257,8 +252,7 @@ public interface InstanceAsyncClient { @POST @Path("/") @FormParams(keys = { ACTION, "Attribute" }, values = { "ModifyInstanceAttribute", "userData" }) - ListenableFuture setUserDataForInstanceInRegion( - @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, + ListenableFuture setUserDataForInstanceInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @FormParam("InstanceId") String instanceId, @FormParam("Value") @ParamParser(ConvertUnencodedBytesToBase64EncodedString.class) byte[] unencodedData); @@ -268,8 +262,7 @@ public interface InstanceAsyncClient { @POST @Path("/") @FormParams(keys = { ACTION, "Attribute" }, values = { "ModifyInstanceAttribute", "ramdisk" }) - ListenableFuture setRamdiskForInstanceInRegion( - @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, + ListenableFuture setRamdiskForInstanceInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @FormParam("InstanceId") String instanceId, @FormParam("Value") String ramdisk); /** @@ -278,8 +271,7 @@ public interface InstanceAsyncClient { @POST @Path("/") @FormParams(keys = { ACTION, "Attribute" }, values = { "ModifyInstanceAttribute", "kernel" }) - ListenableFuture setKernelForInstanceInRegion( - @EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, + ListenableFuture setKernelForInstanceInRegion(@EndpointParam(parser = RegionToEndpoint.class) @Nullable String region, @FormParam("InstanceId") String instanceId, @FormParam("Value") String kernel); /** diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/services/InstanceClient.java b/aws/core/src/main/java/org/jclouds/aws/ec2/services/InstanceClient.java index 59227a8bd3..b8347d9044 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/services/InstanceClient.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/services/InstanceClient.java @@ -61,7 +61,7 @@ public interface InstanceClient { * @see */ - Set describeInstancesInRegion(@Nullable String region, + Set> describeInstancesInRegion(@Nullable String region, String... instanceIds); /** @@ -141,7 +141,7 @@ public interface InstanceClient { * /> * @see RunInstancesOptions */ - Reservation runInstancesInRegion(@Nullable String region, + Reservation runInstancesInRegion(@Nullable String region, @Nullable String nullableAvailabilityZone, String imageId, int minCount, int maxCount, RunInstancesOptions... options); @@ -161,7 +161,7 @@ public interface InstanceClient { * @see */ - Set terminateInstancesInRegion(@Nullable String region, + Set terminateInstancesInRegion(@Nullable String region, String... instanceIds); /** @@ -199,7 +199,7 @@ public interface InstanceClient { * @see */ - Set stopInstancesInRegion(@Nullable String region, + Set stopInstancesInRegion(@Nullable String region, boolean force, String... instanceIds); /** @@ -255,7 +255,7 @@ public interface InstanceClient { * @see */ - Set startInstancesInRegion(@Nullable String region, + Set startInstancesInRegion(@Nullable String region, String... instanceIds); /** diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/xml/BaseReservationHandler.java b/aws/core/src/main/java/org/jclouds/aws/ec2/xml/BaseReservationHandler.java index f023aa7014..c39dd9aa86 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/xml/BaseReservationHandler.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/xml/BaseReservationHandler.java @@ -270,11 +270,12 @@ public abstract class BaseReservationHandler extends HandlerForGeneratedReque currentText.append(ch, start, length); } - protected Reservation newReservation() { + protected Reservation newReservation() { String region = EC2Utils.findRegionInArgsOrNull((GeneratedHttpRequest) request); if (region == null) region = defaultRegion; - Reservation info = new Reservation(region, groupIds, instances, ownerId, requesterId, reservationId); + Reservation info = new Reservation(region, groupIds, instances, + ownerId, requesterId, reservationId); this.groupIds = Sets.newLinkedHashSet(); this.instances = Sets.newLinkedHashSet(); this.ownerId = null; diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/xml/DescribeInstancesResponseHandler.java b/aws/core/src/main/java/org/jclouds/aws/ec2/xml/DescribeInstancesResponseHandler.java index c48f8559c5..151a378445 100644 --- a/aws/core/src/main/java/org/jclouds/aws/ec2/xml/DescribeInstancesResponseHandler.java +++ b/aws/core/src/main/java/org/jclouds/aws/ec2/xml/DescribeInstancesResponseHandler.java @@ -24,6 +24,7 @@ import javax.inject.Inject; import org.jclouds.aws.Region; import org.jclouds.aws.ec2.domain.Reservation; +import org.jclouds.aws.ec2.domain.RunningInstance; import org.jclouds.date.DateService; import com.google.common.collect.Sets; @@ -37,8 +38,8 @@ import com.google.common.collect.Sets; * @see */ -public class RunInstancesResponseHandler extends BaseReservationHandler { +public class RunInstancesResponseHandler extends BaseReservationHandler> { @Inject RunInstancesResponseHandler(DateService dateService, @Region String defaultRegion) { @@ -40,7 +41,7 @@ public class RunInstancesResponseHandler extends BaseReservationHandler getResult() { return newReservation(); } diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/S3Client.java b/aws/core/src/main/java/org/jclouds/aws/s3/S3Client.java index bb4e8bc7a6..4c5a8fcb84 100644 --- a/aws/core/src/main/java/org/jclouds/aws/s3/S3Client.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/S3Client.java @@ -38,14 +38,14 @@ import org.jclouds.aws.s3.options.PutObjectOptions; import org.jclouds.concurrent.Timeout; import org.jclouds.http.options.GetOptions; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; /** * Provides access to S3 via their REST API. *

- * All commands return a ListenableFuture of the result from S3. Any exceptions incurred during + * All commands return a Future of the result from S3. Any exceptions incurred during * processing will be wrapped in an {@link ExecutionException} as documented in - * {@link ListenableFuture#get()}. + * {@link Future#get()}. * * @author Adrian Cole * @author James Murty @@ -85,7 +85,7 @@ public interface S3Client { * namespace of the object you are retrieving * @param key * unique key in the s3Bucket identifying the object - * @return ListenableFuture reference to a fully populated S3Object including data stored in S3 + * @return Future reference to a fully populated S3Object including data stored in S3 * or {@link S3Object#NOT_FOUND} if not present. * * @throws org.jclouds.http.HttpResponseException @@ -231,7 +231,7 @@ public interface S3Client { * * @param bucketName * namespace of the objects you wish to list - * @return ListenableFuture reference to a fully populated S3Bucket including metadata of the + * @return Future reference to a fully populated S3Bucket including metadata of the * S3Objects it contains or {@link BoundedList#NOT_FOUND} if not present. * @see ListBucketOptions * diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3AsyncBlobStore.java b/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3AsyncBlobStore.java index c633988888..3aaf48d8aa 100644 --- a/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3AsyncBlobStore.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/blobstore/S3AsyncBlobStore.java @@ -19,10 +19,11 @@ package org.jclouds.aws.s3.blobstore; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.util.concurrent.Futures.compose; +import static org.jclouds.concurrent.ConcurrentUtils.compose; import java.util.Set; import java.util.concurrent.ExecutorService; +import com.google.common.util.concurrent.ListenableFuture; import javax.inject.Inject; import javax.inject.Named; @@ -60,7 +61,6 @@ import org.jclouds.http.options.GetOptions; import com.google.common.base.Function; import com.google.common.collect.Iterables; -import com.google.common.util.concurrent.ListenableFuture; /** * @@ -82,41 +82,34 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore { @Inject S3AsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, - Location defaultLocation, Set locations, S3AsyncClient async, - S3Client sync, BucketToResourceMetadata bucket2ResourceMd, - ContainerToBucketListOptions container2BucketListOptions, + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Location defaultLocation, + Set locations, S3AsyncClient async, S3Client sync, + BucketToResourceMetadata bucket2ResourceMd, ContainerToBucketListOptions container2BucketListOptions, BucketToResourceList bucket2ResourceList, ObjectToBlob object2Blob, - BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, - ObjectToBlobMetadata object2BlobMd, + BlobToHttpGetOptions blob2ObjectGetOptions, BlobToObject blob2Object, ObjectToBlobMetadata object2BlobMd, Provider fetchBlobMetadataProvider) { super(context, blobUtils, service, defaultLocation, locations); this.blob2ObjectGetOptions = checkNotNull(blob2ObjectGetOptions, "blob2ObjectGetOptions"); this.async = checkNotNull(async, "async"); this.sync = checkNotNull(sync, "sync"); this.bucket2ResourceMd = checkNotNull(bucket2ResourceMd, "bucket2ResourceMd"); - this.container2BucketListOptions = checkNotNull(container2BucketListOptions, - "container2BucketListOptions"); + this.container2BucketListOptions = checkNotNull(container2BucketListOptions, "container2BucketListOptions"); this.bucket2ResourceList = checkNotNull(bucket2ResourceList, "bucket2ResourceList"); this.object2Blob = checkNotNull(object2Blob, "object2Blob"); this.blob2Object = checkNotNull(blob2Object, "blob2Object"); this.object2BlobMd = checkNotNull(object2BlobMd, "object2BlobMd"); - this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, - "fetchBlobMetadataProvider"); + this.fetchBlobMetadataProvider = checkNotNull(fetchBlobMetadataProvider, "fetchBlobMetadataProvider"); } /** * This implementation invokes {@link S3AsyncClient#listOwnedBuckets} */ @Override - public ListenableFuture> list() { - return compose( - async.listOwnedBuckets(), + public ListenableFuture> list() { + return compose(async.listOwnedBuckets(), new Function, org.jclouds.blobstore.domain.PageSet>() { - public org.jclouds.blobstore.domain.PageSet apply( - Set from) { - return new PageSetImpl(Iterables.transform(from, - bucket2ResourceMd), null); + public org.jclouds.blobstore.domain.PageSet apply(Set from) { + return new PageSetImpl(Iterables.transform(from, bucket2ResourceMd), null); } }, service); } @@ -153,14 +146,13 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore { * bucket name */ @Override - public ListenableFuture> list(String container, - ListContainerOptions options) { + // TODO get rid of compose, as it serializes async results when the executor is single-threaded. + public ListenableFuture> list(String container, ListContainerOptions options) { ListBucketOptions httpOptions = container2BucketListOptions.apply(options); ListenableFuture returnVal = async.listBucket(container, httpOptions); - ListenableFuture> list = compose(returnVal, - bucket2ResourceList, service); - return options.isDetailed() ? compose(list, fetchBlobMetadataProvider.get().setContainerName( - container), service) : list; + ListenableFuture> list = compose(returnVal, bucket2ResourceList, service); + return (options.isDetailed()) ? compose(list, fetchBlobMetadataProvider.get().setContainerName(container), + service) : list; } /** @@ -193,15 +185,14 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore { */ @Override public ListenableFuture blobMetadata(String container, String key) { - return compose(async.headObject(container, key), - new Function() { + return compose(async.headObject(container, key), new Function() { - @Override - public BlobMetadata apply(ObjectMetadata from) { - return object2BlobMd.apply(from); - } + @Override + public BlobMetadata apply(ObjectMetadata from) { + return object2BlobMd.apply(from); + } - }, service); + }, service); } /** @@ -213,8 +204,7 @@ public class S3AsyncBlobStore extends BaseAsyncBlobStore { * object key */ @Override - public ListenableFuture getBlob(String container, String key, - org.jclouds.blobstore.options.GetOptions options) { + public ListenableFuture getBlob(String container, String key, org.jclouds.blobstore.options.GetOptions options) { GetOptions httpOptions = blob2ObjectGetOptions.apply(options); return compose(async.getObject(container, key, httpOptions), object2Blob, service); } diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/options/CopyObjectOptions.java b/aws/core/src/main/java/org/jclouds/aws/s3/options/CopyObjectOptions.java index 7d0e283dd5..1e63747aed 100755 --- a/aws/core/src/main/java/org/jclouds/aws/s3/options/CopyObjectOptions.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/options/CopyObjectOptions.java @@ -58,7 +58,7 @@ import com.google.common.collect.Multimap; *

* // this will copy the object, provided it wasn't modified since yesterday. * // it will not use metadata from the source, and instead use what we pass in. - * ListenableFuture object = connection.copyObject("sourceBucket", "objectName", + * Future object = connection.copyObject("sourceBucket", "objectName", * "destinationBucket", "destinationName", * overrideMetadataWith(meta). * ifSourceModifiedSince(new Date().minusDays(1)) diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/options/ListBucketOptions.java b/aws/core/src/main/java/org/jclouds/aws/s3/options/ListBucketOptions.java index 3a3d9b074a..8ce6e7dd01 100755 --- a/aws/core/src/main/java/org/jclouds/aws/s3/options/ListBucketOptions.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/options/ListBucketOptions.java @@ -33,7 +33,7 @@ import org.jclouds.http.options.BaseHttpRequestOptions; * import static org.jclouds.aws.s3.commands.options.GetBucketOptions.Builder.* *

* S3Client connection = // get connection - * ListenableFuture bucket = connection.listBucket("bucketName",withPrefix("home/users").maxKeys(1000)); + * Future bucket = connection.listBucket("bucketName",withPrefix("home/users").maxKeys(1000)); * * * @author Adrian Cole diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/options/PutBucketOptions.java b/aws/core/src/main/java/org/jclouds/aws/s3/options/PutBucketOptions.java index 40e26d1c05..72fa560f44 100755 --- a/aws/core/src/main/java/org/jclouds/aws/s3/options/PutBucketOptions.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/options/PutBucketOptions.java @@ -46,7 +46,7 @@ import com.google.common.collect.Multimap; * import org.jclouds.aws.s3.S3Client; *

* S3Client connection = // get connection - * ListenableFuture createdInEu = connection.putBucketIfNotExists("bucketName",createIn(EU)); + * Future createdInEu = connection.putBucketIfNotExists("bucketName",createIn(EU)); * * * @author Adrian Cole diff --git a/aws/core/src/main/java/org/jclouds/aws/s3/options/PutObjectOptions.java b/aws/core/src/main/java/org/jclouds/aws/s3/options/PutObjectOptions.java index aab8c3416e..164894cbfa 100755 --- a/aws/core/src/main/java/org/jclouds/aws/s3/options/PutObjectOptions.java +++ b/aws/core/src/main/java/org/jclouds/aws/s3/options/PutObjectOptions.java @@ -46,7 +46,7 @@ import static org.jclouds.aws.reference.AWSConstants.PROPERTY_HEADER_TAG; * import org.jclouds.aws.s3.S3Client; * * S3Client connection = // get connection - * ListenableFuture publicly readable = connection.putObject("bucketName",new S3Object("key","value"), withAcl(CannedAccessPolicy.PUBLIC_READ)); + * Future publicly readable = connection.putObject("bucketName",new S3Object("key","value"), withAcl(CannedAccessPolicy.PUBLIC_READ)); * * * @see - * adds in functionality to boot a lamp instance: - * http://alestic.com/2009/06/ec2-user-data-scripts + * adds in functionality to boot a lamp instance: http://alestic.com/2009/06/ec2-user-data-scripts *

* Generally disabled, as it incurs higher fees. * @@ -92,49 +91,38 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { private RetryablePredicate runningTester; @BeforeGroups(groups = { "live" }) - public void setupClient() throws InterruptedException, ExecutionException, - TimeoutException, - IOException { - String identity = checkNotNull(System.getProperty("jclouds.test.identity"), - "jclouds.test.identity"); - String credential = checkNotNull(System.getProperty("jclouds.test.credential"), - "jclouds.test.credential"); - Injector injector = new RestContextFactory().createContextBuilder( - "ec2", identity, credential, ImmutableSet. of(new Log4JLoggingModule())).buildInjector(); + public void setupClient() throws InterruptedException, ExecutionException, TimeoutException, IOException { + String identity = checkNotNull(System.getProperty("jclouds.test.identity"), "jclouds.test.identity"); + String credential = checkNotNull(System.getProperty("jclouds.test.credential"), "jclouds.test.credential"); + Injector injector = new RestContextFactory().createContextBuilder("ec2", identity, credential, + ImmutableSet. of(new Log4JLoggingModule())).buildInjector(); client = injector.getInstance(EC2Client.class); sshFactory = injector.getInstance(SshClient.Factory.class); - runningTester = new RetryablePredicate( - new InstanceStateRunning(client), 180, 5, TimeUnit.SECONDS); - hasIpTester = new RetryablePredicate( - new InstanceHasIpAddress(client), 180, 5, TimeUnit.SECONDS); + runningTester = new RetryablePredicate(new InstanceStateRunning(client), 180, 5, + TimeUnit.SECONDS); + hasIpTester = new RetryablePredicate(new InstanceHasIpAddress(client), 180, 5, TimeUnit.SECONDS); SocketOpen socketOpen = injector.getInstance(SocketOpen.class); - socketTester = new RetryablePredicate(socketOpen, 180, 1, - TimeUnit.SECONDS); + socketTester = new RetryablePredicate(socketOpen, 180, 1, TimeUnit.SECONDS); } @Test(enabled = false) - void testCreateSecurityGroupIngressCidr() throws InterruptedException, - ExecutionException, TimeoutException { + void testCreateSecurityGroupIngressCidr() throws InterruptedException, ExecutionException, TimeoutException { securityGroupName = instancePrefix + "ingress"; try { - client.getSecurityGroupServices().deleteSecurityGroupInRegion(null, - securityGroupName); + client.getSecurityGroupServices().deleteSecurityGroupInRegion(null, securityGroupName); } catch (Exception e) { } - client.getSecurityGroupServices().createSecurityGroupInRegion(null, - securityGroupName, securityGroupName); + client.getSecurityGroupServices().createSecurityGroupInRegion(null, securityGroupName, securityGroupName); for (int port : new int[] { 80, 443, 22 }) { - client.getSecurityGroupServices() - .authorizeSecurityGroupIngressInRegion(null, securityGroupName, - IpProtocol.TCP, port, port, "0.0.0.0/0"); + client.getSecurityGroupServices().authorizeSecurityGroupIngressInRegion(null, securityGroupName, + IpProtocol.TCP, port, port, "0.0.0.0/0"); } } @Test(enabled = false) - void testCreateKeyPair() throws InterruptedException, ExecutionException, - TimeoutException { + void testCreateKeyPair() throws InterruptedException, ExecutionException, TimeoutException { String keyName = instancePrefix + "1"; try { client.getKeyPairServices().deleteKeyPairInRegion(null, keyName); @@ -143,41 +131,37 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { } client.getKeyPairServices().deleteKeyPairInRegion(null, keyName); - keyPair = client.getKeyPairServices() - .createKeyPairInRegion(null, keyName); + keyPair = client.getKeyPairServices().createKeyPairInRegion(null, keyName); assertNotNull(keyPair); assertNotNull(keyPair.getKeyMaterial()); assertNotNull(keyPair.getKeyFingerprint()); assertEquals(keyPair.getKeyName(), keyName); } - @Test(enabled = false, dependsOnMethods = { "testCreateKeyPair", - "testCreateSecurityGroupIngressCidr" }) + @Test(enabled = false, dependsOnMethods = { "testCreateKeyPair", "testCreateSecurityGroupIngressCidr" }) public void testCreateRunningInstance() throws Exception { String script = new ScriptBuilder() // lamp install script - .addStatement(exec("runurl run.alestic.com/apt/upgrade"))// - .addStatement(exec("runurl run.alestic.com/install/lamp"))// - .build(OsFamily.UNIX); + .addStatement(exec("runurl run.alestic.com/apt/upgrade"))// + .addStatement(exec("runurl run.alestic.com/install/lamp"))// + .build(OsFamily.UNIX); RunningInstance instance = null; while (instance == null) { try { - System.out.printf("%d: running instance%n", System - .currentTimeMillis()); - Reservation reservation = client.getInstanceServices() - .runInstancesInRegion(null, null, // allow - // ec2 - // to - // chose - // an - // availability - // zone - "ami-ccf615a5", // alestic ami allows auto-invoke of - // user data scripts - 1, // minimum instances - 1, // maximum instances - asType(InstanceType.M1_SMALL) // smallest instance size + System.out.printf("%d: running instance%n", System.currentTimeMillis()); + Reservation reservation = client.getInstanceServices().runInstancesInRegion(null, null, // allow + // ec2 + // to + // chose + // an + // availability + // zone + "ami-ccf615a5", // alestic ami allows auto-invoke of + // user data scripts + 1, // minimum instances + 1, // maximum instances + asType(InstanceType.M1_SMALL) // smallest instance size .withKeyName(keyPair.getKeyName()) // key I // created // above @@ -203,51 +187,36 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { verifyInstanceProperties(script); tryToChangeStuff(); sshPing(instance); - System.out.printf("%d: %s ssh connection made%n", System - .currentTimeMillis(), instanceId); + System.out.printf("%d: %s ssh connection made%n", System.currentTimeMillis(), instanceId); } private void verifyInstanceProperties(String script) { - assertEquals(script, client.getInstanceServices() - .getUserDataForInstanceInRegion(null, instanceId)); + assertEquals(script, client.getInstanceServices().getUserDataForInstanceInRegion(null, instanceId)); - assertEquals(null, client.getInstanceServices() - .getRootDeviceNameForInstanceInRegion(null, instanceId)); + assertEquals(null, client.getInstanceServices().getRootDeviceNameForInstanceInRegion(null, instanceId)); - assert client.getInstanceServices().getRamdiskForInstanceInRegion(null, - instanceId).startsWith("ari-"); + assert client.getInstanceServices().getRamdiskForInstanceInRegion(null, instanceId).startsWith("ari-"); - assertEquals(false, client.getInstanceServices() - .isApiTerminationDisabledForInstanceInRegion(null, instanceId)); + assertEquals(false, client.getInstanceServices().isApiTerminationDisabledForInstanceInRegion(null, instanceId)); - assert client.getInstanceServices().getKernelForInstanceInRegion(null, - instanceId).startsWith("aki-"); + assert client.getInstanceServices().getKernelForInstanceInRegion(null, instanceId).startsWith("aki-"); - assertEquals(InstanceType.M1_SMALL, client.getInstanceServices() - .getInstanceTypeForInstanceInRegion(null, instanceId)); + assertEquals(InstanceType.M1_SMALL, client.getInstanceServices().getInstanceTypeForInstanceInRegion(null, + instanceId)); - assertEquals(InstanceInitiatedShutdownBehavior.TERMINATE, client - .getInstanceServices() - .getInstanceInitiatedShutdownBehaviorForInstanceInRegion(null, - instanceId)); + assertEquals(InstanceInitiatedShutdownBehavior.TERMINATE, client.getInstanceServices() + .getInstanceInitiatedShutdownBehaviorForInstanceInRegion(null, instanceId)); - assertEquals(ImmutableMap. of(), client - .getInstanceServices().getBlockDeviceMappingForInstanceInRegion( - null, instanceId)); + assertEquals(ImmutableMap. of(), client.getInstanceServices() + .getBlockDeviceMappingForInstanceInRegion(null, instanceId)); } private void setApiTerminationDisabledForInstanceInRegion() { - client.getInstanceServices() - .setApiTerminationDisabledForInstanceInRegion(null, instanceId, - true); - assertEquals(true, client.getInstanceServices() - .isApiTerminationDisabledForInstanceInRegion(null, instanceId)); - client.getInstanceServices() - .setApiTerminationDisabledForInstanceInRegion(null, instanceId, - false); - assertEquals(false, client.getInstanceServices() - .isApiTerminationDisabledForInstanceInRegion(null, instanceId)); + client.getInstanceServices().setApiTerminationDisabledForInstanceInRegion(null, instanceId, true); + assertEquals(true, client.getInstanceServices().isApiTerminationDisabledForInstanceInRegion(null, instanceId)); + client.getInstanceServices().setApiTerminationDisabledForInstanceInRegion(null, instanceId, false); + assertEquals(false, client.getInstanceServices().isApiTerminationDisabledForInstanceInRegion(null, instanceId)); } private void tryToChangeStuff() { @@ -262,8 +231,7 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { private void setUserDataForInstanceInRegion() { try { - client.getInstanceServices().setUserDataForInstanceInRegion(null, - instanceId, "test".getBytes()); + client.getInstanceServices().setUserDataForInstanceInRegion(null, instanceId, "test".getBytes()); assert false : "shouldn't be allowed, as instance needs to be stopped"; } catch (AWSResponseException e) { assertEquals("IncorrectInstanceState", e.getError().getCode()); @@ -272,10 +240,8 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { private void setRamdiskForInstanceInRegion() { try { - String ramdisk = client.getInstanceServices() - .getRamdiskForInstanceInRegion(null, instanceId); - client.getInstanceServices().setRamdiskForInstanceInRegion(null, - instanceId, ramdisk); + String ramdisk = client.getInstanceServices().getRamdiskForInstanceInRegion(null, instanceId); + client.getInstanceServices().setRamdiskForInstanceInRegion(null, instanceId, ramdisk); assert false : "shouldn't be allowed, as instance needs to be stopped"; } catch (AWSResponseException e) { assertEquals("IncorrectInstanceState", e.getError().getCode()); @@ -284,10 +250,8 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { private void setKernelForInstanceInRegion() { try { - String oldKernel = client.getInstanceServices() - .getKernelForInstanceInRegion(null, instanceId); - client.getInstanceServices().setKernelForInstanceInRegion(null, - instanceId, oldKernel); + String oldKernel = client.getInstanceServices().getKernelForInstanceInRegion(null, instanceId); + client.getInstanceServices().setKernelForInstanceInRegion(null, instanceId, oldKernel); assert false : "shouldn't be allowed, as instance needs to be stopped"; } catch (AWSResponseException e) { assertEquals("IncorrectInstanceState", e.getError().getCode()); @@ -296,8 +260,7 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { private void setInstanceTypeForInstanceInRegion() { try { - client.getInstanceServices().setInstanceTypeForInstanceInRegion(null, - instanceId, InstanceType.C1_MEDIUM); + client.getInstanceServices().setInstanceTypeForInstanceInRegion(null, instanceId, InstanceType.C1_MEDIUM); assert false : "shouldn't be allowed, as instance needs to be stopped"; } catch (AWSResponseException e) { assertEquals("IncorrectInstanceState", e.getError().getCode()); @@ -307,8 +270,7 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { private void setBlockDeviceMappingForInstanceInRegion() { BlockDeviceMapping blockDeviceMapping = new BlockDeviceMapping(); try { - client.getInstanceServices().setBlockDeviceMappingForInstanceInRegion( - null, instanceId, blockDeviceMapping); + client.getInstanceServices().setBlockDeviceMappingForInstanceInRegion(null, instanceId, blockDeviceMapping); assert false : "shouldn't be allowed, as instance needs to be ebs based-ami"; } catch (AWSResponseException e) { assertEquals("InvalidParameterCombination", e.getError().getCode()); @@ -317,9 +279,8 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { private void setInstanceInitiatedShutdownBehaviorForInstanceInRegion() { try { - client.getInstanceServices() - .setInstanceInitiatedShutdownBehaviorForInstanceInRegion(null, - instanceId, InstanceInitiatedShutdownBehavior.STOP); + client.getInstanceServices().setInstanceInitiatedShutdownBehaviorForInstanceInRegion(null, instanceId, + InstanceInitiatedShutdownBehavior.STOP); assert false : "shouldn't be allowed, as instance needs to be ebs based-ami"; } catch (AWSResponseException e) { assertEquals("UnsupportedInstanceAttribute", e.getError().getCode()); @@ -327,22 +288,19 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { } @Test(enabled = false, dependsOnMethods = "testCreateRunningInstance") - void testReboot() throws InterruptedException, ExecutionException, - TimeoutException, IOException { + void testReboot() throws InterruptedException, ExecutionException, TimeoutException, IOException { RunningInstance instance = getInstance(instanceId); - System.out.printf("%d: %s rebooting instance %n", System - .currentTimeMillis(), instanceId); + System.out.printf("%d: %s rebooting instance %n", System.currentTimeMillis(), instanceId); client.getInstanceServices().rebootInstancesInRegion(null, instanceId); Thread.sleep(1000); instance = getInstance(instanceId); blockUntilWeCanSshIntoInstance(instance); - SshClient ssh = sshFactory.create(new IPSocket(instance.getIpAddress(), - 22), "root", keyPair.getKeyMaterial().getBytes()); + SshClient ssh = sshFactory.create(new IPSocket(instance.getIpAddress(), 22), "root", keyPair.getKeyMaterial() + .getBytes()); try { ssh.connect(); ExecResponse uptime = ssh.exec("uptime"); - assert uptime.getOutput().indexOf("0 min") != -1 : "reboot didn't work: " - + uptime; + assert uptime.getOutput().indexOf("0 min") != -1 : "reboot didn't work: " + uptime; } finally { if (ssh != null) ssh.disconnect(); @@ -350,86 +308,70 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { } @Test(enabled = false, dependsOnMethods = "testReboot") - void testElasticIpAddress() throws InterruptedException, ExecutionException, - TimeoutException, IOException { - address = client.getElasticIPAddressServices().allocateAddressInRegion( - null); + void testElasticIpAddress() throws InterruptedException, ExecutionException, TimeoutException, IOException { + address = client.getElasticIPAddressServices().allocateAddressInRegion(null); assertNotNull(address); - PublicIpInstanceIdPair compare = Iterables.getLast(client - .getElasticIPAddressServices().describeAddressesInRegion(null, - address)); + PublicIpInstanceIdPair compare = Iterables.getLast(client.getElasticIPAddressServices() + .describeAddressesInRegion(null, address)); assertEquals(compare.getPublicIp(), address); assert compare.getInstanceId() == null; - client.getElasticIPAddressServices().associateAddressInRegion(null, - address, instanceId); + client.getElasticIPAddressServices().associateAddressInRegion(null, address, instanceId); - compare = Iterables.getLast(client.getElasticIPAddressServices() - .describeAddressesInRegion(null, address)); + compare = Iterables.getLast(client.getElasticIPAddressServices().describeAddressesInRegion(null, address)); assertEquals(compare.getPublicIp(), address); assertEquals(compare.getInstanceId(), instanceId); - Reservation reservation = Iterables.getOnlyElement(client - .getInstanceServices().describeInstancesInRegion(null, instanceId)); + Reservation reservation = Iterables.getOnlyElement(client.getInstanceServices() + .describeInstancesInRegion(null, instanceId)); assertNotNull(Iterables.getOnlyElement(reservation).getIpAddress()); - assertFalse(Iterables.getOnlyElement(reservation).getIpAddress().equals( - address)); + assertFalse(Iterables.getOnlyElement(reservation).getIpAddress().equals(address)); doCheckKey(address); - client.getElasticIPAddressServices().disassociateAddressInRegion(null, - address); + client.getElasticIPAddressServices().disassociateAddressInRegion(null, address); - compare = Iterables.getLast(client.getElasticIPAddressServices() - .describeAddressesInRegion(null, address)); + compare = Iterables.getLast(client.getElasticIPAddressServices().describeAddressesInRegion(null, address)); assertEquals(compare.getPublicIp(), address); assert compare.getInstanceId() == null; - reservation = Iterables.getOnlyElement(client.getInstanceServices() - .describeInstancesInRegion(null, instanceId)); + reservation = Iterables.getOnlyElement(client.getInstanceServices().describeInstancesInRegion(null, instanceId)); // assert reservation.getRunningInstances().last().getIpAddress() == null; // TODO } - private RunningInstance blockUntilWeCanSshIntoInstance( - RunningInstance instance) throws UnknownHostException { - System.out.printf("%d: %s awaiting instance to run %n", System - .currentTimeMillis(), instance.getId()); + private RunningInstance blockUntilWeCanSshIntoInstance(RunningInstance instance) throws UnknownHostException { + System.out.printf("%d: %s awaiting instance to run %n", System.currentTimeMillis(), instance.getId()); assert runningTester.apply(instance); instance = getInstance(instance.getId()); - System.out.printf("%d: %s awaiting instance to have ip assigned %n", - System.currentTimeMillis(), instance.getId()); + System.out + .printf("%d: %s awaiting instance to have ip assigned %n", System.currentTimeMillis(), instance.getId()); assert hasIpTester.apply(instance); - System.out.printf("%d: %s awaiting ssh service to start%n", System - .currentTimeMillis(), instance.getIpAddress()); + System.out.printf("%d: %s awaiting ssh service to start%n", System.currentTimeMillis(), instance.getIpAddress()); assert socketTester.apply(new IPSocket(instance.getIpAddress(), 22)); - System.out.printf("%d: %s ssh service started%n", System - .currentTimeMillis(), instance.getDnsName()); + System.out.printf("%d: %s ssh service started%n", System.currentTimeMillis(), instance.getDnsName()); sshPing(instance); - System.out.printf("%d: %s ssh connection made%n", System - .currentTimeMillis(), instance.getId()); + System.out.printf("%d: %s ssh connection made%n", System.currentTimeMillis(), instance.getId()); - System.out.printf("%d: %s awaiting http service to start%n", System - .currentTimeMillis(), instance.getIpAddress()); + System.out.printf("%d: %s awaiting http service to start%n", System.currentTimeMillis(), instance.getIpAddress()); assert socketTester.apply(new IPSocket(instance.getIpAddress(), 80)); - System.out.printf("%d: %s http service started%n", System - .currentTimeMillis(), instance.getDnsName()); + System.out.printf("%d: %s http service started%n", System.currentTimeMillis(), instance.getDnsName()); return instance; } private RunningInstance getInstance(String instanceId) { // search my identity for the instance I just created - Set reservations = client.getInstanceServices() - .describeInstancesInRegion(null, instanceId); // last parameter + Set> reservations = client.getInstanceServices() + .describeInstancesInRegion(null, instanceId); // last parameter // (ids) narrows the // search @@ -437,8 +379,7 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { } /** - * this tests "personality" as the file looked up was sent during instance - * creation + * this tests "personality" as the file looked up was sent during instance creation * * @throws UnknownHostException */ @@ -454,14 +395,12 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { } } - private void doCheckKey(RunningInstance newDetails) - throws UnknownHostException { + private void doCheckKey(RunningInstance newDetails) throws UnknownHostException { doCheckKey(newDetails.getIpAddress()); } private void doCheckKey(String address) { - SshClient ssh = sshFactory.create(new IPSocket(address, 22), "root", - keyPair.getKeyMaterial().getBytes()); + SshClient ssh = sshFactory.create(new IPSocket(address, 22), "root", keyPair.getKeyMaterial().getBytes()); try { ssh.connect(); ExecResponse hello = ssh.exec("echo hello"); @@ -473,20 +412,15 @@ public class CloudApplicationArchitecturesEC2ClientLiveTest { } @AfterTest - void cleanup() throws InterruptedException, ExecutionException, - TimeoutException { + void cleanup() throws InterruptedException, ExecutionException, TimeoutException { if (address != null) - client.getElasticIPAddressServices().releaseAddressInRegion(null, - address); + client.getElasticIPAddressServices().releaseAddressInRegion(null, address); if (instanceId != null) - client.getInstanceServices().terminateInstancesInRegion(null, - instanceId); + client.getInstanceServices().terminateInstancesInRegion(null, instanceId); if (keyPair != null) - client.getKeyPairServices().deleteKeyPairInRegion(null, - keyPair.getKeyName()); + client.getKeyPairServices().deleteKeyPairInRegion(null, keyPair.getKeyName()); if (securityGroupName != null) - client.getSecurityGroupServices().deleteSecurityGroupInRegion(null, - securityGroupName); + client.getSecurityGroupServices().deleteSecurityGroupInRegion(null, securityGroupName); } } diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/EBSBootEC2ClientLiveTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/EBSBootEC2ClientLiveTest.java index 4a5888be03..adf5bb79ad 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/EBSBootEC2ClientLiveTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/EBSBootEC2ClientLiveTest.java @@ -81,8 +81,8 @@ import com.google.inject.Module; import com.google.inject.internal.ImmutableMap; /** - * Adapted from the following sources: {@link http://gist.github.com/249915}, - * {@link http ://www.capsunlock.net/2009/12/create-ebs-boot-ami.html} + * Adapted from the following sources: {@link http://gist.github.com/249915}, {@link http + * ://www.capsunlock.net/2009/12/create-ebs-boot-ami.html} *

* * Generally disabled, as it incurs higher fees. @@ -123,7 +123,7 @@ public class EBSBootEC2ClientLiveTest { String identity = checkNotNull(System.getProperty("jclouds.test.identity"), "jclouds.test.identity"); String credential = checkNotNull(System.getProperty("jclouds.test.credential"), "jclouds.test.credential"); Injector injector = new RestContextFactory().createContextBuilder("ec2", identity, credential, - ImmutableSet. of(new Log4JLoggingModule())).buildInjector(); + ImmutableSet. of(new Log4JLoggingModule())).buildInjector(); client = injector.getInstance(EC2Client.class); sshFactory = injector.getInstance(SshClient.Factory.class); SocketOpen socketOpen = injector.getInstance(SocketOpen.class); @@ -139,7 +139,7 @@ public class EBSBootEC2ClientLiveTest { attachTester = new RetryablePredicate(volumeAttached, 60, 1, TimeUnit.SECONDS); runningTester = new RetryablePredicate(new InstanceStateRunning(client), 180, 5, - TimeUnit.SECONDS); + TimeUnit.SECONDS); InstanceStateStopped instanceStateStopped = injector.getInstance(InstanceStateStopped.class); stoppedTester = new RetryablePredicate(instanceStateStopped, 60, 1, TimeUnit.SECONDS); @@ -161,11 +161,11 @@ public class EBSBootEC2ClientLiveTest { client.getSecurityGroupServices().createSecurityGroupInRegion(null, securityGroupName, securityGroupName); client.getSecurityGroupServices().authorizeSecurityGroupIngressInRegion(null, securityGroupName, IpProtocol.TCP, - 80, 80, "0.0.0.0/0"); + 80, 80, "0.0.0.0/0"); client.getSecurityGroupServices().authorizeSecurityGroupIngressInRegion(null, securityGroupName, IpProtocol.TCP, - 443, 443, "0.0.0.0/0"); + 443, 443, "0.0.0.0/0"); client.getSecurityGroupServices().authorizeSecurityGroupIngressInRegion(null, securityGroupName, IpProtocol.TCP, - 22, 22, "0.0.0.0/0"); + 22, 22, "0.0.0.0/0"); } @Test(enabled = false) @@ -194,19 +194,20 @@ public class EBSBootEC2ClientLiveTest { while (instance == null) { try { System.out.printf("%d: running instance%n", System.currentTimeMillis()); - Reservation reservation = client.getInstanceServices().runInstancesInRegion(null, null, // allow - // ec2 - // to - // chose - // an - // availability - // zone - imageId, 1, // minimum instances - 1, // maximum instances - withKeyName(keyPair.getKeyName())// key I created above - .asType(InstanceType.M1_SMALL)// smallest instance - // size - .withSecurityGroup(securityGroupName));// group I + Reservation reservation = client.getInstanceServices().runInstancesInRegion( + null, null, // allow + // ec2 + // to + // chose + // an + // availability + // zone + imageId, 1, // minimum instances + 1, // maximum instances + withKeyName(keyPair.getKeyName())// key I created above + .asType(InstanceType.M1_SMALL)// smallest instance + // size + .withSecurityGroup(securityGroupName));// group I // created // above instance = Iterables.getOnlyElement(reservation); @@ -225,13 +226,13 @@ public class EBSBootEC2ClientLiveTest { @Test(enabled = false, dependsOnMethods = "testCreateRunningInstance") void testCreateAndAttachVolume() { volume = client.getElasticBlockStoreServices().createVolumeInAvailabilityZone(instance.getAvailabilityZone(), - VOLUME_SIZE); + VOLUME_SIZE); System.out.printf("%d: %s awaiting volume to become available%n", System.currentTimeMillis(), volume.getId()); assert volumeTester.apply(volume); Attachment attachment = client.getElasticBlockStoreServices().attachVolumeInRegion(instance.getRegion(), - volume.getId(), instance.getId(), "/dev/sdh"); + volume.getId(), instance.getId(), "/dev/sdh"); System.out.printf("%d: %s awaiting attachment to complete%n", System.currentTimeMillis(), attachment.getId()); @@ -245,35 +246,35 @@ public class EBSBootEC2ClientLiveTest { void makeScript() { mkEbsBoot = new InitBuilder( - "mkebsboot",// name of the script - "/tmp",// working directory - "/tmp/logs",// location of stdout.log and stderr.log - ImmutableMap.of("imageDir", "/mnt/tmp", "ebsDevice", "/dev/sdh", "ebsMountPoint", "/mnt/ebs"),// variables - // used - // inside - // of - // the - // script - "echo creating a filesystem and mounting the ebs volume",// what to - // execute - "{md} {varl}IMAGE_DIR{varr} {varl}EBS_MOUNT_POINT{varr}", - "rm -rf {varl}IMAGE_DIR{varr}/*", - "yes| mkfs -t ext3 {varl}EBS_DEVICE{varr} 2>&-", - "mount {varl}EBS_DEVICE{varr} {varl}EBS_MOUNT_POINT{varr}", - "echo making a local working copy of the boot disk", - "rsync -ax --exclude /ubuntu/.bash_history --exclude /home/*/.bash_history --exclude /etc/ssh/ssh_host_* --exclude /etc/ssh/moduli --exclude /etc/udev/rules.d/*persistent-net.rules --exclude /var/lib/ec2/* --exclude=/mnt/* --exclude=/proc/* --exclude=/tmp/* --exclude=/dev/log / {varl}IMAGE_DIR{varr}", - "echo preparing the local working copy", "touch {varl}IMAGE_DIR{varr}/etc/init.d/ec2-init-user-data", - "echo copying the local working copy to the ebs mount", "{cd} {varl}IMAGE_DIR{varr}", - "tar -cSf - * | tar xf - -C {varl}EBS_MOUNT_POINT{varr}", "echo size of ebs", - "du -sk {varl}EBS_MOUNT_POINT{varr}", "echo size of source", "du -sk {varl}IMAGE_DIR{varr}", - "rm -rf {varl}IMAGE_DIR{varr}/*", "umount {varl}EBS_MOUNT_POINT{varr}", "echo " + SCRIPT_END) - .build(OsFamily.UNIX); + "mkebsboot",// name of the script + "/tmp",// working directory + "/tmp/logs",// location of stdout.log and stderr.log + ImmutableMap.of("imageDir", "/mnt/tmp", "ebsDevice", "/dev/sdh", "ebsMountPoint", "/mnt/ebs"),// variables + // used + // inside + // of + // the + // script + "echo creating a filesystem and mounting the ebs volume",// what to + // execute + "{md} {varl}IMAGE_DIR{varr} {varl}EBS_MOUNT_POINT{varr}", + "rm -rf {varl}IMAGE_DIR{varr}/*", + "yes| mkfs -t ext3 {varl}EBS_DEVICE{varr} 2>&-", + "mount {varl}EBS_DEVICE{varr} {varl}EBS_MOUNT_POINT{varr}", + "echo making a local working copy of the boot disk", + "rsync -ax --exclude /ubuntu/.bash_history --exclude /home/*/.bash_history --exclude /etc/ssh/ssh_host_* --exclude /etc/ssh/moduli --exclude /etc/udev/rules.d/*persistent-net.rules --exclude /var/lib/ec2/* --exclude=/mnt/* --exclude=/proc/* --exclude=/tmp/* --exclude=/dev/log / {varl}IMAGE_DIR{varr}", + "echo preparing the local working copy", "touch {varl}IMAGE_DIR{varr}/etc/init.d/ec2-init-user-data", + "echo copying the local working copy to the ebs mount", "{cd} {varl}IMAGE_DIR{varr}", + "tar -cSf - * | tar xf - -C {varl}EBS_MOUNT_POINT{varr}", "echo size of ebs", + "du -sk {varl}EBS_MOUNT_POINT{varr}", "echo size of source", "du -sk {varl}IMAGE_DIR{varr}", + "rm -rf {varl}IMAGE_DIR{varr}/*", "umount {varl}EBS_MOUNT_POINT{varr}", "echo " + SCRIPT_END) + .build(OsFamily.UNIX); } @Test(enabled = false, dependsOnMethods = "testCreateAndAttachVolume") void testBundleInstance() { SshClient ssh = sshFactory.create(new IPSocket(instance.getIpAddress(), 22), "ubuntu", keyPair.getKeyMaterial() - .getBytes()); + .getBytes()); try { ssh.connect(); } catch (SshException e) {// try twice in case there is a network timeout @@ -298,7 +299,7 @@ public class EBSBootEC2ClientLiveTest { assert !output.getOutput().trim().equals("") : output; RetryablePredicate scriptTester = new RetryablePredicate(new ScriptTester(ssh, SCRIPT_END), - 600, 10, TimeUnit.SECONDS); + 600, 10, TimeUnit.SECONDS); scriptTester.apply(script); } finally { if (ssh != null) @@ -328,7 +329,7 @@ public class EBSBootEC2ClientLiveTest { output = ssh.exec(script + " tailerr"); String stderr = output.getOutput().trim(); throw new RuntimeException(String.format( - "script %s ended without token: stdout.log: [%s]; stderr.log: [%s]; ", script, stdout, stderr)); + "script %s ended without token: stdout.log: [%s]; stderr.log: [%s]; ", script, stdout, stderr)); } } return false; @@ -339,7 +340,7 @@ public class EBSBootEC2ClientLiveTest { @Test(enabled = false, dependsOnMethods = "testBundleInstance") void testAMIFromBundle() { volume = Iterables.getOnlyElement(client.getElasticBlockStoreServices().describeVolumesInRegion( - volume.getRegion(), volume.getId())); + volume.getRegion(), volume.getId())); if (volume.getAttachments().size() > 0) { // should be cleanly unmounted, so force is not necessary. client.getElasticBlockStoreServices().detachVolumeInRegion(instance.getRegion(), volume.getId(), false); @@ -350,31 +351,31 @@ public class EBSBootEC2ClientLiveTest { // detach } snapshot = client.getElasticBlockStoreServices().createSnapshotInRegion(volume.getRegion(), volume.getId(), - withDescription("EBS Ubuntu Hardy")); + withDescription("EBS Ubuntu Hardy")); System.out.printf("%d: %s awaiting snapshot to complete%n", System.currentTimeMillis(), snapshot.getId()); assert snapshotTester.apply(snapshot); Image image = Iterables.getOnlyElement(client.getAMIServices().describeImagesInRegion(snapshot.getRegion(), - imageIds(IMAGE_ID))); + imageIds(IMAGE_ID))); String description = image.getDescription() == null ? "jclouds" : image.getDescription(); System.out.printf("%d: %s creating ami from snapshot%n", System.currentTimeMillis(), snapshot.getId()); String amiId = client.getAMIServices().registerUnixImageBackedByEbsInRegion( - snapshot.getRegion(), - "ebsboot-" + image.getId(), - snapshot.getId(), - withKernelId(image.getKernelId()).withRamdisk(image.getRamdiskId()).withDescription(description) - .asArchitecture(Architecture.I386)); + snapshot.getRegion(), + "ebsboot-" + image.getId(), + snapshot.getId(), + withKernelId(image.getKernelId()).withRamdisk(image.getRamdiskId()).withDescription(description) + .asArchitecture(Architecture.I386)); try { ebsImage = Iterables.getOnlyElement(client.getAMIServices().describeImagesInRegion(snapshot.getRegion(), - imageIds(amiId))); + imageIds(amiId))); } catch (AWSResponseException e) { // TODO add a retry handler for this HTTP code 400 and the below error if (e.getError().getClass().equals("InvalidAMIID.NotFound")) ebsImage = Iterables.getOnlyElement(client.getAMIServices().describeImagesInRegion(snapshot.getRegion(), - imageIds(amiId))); + imageIds(amiId))); else throw e; } @@ -402,7 +403,7 @@ public class EBSBootEC2ClientLiveTest { assertEquals(ebsImage.getRootDeviceType(), RootDeviceType.EBS); assertEquals(ebsImage.getRootDeviceName(), "/dev/sda1"); assertEquals(ebsImage.getEbsBlockDevices().entrySet(), ImmutableMap.of("/dev/sda1", - new Image.EbsBlockDevice(snapshot.getId(), VOLUME_SIZE, true)).entrySet()); + new Image.EbsBlockDevice(snapshot.getId(), VOLUME_SIZE, true)).entrySet()); } private void tryToChangeStuff() { @@ -433,12 +434,12 @@ public class EBSBootEC2ClientLiveTest { private void setInstanceTypeForInstanceInRegion() { client.getInstanceServices() - .setInstanceTypeForInstanceInRegion(null, ebsInstance.getId(), InstanceType.C1_MEDIUM); + .setInstanceTypeForInstanceInRegion(null, ebsInstance.getId(), InstanceType.C1_MEDIUM); assertEquals(InstanceType.C1_MEDIUM, client.getInstanceServices().getInstanceTypeForInstanceInRegion(null, - ebsInstance.getId())); + ebsInstance.getId())); client.getInstanceServices().setInstanceTypeForInstanceInRegion(null, ebsInstance.getId(), InstanceType.M1_SMALL); assertEquals(InstanceType.M1_SMALL, client.getInstanceServices().getInstanceTypeForInstanceInRegion(null, - ebsInstance.getId())); + ebsInstance.getId())); } private void setBlockDeviceMappingForInstanceInRegion() { @@ -448,10 +449,10 @@ public class EBSBootEC2ClientLiveTest { blockDeviceMapping.addEbsBlockDevice("/dev/sda1", new RunningInstance.EbsBlockDevice(volumeId, false)); try { client.getInstanceServices().setBlockDeviceMappingForInstanceInRegion(null, ebsInstance.getId(), - blockDeviceMapping); + blockDeviceMapping); Map devices = client.getInstanceServices() - .getBlockDeviceMappingForInstanceInRegion(null, ebsInstance.getId()); + .getBlockDeviceMappingForInstanceInRegion(null, ebsInstance.getId()); assertEquals(devices.size(), 1); String deviceName = Iterables.getOnlyElement(devices.keySet()); RunningInstance.EbsBlockDevice device = Iterables.getOnlyElement(devices.values()); @@ -472,15 +473,15 @@ public class EBSBootEC2ClientLiveTest { try { client.getInstanceServices().setInstanceInitiatedShutdownBehaviorForInstanceInRegion(null, - ebsInstance.getId(), InstanceInitiatedShutdownBehavior.STOP); + ebsInstance.getId(), InstanceInitiatedShutdownBehavior.STOP); assertEquals(InstanceInitiatedShutdownBehavior.STOP, client.getInstanceServices() - .getInstanceInitiatedShutdownBehaviorForInstanceInRegion(null, ebsInstance.getId())); + .getInstanceInitiatedShutdownBehaviorForInstanceInRegion(null, ebsInstance.getId())); client.getInstanceServices().setInstanceInitiatedShutdownBehaviorForInstanceInRegion(null, - ebsInstance.getId(), InstanceInitiatedShutdownBehavior.TERMINATE); + ebsInstance.getId(), InstanceInitiatedShutdownBehavior.TERMINATE); assertEquals(InstanceInitiatedShutdownBehavior.TERMINATE, client.getInstanceServices() - .getInstanceInitiatedShutdownBehaviorForInstanceInRegion(null, ebsInstance.getId())); + .getInstanceInitiatedShutdownBehaviorForInstanceInRegion(null, ebsInstance.getId())); System.out.println("OK: setInstanceInitiatedShutdownBehaviorForInstanceInRegion"); } catch (Exception e) { System.err.println("setInstanceInitiatedShutdownBehaviorForInstanceInRegion"); @@ -489,8 +490,7 @@ public class EBSBootEC2ClientLiveTest { } /** - * this tests "personality" as the file looked up was sent during instance - * creation + * this tests "personality" as the file looked up was sent during instance creation * * @throws UnknownHostException */ @@ -527,8 +527,8 @@ public class EBSBootEC2ClientLiveTest { assert runningTester.apply(instance); // search my identity for the instance I just created - Set reservations = client.getInstanceServices().describeInstancesInRegion(instance.getRegion(), - instance.getId()); // last + Set> reservations = client.getInstanceServices() + .describeInstancesInRegion(instance.getRegion(), instance.getId()); // last // parameter // (ids) // narrows diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/functions/RegionAndIdToImageTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/functions/RegionAndIdToImageTest.java index ce8d96f8f5..b2a64744b0 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/functions/RegionAndIdToImageTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/functions/RegionAndIdToImageTest.java @@ -43,6 +43,7 @@ import com.google.common.collect.ImmutableSet; @Test(groups = "unit", testName = "ec2.RegionAndIdToImageTest") public class RegionAndIdToImageTest { + @SuppressWarnings("unchecked") @Test public void testApply() { @@ -51,11 +52,11 @@ public class RegionAndIdToImageTest { AMIClient client = createMock(AMIClient.class); org.jclouds.aws.ec2.domain.Image ec2Image = createMock(org.jclouds.aws.ec2.domain.Image.class); Image image = createNiceMock(Image.class); - Set images = ImmutableSet + Set images = ImmutableSet . of(ec2Image); expect(caller.getAMIServices()).andReturn(client).atLeastOnce(); - expect(client.describeImagesInRegion("region", imageIds("ami"))).andReturn(images); + expect(client.describeImagesInRegion("region", imageIds("ami"))).andReturn((Set)images); expect(parser.apply(ec2Image)).andReturn(image); replay(caller); @@ -74,6 +75,7 @@ public class RegionAndIdToImageTest { } + @SuppressWarnings("unchecked") @Test public void testApplyNotFound() { @@ -82,11 +84,11 @@ public class RegionAndIdToImageTest { AMIClient client = createMock(AMIClient.class); org.jclouds.aws.ec2.domain.Image ec2Image = createMock(org.jclouds.aws.ec2.domain.Image.class); Image image = createNiceMock(Image.class); - Set images = ImmutableSet + Set images = ImmutableSet . of(ec2Image); expect(caller.getAMIServices()).andReturn(client).atLeastOnce(); - expect(client.describeImagesInRegion("region", imageIds("ami"))).andReturn(images); + expect(client.describeImagesInRegion("region", imageIds("ami"))).andReturn((Set)images); expect(parser.apply(ec2Image)).andThrow(new ResourceNotFoundException()); replay(caller); diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/functions/RunningInstanceToNodeMetadataTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/functions/RunningInstanceToNodeMetadataTest.java index 27e1e94f06..227d0d665f 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/functions/RunningInstanceToNodeMetadataTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/functions/RunningInstanceToNodeMetadataTest.java @@ -58,8 +58,7 @@ import com.google.common.collect.ImmutableSet; */ @Test(groups = "unit", testName = "ec2.RunningInstanceToNodeMetadataTest") public class RunningInstanceToNodeMetadataTest { - private static class ImageProvider implements - Provider> { + private static class ImageProvider implements Provider> { private final Set images; private ImageProvider(org.jclouds.compute.domain.Image jcImage) { @@ -117,8 +116,8 @@ public class RunningInstanceToNodeMetadataTest { replay(credentialProvider); replay(instance); - RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, - credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations, + RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, credentialsMap, + credentialProvider, new ImageProvider(jcImage), imageMap, locations, new RunningInstanceToStorageMappingUnix()); NodeMetadata metadata = parser.apply(instance); @@ -168,8 +167,8 @@ public class RunningInstanceToNodeMetadataTest { expect(jcImage.getProviderId()).andReturn("notImageId").atLeastOnce(); expect(instance.getRegion()).andReturn("us-east-1").atLeastOnce(); - expect(imageMap.get(new RegionAndName("us-east-1", "imageId"))).andThrow( - new NullPointerException()).atLeastOnce(); + expect(imageMap.get(new RegionAndName("us-east-1", "imageId"))).andThrow(new NullPointerException()) + .atLeastOnce(); expect(instance.getInstanceType()).andReturn(InstanceType.C1_XLARGE).atLeastOnce(); @@ -181,8 +180,8 @@ public class RunningInstanceToNodeMetadataTest { replay(credentialProvider); replay(instance); - RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, - credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations, + RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, credentialsMap, + credentialProvider, new ImageProvider(jcImage), imageMap, locations, new RunningInstanceToStorageMappingUnix()); NodeMetadata metadata = parser.apply(instance); @@ -234,8 +233,7 @@ public class RunningInstanceToNodeMetadataTest { org.jclouds.compute.domain.Image lateImage = createMock(org.jclouds.compute.domain.Image.class); - expect(imageMap.get(new RegionAndName("us-east-1", "imageId"))).andReturn(lateImage) - .atLeastOnce(); + expect(imageMap.get(new RegionAndName("us-east-1", "imageId"))).andReturn(lateImage).atLeastOnce(); expect(instance.getInstanceType()).andReturn(InstanceType.C1_XLARGE).atLeastOnce(); @@ -248,8 +246,8 @@ public class RunningInstanceToNodeMetadataTest { replay(credentialProvider); replay(instance); - RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, - credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations, + RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, credentialsMap, + credentialProvider, new ImageProvider(jcImage), imageMap, locations, new RunningInstanceToStorageMappingUnix()); NodeMetadata metadata = parser.apply(instance); @@ -270,8 +268,7 @@ public class RunningInstanceToNodeMetadataTest { @SuppressWarnings("unchecked") @Test - public void testApplyWithNoSecurityGroupCreatesTagOfIdPrefixedByTagAndNullCredentials() - throws UnknownHostException { + public void testApplyWithNoSecurityGroupCreatesTagOfIdPrefixedByTagAndNullCredentials() throws UnknownHostException { EC2Client client = createMock(EC2Client.class); AMIClient amiClient = createMock(AMIClient.class); Map credentialsMap = createMock(Map.class); @@ -312,8 +309,8 @@ public class RunningInstanceToNodeMetadataTest { replay(credentialProvider); replay(instance); - RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, - credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations, + RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, credentialsMap, + credentialProvider, new ImageProvider(jcImage), imageMap, locations, new RunningInstanceToStorageMappingUnix()); NodeMetadata metadata = parser.apply(instance); @@ -333,8 +330,7 @@ public class RunningInstanceToNodeMetadataTest { @SuppressWarnings("unchecked") @Test - public void testApplyWithNoKeyPairCreatesTagOfParsedSecurityGroupAndNullCredentials() - throws UnknownHostException { + public void testApplyWithNoKeyPairCreatesTagOfParsedSecurityGroupAndNullCredentials() throws UnknownHostException { EC2Client client = createMock(EC2Client.class); AMIClient amiClient = createMock(AMIClient.class); Map credentialsMap = createMock(Map.class); @@ -374,8 +370,8 @@ public class RunningInstanceToNodeMetadataTest { replay(credentialProvider); replay(instance); - RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, - credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations, + RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, credentialsMap, + credentialProvider, new ImageProvider(jcImage), imageMap, locations, new RunningInstanceToStorageMappingUnix()); NodeMetadata metadata = parser.apply(instance); @@ -395,8 +391,7 @@ public class RunningInstanceToNodeMetadataTest { @SuppressWarnings("unchecked") @Test - public void testApplyWithKeyPairCreatesTagOfParsedSecurityGroupAndCredentialsBasedOnIt() - throws UnknownHostException { + public void testApplyWithKeyPairCreatesTagOfParsedSecurityGroupAndCredentialsBasedOnIt() throws UnknownHostException { EC2Client client = createMock(EC2Client.class); AMIClient amiClient = createMock(AMIClient.class); expect(client.getAMIServices()).andReturn(amiClient).atLeastOnce(); @@ -429,7 +424,7 @@ public class RunningInstanceToNodeMetadataTest { expect(jcImage.getLocation()).andReturn(location).atLeastOnce(); expect(amiClient.describeImagesInRegion(Region.US_EAST_1, imageIds("imageId"))).andReturn( - ImmutableSet. of(image)); + (Set) ImmutableSet. of(image)); expect(credentialProvider.execute(image)).andReturn(new Credentials("user", "pass")); @@ -448,8 +443,8 @@ public class RunningInstanceToNodeMetadataTest { replay(instance); replay(jcImage); - RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, - credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations, + RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, credentialsMap, + credentialProvider, new ImageProvider(jcImage), imageMap, locations, new RunningInstanceToStorageMappingUnix()); NodeMetadata metadata = parser.apply(instance); @@ -483,8 +478,7 @@ public class RunningInstanceToNodeMetadataTest { Image image = createMock(Image.class); expect(instance.getId()).andReturn("id").atLeastOnce(); - expect(instance.getGroupIds()).andReturn(ImmutableSet.of("jclouds1", "jclouds2")) - .atLeastOnce(); + expect(instance.getGroupIds()).andReturn(ImmutableSet.of("jclouds1", "jclouds2")).atLeastOnce(); expect(instance.getKeyName()).andReturn("jclouds#tag#us-east-1#50").atLeastOnce(); expect(instance.getInstanceState()).andReturn(InstanceState.RUNNING); expect(instance.getVirtualizationType()).andReturn("paravirtual"); @@ -505,7 +499,7 @@ public class RunningInstanceToNodeMetadataTest { expect(jcImage.getLocation()).andReturn(location).atLeastOnce(); expect(amiClient.describeImagesInRegion(Region.US_EAST_1, imageIds("imageId"))).andReturn( - ImmutableSet. of(image)); + (Set) ImmutableSet. of(image)); expect(credentialProvider.execute(image)).andReturn(new Credentials("user", "pass")); @@ -524,8 +518,8 @@ public class RunningInstanceToNodeMetadataTest { replay(instance); replay(jcImage); - RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, - credentialsMap, credentialProvider, new ImageProvider(jcImage), imageMap, locations, + RunningInstanceToNodeMetadata parser = new RunningInstanceToNodeMetadata(client, credentialsMap, + credentialProvider, new ImageProvider(jcImage), imageMap, locations, new RunningInstanceToStorageMappingUnix()); NodeMetadata metadata = parser.apply(instance); diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/strategy/EC2RunNodesAndAddToSetStrategyTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/strategy/EC2RunNodesAndAddToSetStrategyTest.java index eb81b63ce0..bbcc49a8ba 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/compute/strategy/EC2RunNodesAndAddToSetStrategyTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/compute/strategy/EC2RunNodesAndAddToSetStrategyTest.java @@ -60,20 +60,17 @@ public class EC2RunNodesAndAddToSetStrategyTest { @Test public void testZoneAsALocation() { - assertRegionAndZoneForLocation(ZONE_AP_SOUTHEAST_1A, - Region.AP_SOUTHEAST_1, AvailabilityZone.AP_SOUTHEAST_1A); + assertRegionAndZoneForLocation(ZONE_AP_SOUTHEAST_1A, Region.AP_SOUTHEAST_1, AvailabilityZone.AP_SOUTHEAST_1A); } @Test public void testRegionAsALocation() { - assertRegionAndZoneForLocation(REGION_AP_SOUTHEAST_1, - Region.AP_SOUTHEAST_1, null); + assertRegionAndZoneForLocation(REGION_AP_SOUTHEAST_1, Region.AP_SOUTHEAST_1, null); } // // fixtures - public static Iterable containsNodeMetadata( - final NodeMetadata in) { + public static Iterable containsNodeMetadata(final NodeMetadata in) { reportMatcher(new IArgumentMatcher() { @Override @@ -92,8 +89,8 @@ public class EC2RunNodesAndAddToSetStrategyTest { return null; } - private void assertRegionAndZoneForLocation(Location location, - String region, String zone) { + @SuppressWarnings("unchecked") + private void assertRegionAndZoneForLocation(Location location, String region, String zone) { String imageId = "ami1"; String instanceCreatedId = "instance1"; // setup mocks @@ -102,37 +99,28 @@ public class EC2RunNodesAndAddToSetStrategyTest { InstanceClient instanceClient = createMock(InstanceClient.class); RunInstancesOptions ec2Options = createMock(RunInstancesOptions.class); RunningInstance instance = createMock(RunningInstance.class); - Reservation reservation = new Reservation(region, ImmutableSet - . of(), ImmutableSet. of(instance), - "ownerId", "requesterId", "reservationId"); + Reservation reservation = new Reservation(region, ImmutableSet + . of(), ImmutableSet. of(instance), "ownerId", "requesterId", "reservationId"); NodeMetadata nodeMetadata = createMock(NodeMetadata.class); // setup expectations - expect(strategy.client.getInstanceServices()).andReturn(instanceClient) - .atLeastOnce(); + expect(strategy.client.getInstanceServices()).andReturn(instanceClient).atLeastOnce(); expect( - strategy.createKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions - .execute(region, input.tag, input.template)).andReturn( - ec2Options); - expect(input.template.getLocation()).andReturn(input.location) - .atLeastOnce(); + strategy.createKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions.execute(region, input.tag, + input.template)).andReturn(ec2Options); + expect(input.template.getLocation()).andReturn(input.location).atLeastOnce(); expect(input.template.getImage()).andReturn(input.image).atLeastOnce(); expect(input.image.getProviderId()).andReturn(imageId).atLeastOnce(); - expect( - instanceClient.runInstancesInRegion(region, zone, imageId, 1, - input.count, ec2Options)).andReturn(reservation); + expect(instanceClient.runInstancesInRegion(region, zone, imageId, 1, input.count, ec2Options)).andReturn( + (Reservation)reservation); expect(instance.getId()).andReturn(instanceCreatedId).atLeastOnce(); expect(strategy.instancePresent.apply(instance)).andReturn(true); - expect(input.template.getOptions()).andReturn(input.options) - .atLeastOnce(); + expect(input.template.getOptions()).andReturn(input.options).atLeastOnce(); - expect(strategy.runningInstanceToNodeMetadata.apply(instance)).andReturn( - nodeMetadata); + expect(strategy.runningInstanceToNodeMetadata.apply(instance)).andReturn(nodeMetadata); expect( - strategy.utils - .runOptionsOnNodesAndAddToGoodSetOrPutExceptionIntoBadMap( - eq(input.options), containsNodeMetadata(nodeMetadata), - eq(input.nodes), eq(input.badNodes))).andReturn(null); + strategy.utils.runOptionsOnNodesAndAddToGoodSetOrPutExceptionIntoBadMap(eq(input.options), + containsNodeMetadata(nodeMetadata), eq(input.nodes), eq(input.badNodes))).andReturn(null); // replay mocks replay(instanceClient); @@ -143,8 +131,7 @@ public class EC2RunNodesAndAddToSetStrategyTest { replayStrategy(strategy); // run - strategy.execute(input.tag, input.count, input.template, input.nodes, - input.badNodes); + strategy.execute(input.tag, input.count, input.template, input.nodes, input.badNodes); // verify mocks verify(instanceClient); @@ -155,12 +142,10 @@ public class EC2RunNodesAndAddToSetStrategyTest { verifyStrategy(strategy); } - private static final Location REGION_AP_SOUTHEAST_1 = new LocationImpl( - LocationScope.REGION, Region.AP_SOUTHEAST_1, Region.AP_SOUTHEAST_1, - new LocationImpl(LocationScope.PROVIDER, "ec2", "ec2", null)); - private static final Location ZONE_AP_SOUTHEAST_1A = new LocationImpl( - LocationScope.ZONE, AvailabilityZone.AP_SOUTHEAST_1A, - AvailabilityZone.AP_SOUTHEAST_1A, REGION_AP_SOUTHEAST_1); + private static final Location REGION_AP_SOUTHEAST_1 = new LocationImpl(LocationScope.REGION, Region.AP_SOUTHEAST_1, + Region.AP_SOUTHEAST_1, new LocationImpl(LocationScope.PROVIDER, "ec2", "ec2", null)); + private static final Location ZONE_AP_SOUTHEAST_1A = new LocationImpl(LocationScope.ZONE, + AvailabilityZone.AP_SOUTHEAST_1A, AvailabilityZone.AP_SOUTHEAST_1A, REGION_AP_SOUTHEAST_1); // ///////////////////////////////////////////////////////////////////// @SuppressWarnings("unchecked") @@ -213,9 +198,8 @@ public class EC2RunNodesAndAddToSetStrategyTest { Predicate instanceStateRunning = createMock(Predicate.class); RunningInstanceToNodeMetadata runningInstanceToNodeMetadata = createMock(RunningInstanceToNodeMetadata.class); ComputeUtils utils = createMock(ComputeUtils.class); - return new EC2RunNodesAndAddToSetStrategy(client, - createKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions, - instanceStateRunning, runningInstanceToNodeMetadata, utils); + return new EC2RunNodesAndAddToSetStrategy(client, createKeyPairAndSecurityGroupsAsNeededAndReturnRunOptions, + instanceStateRunning, runningInstanceToNodeMetadata, utils); } private void replayStrategy(EC2RunNodesAndAddToSetStrategy strategy) { diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/services/InstanceClientLiveTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/services/InstanceClientLiveTest.java index 8a50b45e26..39867e13b5 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/services/InstanceClientLiveTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/services/InstanceClientLiveTest.java @@ -28,6 +28,7 @@ import org.jclouds.aws.domain.Region; import org.jclouds.aws.ec2.EC2AsyncClient; import org.jclouds.aws.ec2.EC2Client; import org.jclouds.aws.ec2.domain.Reservation; +import org.jclouds.aws.ec2.domain.RunningInstance; import org.jclouds.logging.log4j.config.Log4JLoggingModule; import org.jclouds.rest.RestContext; import org.jclouds.rest.RestContextFactory; @@ -64,9 +65,9 @@ public class InstanceClientLiveTest { @Test void testDescribeInstances() { - for (String region : Lists.newArrayList(null, Region.EU_WEST_1, Region.US_EAST_1, - Region.US_WEST_1, Region.AP_SOUTHEAST_1)) { - Set allResults = client.describeInstancesInRegion(region); + for (String region : Lists.newArrayList(null, Region.EU_WEST_1, Region.US_EAST_1, Region.US_WEST_1, + Region.AP_SOUTHEAST_1)) { + Set> allResults = client.describeInstancesInRegion(region); assertNotNull(allResults); assert allResults.size() >= 0 : allResults.size(); } diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/xml/DescribeInstancesResponseHandlerTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/xml/DescribeInstancesResponseHandlerTest.java index 59bad9f49a..ebb08f6dee 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/xml/DescribeInstancesResponseHandlerTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/xml/DescribeInstancesResponseHandlerTest.java @@ -66,10 +66,10 @@ public class DescribeInstancesResponseHandlerTest extends BaseEC2HandlerTest { public void testWhenRunning() throws UnknownHostException { InputStream is = getClass().getResourceAsStream("/ec2/describe_instances_running.xml"); - Set contents = Sets.newLinkedHashSet(); + Set> contents = Sets.newLinkedHashSet(); - contents.add(new Reservation(defaultRegion, ImmutableSet.of("adriancole.ec2ingress"), ImmutableSet - .of(new RunningInstance(defaultRegion, ImmutableSet.of("adriancole.ec2ingress"), "0", + contents.add(new Reservation(defaultRegion, ImmutableSet.of("adriancole.ec2ingress"), + ImmutableSet.of(new RunningInstance(defaultRegion, ImmutableSet.of("adriancole.ec2ingress"), "0", "ec2-174-129-81-68.compute-1.amazonaws.com", "ami-1fd73376", "i-0799056f", InstanceState.RUNNING, InstanceType.M1_SMALL, "174.129.81.68", "aki-a71cf9ce", "adriancole.ec21", dateService.iso8601DateParse("2009-11-09T03:00:34.000Z"), false, @@ -78,7 +78,7 @@ public class DescribeInstancesResponseHandlerTest extends BaseEC2HandlerTest { RootDeviceType.INSTANCE_STORE, null, ImmutableMap. of())), "993194456877", null, "r-a3c508cb")); - Set result = getReservations(is); + Set> result = getReservations(is); assertEquals(result, contents); } @@ -86,23 +86,25 @@ public class DescribeInstancesResponseHandlerTest extends BaseEC2HandlerTest { public void testApplyInputStream() { InputStream is = getClass().getResourceAsStream("/ec2/describe_instances.xml"); - Set contents = Sets.newLinkedHashSet(); + Set> contents = Sets.newLinkedHashSet(); - contents.add(new Reservation(defaultRegion, ImmutableSet.of("default"), ImmutableSet.of(new RunningInstance( - defaultRegion, ImmutableSet.of("default"), "23", "ec2-72-44-33-4.compute-1.amazonaws.com", - "ami-6ea54007", "i-28a64341", InstanceState.RUNNING, InstanceType.M1_LARGE, (String) null, - "aki-ba3adfd3", "example-key-name", dateService.iso8601DateParse("2007-08-07T11:54:42.000Z"), false, - AvailabilityZone.US_EAST_1B, null, "paravirtual", null, "10-251-50-132.ec2.internal", null, ImmutableSet - .of("774F4FF8"), "ari-badbad00", null, null, null, RootDeviceType.INSTANCE_STORE, null, - ImmutableMap. of()), new RunningInstance(defaultRegion, ImmutableSet - .of("default"), "23", "ec2-72-44-33-6.compute-1.amazonaws.com", "ami-6ea54007", "i-28a64435", - InstanceState.RUNNING, InstanceType.M1_LARGE, (String) null, "aki-ba3adfd3", "example-key-name", - dateService.iso8601DateParse("2007-08-07T11:54:42.000Z"), false, AvailabilityZone.US_EAST_1B, null, - "paravirtual", null, "10-251-50-134.ec2.internal", null, ImmutableSet.of("774F4FF8"), "ari-badbad00", - null, null, null, RootDeviceType.INSTANCE_STORE, null, ImmutableMap. of())), - "UYY3TLBUXIEON5NQVUUX6OMPWBZIQNFM", null, "r-44a5402d")); + contents.add(new Reservation(defaultRegion, ImmutableSet.of("default"), ImmutableSet.of( + new RunningInstance(defaultRegion, ImmutableSet.of("default"), "23", + "ec2-72-44-33-4.compute-1.amazonaws.com", "ami-6ea54007", "i-28a64341", InstanceState.RUNNING, + InstanceType.M1_LARGE, (String) null, "aki-ba3adfd3", "example-key-name", dateService + .iso8601DateParse("2007-08-07T11:54:42.000Z"), false, AvailabilityZone.US_EAST_1B, + null, "paravirtual", null, "10-251-50-132.ec2.internal", null, ImmutableSet.of("774F4FF8"), + "ari-badbad00", null, null, null, RootDeviceType.INSTANCE_STORE, null, ImmutableMap + . of()), new RunningInstance(defaultRegion, ImmutableSet + .of("default"), "23", "ec2-72-44-33-6.compute-1.amazonaws.com", "ami-6ea54007", "i-28a64435", + InstanceState.RUNNING, InstanceType.M1_LARGE, (String) null, "aki-ba3adfd3", + "example-key-name", dateService.iso8601DateParse("2007-08-07T11:54:42.000Z"), false, + AvailabilityZone.US_EAST_1B, null, "paravirtual", null, "10-251-50-134.ec2.internal", null, + ImmutableSet.of("774F4FF8"), "ari-badbad00", null, null, null, RootDeviceType.INSTANCE_STORE, + null, ImmutableMap. of())), "UYY3TLBUXIEON5NQVUUX6OMPWBZIQNFM", null, + "r-44a5402d")); - Set result = getReservations(is); + Set> result = getReservations(is); assertEquals(result, contents); } @@ -112,17 +114,17 @@ public class DescribeInstancesResponseHandlerTest extends BaseEC2HandlerTest { public void testApplyInputStreamEuc() { InputStream is = getClass().getResourceAsStream("/ec2/describe_instances_euc.xml"); - Set contents = Sets.newLinkedHashSet(); + Set> contents = Sets.newLinkedHashSet(); - contents.add(new Reservation(defaultRegion, ImmutableSet.of("default"), ImmutableSet.of(new RunningInstance( - defaultRegion, ImmutableSet.of("jclouds#euc"), "1", null, "emi-9ACB1363", "i-3FFA0762", - InstanceState.SHUTTING_DOWN, InstanceType.M1_LARGE, null, "eki-6CBD12F2", "jclouds#euc-17", dateService - .iso8601DateParse("2010-06-16T03:06:19.000Z"), false, "open", null, "paravirtual", null, - "10.7.0.179", null, ImmutableSet. of(), "eri-A97113E4", null, null, null, - RootDeviceType.INSTANCE_STORE, null, ImmutableMap. of())), "jclouds", null, - "r-4D2A08AD")); + contents.add(new Reservation(defaultRegion, ImmutableSet.of("default"), ImmutableSet + .of(new RunningInstance(defaultRegion, ImmutableSet.of("jclouds#euc"), "1", null, "emi-9ACB1363", + "i-3FFA0762", InstanceState.SHUTTING_DOWN, InstanceType.M1_LARGE, null, "eki-6CBD12F2", + "jclouds#euc-17", dateService.iso8601DateParse("2010-06-16T03:06:19.000Z"), false, "open", + null, "paravirtual", null, "10.7.0.179", null, ImmutableSet. of(), "eri-A97113E4", + null, null, null, RootDeviceType.INSTANCE_STORE, null, ImmutableMap + . of())), "jclouds", null, "r-4D2A08AD")); - Set result = getReservations(is); + Set> result = getReservations(is); assertEquals(result, contents); } @@ -130,29 +132,30 @@ public class DescribeInstancesResponseHandlerTest extends BaseEC2HandlerTest { public void testEBS() throws UnknownHostException { InputStream is = getClass().getResourceAsStream("/ec2/describe_instances_ebs.xml"); - Set contents = Sets.newLinkedHashSet(); + Set> contents = Sets.newLinkedHashSet(); - contents.add(new Reservation(defaultRegion, ImmutableSet.of("adriancole.ec2ebsingress"), ImmutableSet - .of(new RunningInstance(defaultRegion, ImmutableSet.of("adriancole.ec2ebsingress"), "0", + contents.add(new Reservation(defaultRegion, ImmutableSet.of("adriancole.ec2ebsingress"), + ImmutableSet.of(new RunningInstance(defaultRegion, ImmutableSet.of("adriancole.ec2ebsingress"), "0", "ec2-75-101-203-146.compute-1.amazonaws.com", "ami-849875ed", "i-e564438d", InstanceState.RUNNING, InstanceType.M1_SMALL, "75.101.203.146", "aki-a71cf9ce", "adriancole.ec2ebs1", dateService.iso8601DateParse("2009-12-30T04:06:23.000Z"), false, - AvailabilityZone.US_EAST_1B, "placement","hvm", null, "domU-12-31-39-09-CE-53.compute-1.internal", - "10.210.209.157", ImmutableSet. of(), "ari-a51cf9cc", null, null, null, - RootDeviceType.EBS, "/dev/sda1", ImmutableMap. of("/dev/sda1", - new EbsBlockDevice("vol-dc6ca8b5", Attachment.Status.ATTACHED, dateService - .iso8601DateParse("2009-12-30T04:06:29.000Z"), true)))), "993194456877", - null, "r-596dd731")); + AvailabilityZone.US_EAST_1B, "placement", "hvm", null, + "domU-12-31-39-09-CE-53.compute-1.internal", "10.210.209.157", ImmutableSet. of(), + "ari-a51cf9cc", null, null, null, RootDeviceType.EBS, "/dev/sda1", ImmutableMap + . of("/dev/sda1", new EbsBlockDevice("vol-dc6ca8b5", + Attachment.Status.ATTACHED, dateService + .iso8601DateParse("2009-12-30T04:06:29.000Z"), true)))), + "993194456877", null, "r-596dd731")); - Set result = getReservations(is); + Set> result = getReservations(is); assertEquals(result, contents); } - private Set getReservations(InputStream is) { + private Set> getReservations(InputStream is) { DescribeInstancesResponseHandler handler = injector.getInstance(DescribeInstancesResponseHandler.class); addDefaultRegionToHandler(handler); - Set result = factory.create(handler).parse(is); + Set> result = factory.create(handler).parse(is); return result; } diff --git a/aws/core/src/test/java/org/jclouds/aws/ec2/xml/RunInstancesResponseHandlerTest.java b/aws/core/src/test/java/org/jclouds/aws/ec2/xml/RunInstancesResponseHandlerTest.java index 7b71028b32..bdab9fe044 100644 --- a/aws/core/src/test/java/org/jclouds/aws/ec2/xml/RunInstancesResponseHandlerTest.java +++ b/aws/core/src/test/java/org/jclouds/aws/ec2/xml/RunInstancesResponseHandlerTest.java @@ -64,28 +64,29 @@ public class RunInstancesResponseHandlerTest extends BaseEC2HandlerTest { InputStream is = getClass().getResourceAsStream("/ec2/run_instances.xml"); - Reservation expected = new Reservation(defaultRegion, ImmutableSet.of("default"), ImmutableSet.of( - new RunningInstance(defaultRegion, ImmutableSet.of("default"), "0", null, "ami-60a54009", "i-2ba64342", - InstanceState.PENDING, InstanceType.M1_SMALL, (String) null, null, "example-key-name", - dateService.iso8601DateParse("2007-08-07T11:51:50.000Z"), true, AvailabilityZone.US_EAST_1B, - null, "paravirtual", null, (String) null, null, Sets. newLinkedHashSet(), null, null, - null, null, RootDeviceType.INSTANCE_STORE, null, ImmutableMap. of()), - new RunningInstance(defaultRegion, ImmutableSet.of("default"), "1", null, "ami-60a54009", "i-2bc64242", - InstanceState.PENDING, InstanceType.M1_SMALL, (String) null, null, "example-key-name", - dateService.iso8601DateParse("2007-08-07T11:51:50.000Z"), true, AvailabilityZone.US_EAST_1B, - null, "paravirtual", null, (String) null, null, Sets. newLinkedHashSet(), null, null, - null, null, RootDeviceType.INSTANCE_STORE, null, ImmutableMap. of()), - new RunningInstance(defaultRegion, ImmutableSet.of("default"), "2", null, "ami-60a54009", "i-2be64332", - InstanceState.PENDING, InstanceType.M1_SMALL, (String) null, null, "example-key-name", - dateService.iso8601DateParse("2007-08-07T11:51:50.000Z"), true, AvailabilityZone.US_EAST_1B, - null, "paravirtual", null, (String) null, null, Sets. newLinkedHashSet(), null, null, - null, null, RootDeviceType.INSTANCE_STORE, null, ImmutableMap. of()) + Reservation expected = new Reservation(defaultRegion, ImmutableSet + .of("default"), ImmutableSet.of(new RunningInstance(defaultRegion, ImmutableSet.of("default"), "0", + null, "ami-60a54009", "i-2ba64342", InstanceState.PENDING, InstanceType.M1_SMALL, (String) null, null, + "example-key-name", dateService.iso8601DateParse("2007-08-07T11:51:50.000Z"), true, + AvailabilityZone.US_EAST_1B, null, "paravirtual", null, (String) null, null, Sets + . newLinkedHashSet(), null, null, null, null, RootDeviceType.INSTANCE_STORE, null, + ImmutableMap. of()), new RunningInstance(defaultRegion, ImmutableSet + .of("default"), "1", null, "ami-60a54009", "i-2bc64242", InstanceState.PENDING, InstanceType.M1_SMALL, + (String) null, null, "example-key-name", dateService.iso8601DateParse("2007-08-07T11:51:50.000Z"), true, + AvailabilityZone.US_EAST_1B, null, "paravirtual", null, (String) null, null, Sets + . newLinkedHashSet(), null, null, null, null, RootDeviceType.INSTANCE_STORE, null, + ImmutableMap. of()), new RunningInstance(defaultRegion, ImmutableSet + .of("default"), "2", null, "ami-60a54009", "i-2be64332", InstanceState.PENDING, InstanceType.M1_SMALL, + (String) null, null, "example-key-name", dateService.iso8601DateParse("2007-08-07T11:51:50.000Z"), true, + AvailabilityZone.US_EAST_1B, null, "paravirtual", null, (String) null, null, Sets + . newLinkedHashSet(), null, null, null, null, RootDeviceType.INSTANCE_STORE, null, + ImmutableMap. of()) ), "AIDADH4IGTRXXKCD", null, "r-47a5402e"); RunInstancesResponseHandler handler = injector.getInstance(RunInstancesResponseHandler.class); addDefaultRegionToHandler(handler); - Reservation result = factory.create(handler).parse(is); + Reservation result = factory.create(handler).parse(is); assertEquals(result, expected); } diff --git a/aws/core/src/test/java/org/jclouds/aws/s3/internal/StubS3AsyncClient.java b/aws/core/src/test/java/org/jclouds/aws/s3/internal/StubS3AsyncClient.java index be1999b901..cfa4c49a9f 100755 --- a/aws/core/src/test/java/org/jclouds/aws/s3/internal/StubS3AsyncClient.java +++ b/aws/core/src/test/java/org/jclouds/aws/s3/internal/StubS3AsyncClient.java @@ -19,9 +19,9 @@ package org.jclouds.aws.s3.internal; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.util.concurrent.Futures.compose; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; +import static org.jclouds.concurrent.ConcurrentUtils.compose; import java.util.Date; import java.util.Map; @@ -67,7 +67,6 @@ import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.MutableBlobMetadata; import org.jclouds.blobstore.functions.HttpGetOptionsListToGetOptions; import org.jclouds.blobstore.options.ListContainerOptions; -import org.jclouds.concurrent.ConcurrentUtils; import org.jclouds.date.DateService; import org.jclouds.domain.Location; import org.jclouds.domain.LocationScope; @@ -97,20 +96,19 @@ public class StubS3AsyncClient implements S3AsyncClient { private final BlobToObjectMetadata blob2ObjectMetadata; private final BucketToContainerListOptions bucket2ContainerListOptions; private final ResourceToBucketList resource2BucketList; - private final ExecutorService executorService; private final ConcurrentMap> containerToBlobs; private final ConcurrentMap containerToLocation; + private final ExecutorService service; @Inject - private StubS3AsyncClient(TransientAsyncBlobStore blobStore, - ConcurrentMap> containerToBlobs, + private StubS3AsyncClient(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, + TransientAsyncBlobStore blobStore, ConcurrentMap> containerToBlobs, ConcurrentMap containerToLocation, DateService dateService, S3Object.Factory objectProvider, Blob.Factory blobProvider, - HttpGetOptionsListToGetOptions httpGetOptionsConverter, ObjectToBlob object2Blob, - BlobToObject blob2Object, BlobToObjectMetadata blob2ObjectMetadata, - BucketToContainerListOptions bucket2ContainerListOptions, - ResourceToBucketList resource2BucketList, - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executorService) { + HttpGetOptionsListToGetOptions httpGetOptionsConverter, ObjectToBlob object2Blob, BlobToObject blob2Object, + BlobToObjectMetadata blob2ObjectMetadata, BucketToContainerListOptions bucket2ContainerListOptions, + ResourceToBucketList resource2BucketList) { + this.service = service; this.containerToBlobs = containerToBlobs; this.containerToLocation = containerToLocation; this.blobStore = blobStore; @@ -121,10 +119,8 @@ public class StubS3AsyncClient implements S3AsyncClient { this.object2Blob = checkNotNull(object2Blob, "object2Blob"); this.blob2Object = checkNotNull(blob2Object, "blob2Object"); this.blob2ObjectMetadata = checkNotNull(blob2ObjectMetadata, "blob2ObjectMetadata"); - this.bucket2ContainerListOptions = checkNotNull(bucket2ContainerListOptions, - "bucket2ContainerListOptions"); + this.bucket2ContainerListOptions = checkNotNull(bucket2ContainerListOptions, "bucket2ContainerListOptions"); this.resource2BucketList = checkNotNull(resource2BucketList, "resource2BucketList"); - this.executorService = checkNotNull(executorService, "executorService"); } public static final String TEST_ACL_ID = "1a405254c932b52e5b5caaa88186bc431a1bacb9ece631f835daddaf0c47677c"; @@ -141,24 +137,19 @@ public class StubS3AsyncClient implements S3AsyncClient { public ListenableFuture putBucketInRegion(@Nullable String region, String name, PutBucketOptions... optionsList) { region = region == null ? Region.US_STANDARD : region; - final PutBucketOptions options = (optionsList.length == 0) ? new PutBucketOptions() - : optionsList[0]; + final PutBucketOptions options = (optionsList.length == 0) ? new PutBucketOptions() : optionsList[0]; keyToAcl.put(name, options.getAcl()); - return blobStore.createContainerInLocation(new LocationImpl(LocationScope.REGION, region, - region, null), name); + return blobStore.createContainerInLocation(new LocationImpl(LocationScope.REGION, region, region, null), name); } - public ListenableFuture listBucket(final String name, - ListBucketOptions... optionsList) { + public ListenableFuture listBucket(final String name, ListBucketOptions... optionsList) { ListContainerOptions options = bucket2ContainerListOptions.apply(optionsList); - return compose(blobStore.list(name, options), resource2BucketList); + return compose(blobStore.list(name, options), resource2BucketList, service); } - public ListenableFuture copyObject(final String sourceBucket, - final String sourceObject, final String destinationBucket, - final String destinationObject, CopyObjectOptions... nullableOptions) { - final CopyObjectOptions options = (nullableOptions.length == 0) ? new CopyObjectOptions() - : nullableOptions[0]; + public ListenableFuture copyObject(final String sourceBucket, final String sourceObject, + final String destinationBucket, final String destinationObject, CopyObjectOptions... nullableOptions) { + final CopyObjectOptions options = (nullableOptions.length == 0) ? new CopyObjectOptions() : nullableOptions[0]; ConcurrentMap source = containerToBlobs.get(sourceBucket); ConcurrentMap dest = containerToBlobs.get(destinationBucket); if (source.containsKey(sourceObject)) { @@ -183,8 +174,7 @@ public class StubS3AsyncClient implements S3AsyncClient { return immediateFailedFuture(TransientAsyncBlobStore.returnResponseException(412)); } Blob sourceS3 = source.get(sourceObject); - MutableBlobMetadata newMd = TransientAsyncBlobStore.copy(sourceS3.getMetadata(), - destinationObject); + MutableBlobMetadata newMd = TransientAsyncBlobStore.copy(sourceS3.getMetadata(), destinationObject); if (options.getAcl() != null) keyToAcl.put(destinationBucket + "/" + destinationObject, options.getAcl()); @@ -192,17 +182,15 @@ public class StubS3AsyncClient implements S3AsyncClient { Blob newBlob = blobProvider.create(newMd); newBlob.setPayload(sourceS3.getPayload()); dest.put(destinationObject, newBlob); - return immediateFuture((ObjectMetadata) blob2ObjectMetadata.apply(TransientAsyncBlobStore - .copy(newMd))); + return immediateFuture((ObjectMetadata) blob2ObjectMetadata.apply(TransientAsyncBlobStore.copy(newMd))); } - return immediateFailedFuture(new KeyNotFoundException(sourceBucket, sourceObject, - sourceBucket + "/" + sourceObject)); + return immediateFailedFuture(new KeyNotFoundException(sourceBucket, sourceObject, sourceBucket + "/" + + sourceObject)); } public ListenableFuture putObject(final String bucketName, final S3Object object, PutObjectOptions... nullableOptions) { - final PutObjectOptions options = (nullableOptions.length == 0) ? new PutObjectOptions() - : nullableOptions[0]; + final PutObjectOptions options = (nullableOptions.length == 0) ? new PutObjectOptions() : nullableOptions[0]; if (options.getAcl() != null) keyToAcl.put(bucketName + "/" + object.getMetadata().getKey(), options.getAcl()); return blobStore.putBlob(bucketName, object2Blob.apply(object)); @@ -214,12 +202,10 @@ public class StubS3AsyncClient implements S3AsyncClient { if (aclObj instanceof AccessControlList) { acl = (AccessControlList) aclObj; } else if (aclObj instanceof CannedAccessPolicy) { - acl = AccessControlList.fromCannedAccessPolicy((CannedAccessPolicy) aclObj, - DEFAULT_OWNER_ID); + acl = AccessControlList.fromCannedAccessPolicy((CannedAccessPolicy) aclObj, DEFAULT_OWNER_ID); } else if (aclObj == null) { // Default to private access policy - acl = AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE, - DEFAULT_OWNER_ID); + acl = AccessControlList.fromCannedAccessPolicy(CannedAccessPolicy.PRIVATE, DEFAULT_OWNER_ID); } return acl; } @@ -228,8 +214,7 @@ public class StubS3AsyncClient implements S3AsyncClient { return immediateFuture(getACLforS3Item(bucket)); } - public ListenableFuture getObjectACL(final String bucket, - final String objectKey) { + public ListenableFuture getObjectACL(final String bucket, final String objectKey) { return immediateFuture(getACLforS3Item(bucket + "/" + objectKey)); } @@ -248,8 +233,7 @@ public class StubS3AsyncClient implements S3AsyncClient { for (Grant grant : acl.getGrants()) { if (grant.getGrantee() instanceof EmailAddressGrantee) { EmailAddressGrantee emailGrantee = (EmailAddressGrantee) grant.getGrantee(); - String id = emailGrantee.getEmailAddress().equals(TEST_ACL_EMAIL) ? TEST_ACL_ID : acl - .getOwner().getId(); + String id = emailGrantee.getEmailAddress().equals(TEST_ACL_EMAIL) ? TEST_ACL_ID : acl.getOwner().getId(); grant.setGrantee(new CanonicalUserGrantee(id, acl.getOwner().getDisplayName())); } } @@ -286,20 +270,18 @@ public class StubS3AsyncClient implements S3AsyncClient { return blobStore.removeBlob(bucketName, key); } - public ListenableFuture getObject(final String bucketName, final String key, - final GetOptions... options) { + public ListenableFuture getObject(final String bucketName, final String key, final GetOptions... options) { org.jclouds.blobstore.options.GetOptions getOptions = httpGetOptionsConverter.apply(options); - return compose(blobStore.getBlob(bucketName, key, getOptions), blob2Object); + return compose(blobStore.getBlob(bucketName, key, getOptions), blob2Object, service); } public ListenableFuture headObject(String bucketName, String key) { - return compose(ConcurrentUtils.makeListenable(blobStore.blobMetadata(bucketName, key), - executorService), new Function() { + return compose(blobStore.blobMetadata(bucketName, key), new Function() { @Override public ObjectMetadata apply(BlobMetadata from) { return blob2ObjectMetadata.apply(from); } - }); + }, service); } public ListenableFuture> listOwnedBuckets() { diff --git a/aws/demos/createlamp/src/main/java/org/jclouds/aws/ec2/demos/createlamp/MainApp.java b/aws/demos/createlamp/src/main/java/org/jclouds/aws/ec2/demos/createlamp/MainApp.java index 975f50af1b..901be38b9c 100755 --- a/aws/demos/createlamp/src/main/java/org/jclouds/aws/ec2/demos/createlamp/MainApp.java +++ b/aws/demos/createlamp/src/main/java/org/jclouds/aws/ec2/demos/createlamp/MainApp.java @@ -72,8 +72,8 @@ public class MainApp { String name = args[3]; // Init - RestContext context = new RestContextFactory().createContext( - "ec2", accesskeyid, secretkey); + RestContext context = new RestContextFactory().createContext("ec2", accesskeyid, + secretkey); // Get a synchronous client EC2Client client = context.getApi(); @@ -107,8 +107,7 @@ public class MainApp { try { String id = findInstanceByKeyName(client, name).getId(); System.out.printf("%d: %s terminating instance%n", System.currentTimeMillis(), id); - client.getInstanceServices().terminateInstancesInRegion(null, - findInstanceByKeyName(client, name).getId()); + client.getInstanceServices().terminateInstancesInRegion(null, findInstanceByKeyName(client, name).getId()); } catch (NoSuchElementException e) { } catch (Exception e) { e.printStackTrace(); @@ -129,8 +128,8 @@ public class MainApp { } } - private static RunningInstance createSecurityGroupKeyPairAndInstance(EC2Client client, - String name) throws TimeoutException { + private static RunningInstance createSecurityGroupKeyPairAndInstance(EC2Client client, String name) + throws TimeoutException { // create a new security group createSecurityGroupAndAuthorizePorts(client, name); @@ -145,8 +144,8 @@ public class MainApp { System.out.printf("%d: creating security group: %s%n", System.currentTimeMillis(), name); client.getSecurityGroupServices().createSecurityGroupInRegion(null, name, name); for (int port : new int[] { 80, 8080, 443, 22 }) { - client.getSecurityGroupServices().authorizeSecurityGroupIngressInRegion(null, name, - IpProtocol.TCP, port, port, "0.0.0.0/0"); + client.getSecurityGroupServices().authorizeSecurityGroupIngressInRegion(null, name, IpProtocol.TCP, port, + port, "0.0.0.0/0"); } } @@ -163,7 +162,7 @@ public class MainApp { .build(OsFamily.UNIX); System.out.printf("%d: running instance%n", System.currentTimeMillis()); - Reservation reservation = client.getInstanceServices().runInstancesInRegion(null, null, // allow + Reservation reservation = client.getInstanceServices().runInstancesInRegion(null, null, // allow // ec2 // to // chose @@ -182,43 +181,38 @@ public class MainApp { } - static RunningInstance blockUntilInstanceRunning(EC2Client client, RunningInstance instance) - throws TimeoutException { + static RunningInstance blockUntilInstanceRunning(EC2Client client, RunningInstance instance) throws TimeoutException { // create utilities that wait for the instance to finish RetryablePredicate runningTester = new RetryablePredicate( new InstanceStateRunning(client), 180, 5, TimeUnit.SECONDS); - System.out.printf("%d: %s awaiting instance to run %n", System.currentTimeMillis(), instance - .getId()); + System.out.printf("%d: %s awaiting instance to run %n", System.currentTimeMillis(), instance.getId()); if (!runningTester.apply(instance)) throw new TimeoutException("timeout waiting for instance to run: " + instance.getId()); instance = findInstanceById(client, instance.getId()); - RetryablePredicate socketTester = new RetryablePredicate( - new InetSocketAddressConnect(), 300, 1, TimeUnit.SECONDS); - System.out.printf("%d: %s awaiting ssh service to start%n", System.currentTimeMillis(), - instance.getIpAddress()); + RetryablePredicate socketTester = new RetryablePredicate(new InetSocketAddressConnect(), 300, + 1, TimeUnit.SECONDS); + System.out.printf("%d: %s awaiting ssh service to start%n", System.currentTimeMillis(), instance.getIpAddress()); if (!socketTester.apply(new IPSocket(instance.getIpAddress(), 22))) throw new TimeoutException("timeout waiting for ssh to start: " + instance.getIpAddress()); - System.out.printf("%d: %s ssh service started%n", System.currentTimeMillis(), instance - .getIpAddress()); + System.out.printf("%d: %s ssh service started%n", System.currentTimeMillis(), instance.getIpAddress()); - System.out.printf("%d: %s awaiting http service to start%n", System.currentTimeMillis(), - instance.getIpAddress()); + System.out.printf("%d: %s awaiting http service to start%n", System.currentTimeMillis(), instance.getIpAddress()); if (!socketTester.apply(new IPSocket(instance.getIpAddress(), 80))) throw new TimeoutException("timeout waiting for http to start: " + instance.getIpAddress()); - System.out.printf("%d: %s http service started%n", System.currentTimeMillis(), instance - .getIpAddress()); + System.out.printf("%d: %s http service started%n", System.currentTimeMillis(), instance.getIpAddress()); return instance; } private static RunningInstance findInstanceById(EC2Client client, String instanceId) { // search my account for the instance I just created - Set reservations = client.getInstanceServices().describeInstancesInRegion(null, - instanceId); // last parameter (ids) narrows the search + Set> reservations = client.getInstanceServices() + .describeInstancesInRegion(null, instanceId); // last parameter (ids) narrows the + // search // since we refined by instanceId there should only be one instance return Iterables.getOnlyElement(Iterables.getOnlyElement(reservations)); @@ -226,11 +220,12 @@ public class MainApp { private static RunningInstance findInstanceByKeyName(EC2Client client, final String keyName) { // search my account for the instance I just created - Set reservations = client.getInstanceServices().describeInstancesInRegion(null); + Set> reservations = client.getInstanceServices() + .describeInstancesInRegion(null); // extract all the instances from all reservations Set allInstances = Sets.newHashSet(); - for (Reservation reservation : reservations) { + for (Reservation reservation : reservations) { allInstances.addAll(reservation); } @@ -239,8 +234,7 @@ public class MainApp { @Override public boolean apply(RunningInstance input) { - return input.getKeyName().equals(keyName) - && input.getInstanceState() != InstanceState.TERMINATED; + return input.getKeyName().equals(keyName) && input.getInstanceState() != InstanceState.TERMINATED; } }); diff --git a/aws/demos/speedtest-sqs/src/main/java/org/jclouds/aws/sqs/SpeedTest.java b/aws/demos/speedtest-sqs/src/main/java/org/jclouds/aws/sqs/SpeedTest.java index 320cfbc2f8..8908d1fc4a 100755 --- a/aws/demos/speedtest-sqs/src/main/java/org/jclouds/aws/sqs/SpeedTest.java +++ b/aws/demos/speedtest-sqs/src/main/java/org/jclouds/aws/sqs/SpeedTest.java @@ -18,7 +18,7 @@ */ package org.jclouds.aws.sqs; -import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; +import static org.jclouds.concurrent.ConcurrentUtils.sameThreadExecutor; import static org.jclouds.aws.sqs.options.ListQueuesOptions.Builder.queuePrefix; import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion; @@ -38,7 +38,7 @@ import org.jclouds.rest.RestContextFactory; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; import com.google.inject.Module; /** @@ -133,7 +133,7 @@ public class SpeedTest { queue.getName()); // fire off all the messages for the test - Map> responses = Maps.newHashMap(); + Map> responses = Maps.newHashMap(); for (int i = 0; i < messageCount; i++) { responses.put(new QueueMessage(queue, message), context.getAsyncApi().sendMessage( queue, message)); diff --git a/aws/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java b/aws/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java index 4b148528c0..42bff71987 100755 --- a/aws/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java +++ b/aws/extensions/jets3t/src/main/java/org/jclouds/aws/s3/jets3t/JCloudsS3Service.java @@ -68,7 +68,7 @@ public class JCloudsS3Service extends S3Service { * @param awsCredentials * - credentials to access S3 * @param modules - * - Module that configures a ListenableFutureHttpClient, if not specified, default is + * - Module that configures a FutureHttpClient, if not specified, default is * URLFetchServiceClientModule * @throws S3ServiceException */ diff --git a/azure/src/main/java/org/jclouds/azure/storage/blob/AzureBlobClient.java b/azure/src/main/java/org/jclouds/azure/storage/blob/AzureBlobClient.java index 2285110514..5ef04c38a5 100644 --- a/azure/src/main/java/org/jclouds/azure/storage/blob/AzureBlobClient.java +++ b/azure/src/main/java/org/jclouds/azure/storage/blob/AzureBlobClient.java @@ -33,14 +33,14 @@ import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.concurrent.Timeout; import org.jclouds.http.options.GetOptions; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; /** * Provides access to Azure Blob via their REST API. *

- * All commands return a ListenableFuture of the result from Azure Blob. Any exceptions incurred + * All commands return a Future of the result from Azure Blob. Any exceptions incurred * during processing will be wrapped in an {@link ExecutionException} as documented in - * {@link ListenableFuture#get()}. + * {@link Future#get()}. * * @see * @author Adrian Cole diff --git a/azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/AzureAsyncBlobStore.java b/azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/AzureAsyncBlobStore.java index b9508b2f3f..4dd2b83c25 100644 --- a/azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/AzureAsyncBlobStore.java +++ b/azure/src/main/java/org/jclouds/azure/storage/blob/blobstore/AzureAsyncBlobStore.java @@ -19,11 +19,12 @@ package org.jclouds.azure.storage.blob.blobstore; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.util.concurrent.Futures.compose; +import static org.jclouds.concurrent.ConcurrentUtils.compose; import static org.jclouds.azure.storage.options.ListOptions.Builder.includeMetadata; import java.util.Set; import java.util.concurrent.ExecutorService; +import com.google.common.util.concurrent.ListenableFuture; import javax.inject.Inject; import javax.inject.Named; @@ -58,7 +59,6 @@ import org.jclouds.http.options.GetOptions; import com.google.common.base.Function; import com.google.common.collect.Iterables; -import com.google.common.util.concurrent.ListenableFuture; /** * @author Adrian Cole @@ -76,21 +76,19 @@ public class AzureAsyncBlobStore extends BaseAsyncBlobStore { @Inject AzureAsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, - Location defaultLocation, Set locations, - - AzureBlobAsyncClient async, ContainerToResourceMetadata container2ResourceMd, + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Location defaultLocation, + Set locations, AzureBlobAsyncClient async, + ContainerToResourceMetadata container2ResourceMd, ListOptionsToListBlobsOptions blobStore2AzureContainerListOptions, - ListBlobsResponseToResourceList azure2BlobStoreResourceList, - AzureBlobToBlob azureBlob2Blob, BlobToAzureBlob blob2AzureBlob, - BlobPropertiesToBlobMetadata blob2BlobMd, BlobToHttpGetOptions blob2ObjectGetOptions) { + ListBlobsResponseToResourceList azure2BlobStoreResourceList, AzureBlobToBlob azureBlob2Blob, + BlobToAzureBlob blob2AzureBlob, BlobPropertiesToBlobMetadata blob2BlobMd, + BlobToHttpGetOptions blob2ObjectGetOptions) { super(context, blobUtils, service, defaultLocation, locations); this.async = checkNotNull(async, "async"); this.container2ResourceMd = checkNotNull(container2ResourceMd, "container2ResourceMd"); this.blobStore2AzureContainerListOptions = checkNotNull(blobStore2AzureContainerListOptions, "blobStore2AzureContainerListOptions"); - this.azure2BlobStoreResourceList = checkNotNull(azure2BlobStoreResourceList, - "azure2BlobStoreResourceList"); + this.azure2BlobStoreResourceList = checkNotNull(azure2BlobStoreResourceList, "azure2BlobStoreResourceList"); this.azureBlob2Blob = checkNotNull(azureBlob2Blob, "azureBlob2Blob"); this.blob2AzureBlob = checkNotNull(blob2AzureBlob, "blob2AzureBlob"); this.blob2BlobMd = checkNotNull(blob2BlobMd, "blob2BlobMd"); @@ -102,14 +100,14 @@ public class AzureAsyncBlobStore extends BaseAsyncBlobStore { * {@link org.jclouds.azure.storage.options.ListOptions#includeMetadata} option. */ @Override - public ListenableFuture> list() { + public ListenableFuture> list() { return compose( async.listContainers(includeMetadata()), new Function, org.jclouds.blobstore.domain.PageSet>() { public org.jclouds.blobstore.domain.PageSet apply( BoundedSet from) { - return new PageSetImpl(Iterables.transform(from, - container2ResourceMd), from.getNextMarker()); + return new PageSetImpl(Iterables.transform(from, container2ResourceMd), from + .getNextMarker()); } }, service); } @@ -145,11 +143,9 @@ public class AzureAsyncBlobStore extends BaseAsyncBlobStore { * container name */ @Override - public ListenableFuture> list(String container, - ListContainerOptions options) { + public ListenableFuture> list(String container, ListContainerOptions options) { ListBlobsOptions azureOptions = blobStore2AzureContainerListOptions.apply(options); - ListenableFuture returnVal = async.listBlobs(container, azureOptions - .includeMetadata()); + ListenableFuture returnVal = async.listBlobs(container, azureOptions.includeMetadata()); return compose(returnVal, azure2BlobStoreResourceList, service); } @@ -173,8 +169,7 @@ public class AzureAsyncBlobStore extends BaseAsyncBlobStore { * blob key */ @Override - public ListenableFuture getBlob(String container, String key, - org.jclouds.blobstore.options.GetOptions options) { + public ListenableFuture getBlob(String container, String key, org.jclouds.blobstore.options.GetOptions options) { GetOptions azureOptions = blob2ObjectGetOptions.apply(options); ListenableFuture returnVal = async.getBlob(container, key, azureOptions); return compose(returnVal, azureBlob2Blob, service); @@ -229,15 +224,14 @@ public class AzureAsyncBlobStore extends BaseAsyncBlobStore { */ @Override public ListenableFuture blobMetadata(String container, String key) { - return compose(async.getBlobProperties(container, key), - new Function() { + return compose(async.getBlobProperties(container, key), new Function() { - @Override - public BlobMetadata apply(BlobProperties from) { - return blob2BlobMd.apply(from); - } + @Override + public BlobMetadata apply(BlobProperties from) { + return blob2BlobMd.apply(from); + } - }, service); + }, service); } @Override diff --git a/azure/src/main/java/org/jclouds/azure/storage/queue/AzureQueueClient.java b/azure/src/main/java/org/jclouds/azure/storage/queue/AzureQueueClient.java index 43c6cd12af..dbf29ac410 100644 --- a/azure/src/main/java/org/jclouds/azure/storage/queue/AzureQueueClient.java +++ b/azure/src/main/java/org/jclouds/azure/storage/queue/AzureQueueClient.java @@ -32,7 +32,7 @@ import org.jclouds.azure.storage.queue.options.PutMessageOptions; import org.jclouds.concurrent.Timeout; import org.jclouds.http.HttpResponseException; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; /** * Provides access to Azure Queue via their REST API. @@ -47,9 +47,9 @@ import com.google.common.util.concurrent.ListenableFuture; * If you need to store messages larger than 8 KB, you can store message data as a blob or in a * table, and then store a reference to the data as a message in a queue. *

- * All commands return a ListenableFuture of the result from Azure Queue. Any exceptions incurred + * All commands return a Future of the result from Azure Queue. Any exceptions incurred * during processing will be wrapped in an {@link ExecutionException} as documented in - * {@link ListenableFuture#get()}. + * {@link Future#get()}. * * @see AzureQueueAsyncClient * @see diff --git a/blobstore/src/main/java/org/jclouds/blobstore/AsyncBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/AsyncBlobStore.java index 7f0dc56b06..307b7c7e3d 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/AsyncBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/AsyncBlobStore.java @@ -19,6 +19,7 @@ package org.jclouds.blobstore; import java.util.Set; +import com.google.common.util.concurrent.ListenableFuture; import javax.annotation.Nullable; @@ -30,8 +31,6 @@ import org.jclouds.blobstore.options.GetOptions; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.domain.Location; -import com.google.common.util.concurrent.ListenableFuture; - /** * Provides hooks needed to run a blob store asynchronously * @@ -51,12 +50,12 @@ public interface AsyncBlobStore { /** * @see BlobStore#listAssignableLocations */ - ListenableFuture> listAssignableLocations(); + ListenableFuture> listAssignableLocations(); /** * @see BlobStore#list */ - ListenableFuture> list(); + ListenableFuture> list(); /** * @see BlobStore#containerExists @@ -71,13 +70,12 @@ public interface AsyncBlobStore { /** * @see BlobStore#list(String) */ - ListenableFuture> list(String container); + ListenableFuture> list(String container); /** * @see BlobStore#list(String, ListContainerOptions) */ - ListenableFuture> list(String container, - ListContainerOptions options); + ListenableFuture> list(String container, ListContainerOptions options); /** * @see BlobStore#clearContainer(String) @@ -127,12 +125,12 @@ public interface AsyncBlobStore { /** * @see BlobStore#getBlob(String, String) */ - ListenableFuture getBlob(String container, String key); + ListenableFuture getBlob(String container, String key); /** * @see BlobStore#getBlob(String, String, GetOptions) */ - ListenableFuture getBlob(String container, String key, GetOptions options); + ListenableFuture getBlob(String container, String key, GetOptions options); /** * @see BlobStore#removeBlob diff --git a/blobstore/src/main/java/org/jclouds/blobstore/TransientAsyncBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/TransientAsyncBlobStore.java index 3d11e0d3dc..3fc30ba267 100755 --- a/blobstore/src/main/java/org/jclouds/blobstore/TransientAsyncBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/TransientAsyncBlobStore.java @@ -93,6 +93,7 @@ import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.Multimaps; import com.google.common.io.Closeables; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.internal.Nullable; @@ -114,13 +115,12 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { @Inject protected TransientAsyncBlobStore(BlobStoreContext context, DateService dateService, - EncryptionService encryptionService, - ConcurrentMap> containerToBlobs, + EncryptionService encryptionService, ConcurrentMap> containerToBlobs, ConcurrentMap containerToLocation, HttpGetOptionsListToGetOptions httpGetOptionsConverter, - IfDirectoryReturnNameStrategy ifDirectoryReturnName, Blob.Factory blobFactory, - BlobUtils blobUtils, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, - Location defaultLocation, Set locations) { + IfDirectoryReturnNameStrategy ifDirectoryReturnName, Blob.Factory blobFactory, BlobUtils blobUtils, + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Location defaultLocation, + Set locations) { super(context, blobUtils, service, defaultLocation, locations); this.blobFactory = blobFactory; this.dateService = dateService; @@ -137,8 +137,7 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { * default maxResults is 1000 */ @Override - public ListenableFuture> list( - final String container, ListContainerOptions options) { + public ListenableFuture> list(final String container, ListContainerOptions options) { final Map realContents = getContainerToBlobs().get(container); if (realContents == null) @@ -148,10 +147,9 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { new Function() { public StorageMetadata apply(String key) { Blob oldBlob = realContents.get(key); - checkState(oldBlob != null, "blob " + key - + " is not present although it was in the list of " + container); - checkState(oldBlob.getMetadata() != null, "blob " + container + "/" + key - + " has no metadata"); + checkState(oldBlob != null, "blob " + key + " is not present although it was in the list of " + + container); + checkState(oldBlob.getMetadata() != null, "blob " + container + "/" + key + " has no metadata"); MutableBlobMetadata md = copy(oldBlob.getMetadata()); String directoryName = ifDirectoryReturnName.execute(md); if (directoryName != null) { @@ -198,13 +196,11 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { final String delimiter = options.isRecursive() ? null : "/"; if (delimiter != null) { SortedSet commonPrefixes = null; - Iterable iterable = transform(contents, new CommonPrefixes(prefix != null ? prefix - : null, delimiter)); + Iterable iterable = transform(contents, new CommonPrefixes(prefix != null ? prefix : null, delimiter)); commonPrefixes = iterable != null ? newTreeSet(iterable) : new TreeSet(); commonPrefixes.remove(CommonPrefixes.NO_PREFIX); - contents = newTreeSet(filter(contents, new DelimiterFilter(prefix != null ? prefix : null, - delimiter))); + contents = newTreeSet(filter(contents, new DelimiterFilter(prefix != null ? prefix : null, delimiter))); Iterables. addAll(contents, transform(commonPrefixes, new Function() { @@ -224,13 +220,14 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { } } - return immediateFuture(new PageSetImpl(contents, marker)); + return Futures.> immediateFuture(new PageSetImpl(contents, + marker)); } private ContainerNotFoundException cnfe(final String name) { - return new ContainerNotFoundException(name, String.format("container %s not in %s", name, - getContainerToBlobs().keySet())); + return new ContainerNotFoundException(name, String.format("container %s not in %s", name, getContainerToBlobs() + .keySet())); } public static MutableBlobMetadata copy(MutableBlobMetadata in) { @@ -318,17 +315,17 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { * {@inheritDoc} */ @Override - public ListenableFuture> list() { - return immediateFuture(new PageSetImpl(transform(getContainerToBlobs() - .keySet(), new Function() { - public StorageMetadata apply(String name) { - MutableStorageMetadata cmd = create(); - cmd.setName(name); - cmd.setType(StorageType.CONTAINER); - cmd.setLocation(getContainerToLocation().get(name)); - return cmd; - } - }), null)); + public ListenableFuture> list() { + return Futures.> immediateFuture(new PageSetImpl(transform( + getContainerToBlobs().keySet(), new Function() { + public StorageMetadata apply(String name) { + MutableStorageMetadata cmd = create(); + cmd.setName(name); + cmd.setType(StorageType.CONTAINER); + cmd.setLocation(getContainerToLocation().get(name)); + return cmd; + } + }), null)); } protected MutableStorageMetadata create() { @@ -339,8 +336,7 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { * {@inheritDoc} */ @Override - public ListenableFuture createContainerInLocation(final Location location, - final String name) { + public ListenableFuture createContainerInLocation(final Location location, final String name) { if (!getContainerToBlobs().containsKey(name)) { getContainerToBlobs().put(name, new ConcurrentHashMap()); getContainerToLocation().put(name, location != null ? location : defaultLocation); @@ -407,8 +403,7 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { } } - public static > SortedSet firstSliceOfSize(Iterable elements, - int size) { + public static > SortedSet firstSliceOfSize(Iterable elements, int size) { List> slices = partition(newArrayList(elements), size); return newTreeSet(slices.get(0)); } @@ -463,15 +458,12 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { new RuntimeException("containerName not found: " + containerName); } - ByteArrayPayload payload = (object.getPayload() instanceof ByteArrayPayload) ? ByteArrayPayload.class - .cast(object.getPayload()) - : null; + ByteArrayPayload payload = (object.getPayload() instanceof ByteArrayPayload) ? ByteArrayPayload.class.cast(object + .getPayload()) : null; if (payload == null) - payload = (object.getPayload() instanceof DelegatingPayload) ? (DelegatingPayload.class - .cast(object.getPayload()).getDelegate() instanceof ByteArrayPayload) ? ByteArrayPayload.class - .cast(DelegatingPayload.class.cast(object.getPayload()).getDelegate()) - : null - : null; + payload = (object.getPayload() instanceof DelegatingPayload) ? (DelegatingPayload.class.cast( + object.getPayload()).getDelegate() instanceof ByteArrayPayload) ? ByteArrayPayload.class + .cast(DelegatingPayload.class.cast(object.getPayload()).getDelegate()) : null : null; if (payload == null || !(payload instanceof ByteArrayPayload)) { InputStream input = object.getPayload().getInput(); try { @@ -483,8 +475,7 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { } } else { if (payload.getContentMD5() == null) - payload = (ByteArrayPayload) encryptionService - .generateMD5BufferingIfNotRepeatable(payload); + payload = (ByteArrayPayload) encryptionService.generateMD5BufferingIfNotRepeatable(payload); } Blob blob = blobFactory.create(copy(object.getMetadata())); @@ -525,8 +516,7 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { * {@inheritDoc} */ @Override - public ListenableFuture getBlob(final String containerName, final String key, - GetOptions options) { + public ListenableFuture getBlob(final String containerName, final String key, GetOptions options) { if (!getContainerToBlobs().containsKey(containerName)) return immediateFailedFuture(cnfe(containerName)); Map realContents = getContainerToBlobs().get(containerName); @@ -547,9 +537,8 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { Date modifiedSince = options.getIfModifiedSince(); if (object.getMetadata().getLastModified().before(modifiedSince)) { HttpResponse response = new HttpResponse(304, null, null); - return immediateFailedFuture(new HttpResponseException(String.format( - "%1$s is before %2$s", object.getMetadata().getLastModified(), modifiedSince), - null, response)); + return immediateFailedFuture(new HttpResponseException(String.format("%1$s is before %2$s", object + .getMetadata().getLastModified(), modifiedSince), null, response)); } } @@ -557,9 +546,8 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { Date unmodifiedSince = options.getIfUnmodifiedSince(); if (object.getMetadata().getLastModified().after(unmodifiedSince)) { HttpResponse response = new HttpResponse(412, null, null); - return immediateFailedFuture(new HttpResponseException( - String.format("%1$s is after %2$s", object.getMetadata().getLastModified(), - unmodifiedSince), null, response)); + return immediateFailedFuture(new HttpResponseException(String.format("%1$s is after %2$s", object + .getMetadata().getLastModified(), unmodifiedSince), null, response)); } } Blob returnVal = copyBlob(object); @@ -586,8 +574,7 @@ public class TransientAsyncBlobStore extends BaseAsyncBlobStore { int length = (last < data.length) ? last + 1 : data.length - offset; out.write(data, offset, length); } else { - return immediateFailedFuture(new IllegalArgumentException( - "first and last were null!")); + return immediateFailedFuture(new IllegalArgumentException("first and last were null!")); } } diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseAsyncBlobStore.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseAsyncBlobStore.java index 970d896632..9ac37548c6 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseAsyncBlobStore.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseAsyncBlobStore.java @@ -19,10 +19,7 @@ package org.jclouds.blobstore.internal; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.util.concurrent.Futures.chain; -import static com.google.common.util.concurrent.Futures.immediateFuture; import static org.jclouds.blobstore.options.ListContainerOptions.Builder.recursive; -import static org.jclouds.concurrent.ConcurrentUtils.makeListenable; import java.util.Set; import java.util.concurrent.Callable; @@ -41,11 +38,12 @@ import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.util.BlobUtils; import org.jclouds.blobstore.util.internal.BlobUtilsImpl; +import static org.jclouds.concurrent.ConcurrentUtils.*; import org.jclouds.domain.Location; import org.jclouds.util.Utils; -import com.google.common.base.Function; import com.google.common.base.Supplier; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; /** @@ -62,8 +60,8 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore { @Inject protected BaseAsyncBlobStore(BlobStoreContext context, BlobUtils blobUtils, - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, - Location defaultLocation, Set locations) { + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, Location defaultLocation, + Set locations) { this.context = checkNotNull(context, "context"); this.blobUtils = checkNotNull(blobUtils, "blobUtils"); this.service = checkNotNull(service, "service"); @@ -92,7 +90,7 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore { * container name */ @Override - public ListenableFuture> list(String container) { + public ListenableFuture> list(String container) { return this.list(container, org.jclouds.blobstore.options.ListContainerOptions.NONE); } @@ -115,8 +113,7 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore { * container name */ @Override - public ListenableFuture countBlobs(final String containerName, - final ListContainerOptions options) { + public ListenableFuture countBlobs(final String containerName, final ListContainerOptions options) { return makeListenable(service.submit(new Callable() { public Long call() throws Exception { return blobUtils.countBlobs(containerName, options); @@ -144,8 +141,7 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore { * container name */ @Override - public ListenableFuture clearContainer(final String containerName, - final ListContainerOptions options) { + public ListenableFuture clearContainer(final String containerName, final ListContainerOptions options) { return makeListenable(service.submit(new Callable() { public Void call() throws Exception { @@ -182,8 +178,7 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore { * @param directory * virtual path */ - public ListenableFuture directoryExists(final String containerName, - final String directory) { + public ListenableFuture directoryExists(final String containerName, final String directory) { return makeListenable(service.submit(new Callable() { public Boolean call() throws Exception { @@ -204,18 +199,13 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore { public ListenableFuture createDirectory(final String containerName, final String directory) { - return chain(directoryExists(containerName, directory), - new Function>() { - - @Override - public ListenableFuture apply(Boolean from) { - if (!from) { - blobUtils.createDirectory(containerName, directory); - } - return immediateFuture(null); + return blobUtils.directoryExists(containerName, directory) ? Futures.immediateFuture((Void) null) + : makeListenable(service.submit(new Callable() { + public Void call() throws Exception { + blobUtils.createDirectory(containerName, directory); + return null; } - - }, service); + }), service); } /** @@ -228,7 +218,7 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore { * blob key */ @Override - public ListenableFuture getBlob(String container, String key) { + public ListenableFuture getBlob(String container, String key) { return getBlob(container, key, org.jclouds.blobstore.options.GetOptions.NONE); } @@ -271,8 +261,8 @@ public abstract class BaseAsyncBlobStore implements AsyncBlobStore { } @Override - public ListenableFuture> listAssignableLocations() { - return immediateFuture(locations); + public ListenableFuture> listAssignableLocations() { + return Futures.> immediateFuture(locations); } protected abstract boolean deleteAndVerifyContainerGone(String container); diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobMap.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobMap.java index 460e52302a..5a7bec88f1 100755 --- a/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobMap.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobMap.java @@ -160,8 +160,8 @@ public abstract class BaseBlobMap implements ListableMap { return (int) blobstore.countBlobs(containerName, options); } - protected Set getAllBlobs() { - Set returnVal = getAllBlobs.execute(containerName, options); + protected Iterable getAllBlobs() { + Iterable returnVal = getAllBlobs.execute(containerName, options); if (options != null) { for (Blob from : returnVal) stripPrefix(from); diff --git a/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobMapImpl.java b/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobMapImpl.java index dd2bd03bba..ecb7c84d73 100755 --- a/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobMapImpl.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/internal/BlobMapImpl.java @@ -33,6 +33,8 @@ import org.jclouds.blobstore.strategy.GetBlobsInListStrategy; import org.jclouds.blobstore.strategy.PutBlobsStrategy; import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders; +import com.google.common.collect.Sets; + /** * Map representation of a live connection to a Blob Service. * @@ -46,10 +48,8 @@ public class BlobMapImpl extends BaseBlobMap implements BlobMap { @Inject public BlobMapImpl(BlobStore blobstore, GetBlobsInListStrategy getAllBlobs, ContainsValueInListStrategy containsValueStrategy, PutBlobsStrategy putBlobsStrategy, - ListContainerAndRecurseThroughFolders listStrategy, String containerName, - ListContainerOptions options) { - super(blobstore, getAllBlobs, containsValueStrategy, putBlobsStrategy, listStrategy, - containerName, options); + ListContainerAndRecurseThroughFolders listStrategy, String containerName, ListContainerOptions options) { + super(blobstore, getAllBlobs, containsValueStrategy, putBlobsStrategy, listStrategy, containerName, options); } @Override @@ -89,10 +89,9 @@ public class BlobMapImpl extends BaseBlobMap implements BlobMap { return old; } - @SuppressWarnings("unchecked") @Override public Collection values() { - return (Collection) getAllBlobs.execute(containerName, options); + return Sets.newLinkedHashSet(getAllBlobs.execute(containerName, options)); } @Override diff --git a/blobstore/src/main/java/org/jclouds/blobstore/options/GetOptions.java b/blobstore/src/main/java/org/jclouds/blobstore/options/GetOptions.java index 046ee633e5..2a62968702 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/options/GetOptions.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/options/GetOptions.java @@ -37,7 +37,7 @@ import java.util.List; * * * // this will get the first megabyte of an object, provided it wasn't modified since yesterday - * ListenableFuture object = client.get("objectName",range(0,1024).ifUnmodifiedSince(new Date().minusDays(1))); + * Future object = client.get("objectName",range(0,1024).ifUnmodifiedSince(new Date().minusDays(1))); * * * @author Adrian Cole diff --git a/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java b/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java index 71c1587618..a4a70bed16 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/options/ListContainerOptions.java @@ -31,7 +31,7 @@ import static com.google.common.base.Preconditions.checkNotNull; * import static org.jclouds.blobstore.options.ListContainerOptions.Builder.* *

* BlobStore connection = // get connection - * ListenableFuture> list = connection.list("container",inDirectory("home/users").maxResults(1000)); + * Future> list = connection.list("container",inDirectory("home/users").maxResults(1000)); * * * @author Adrian Cole diff --git a/blobstore/src/main/java/org/jclouds/blobstore/options/ListOptions.java b/blobstore/src/main/java/org/jclouds/blobstore/options/ListOptions.java index 8730ad41e6..42742c4caa 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/options/ListOptions.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/options/ListOptions.java @@ -30,7 +30,7 @@ import static com.google.common.base.Preconditions.checkNotNull; * import static org.jclouds.blobstore.options.ListOptions.Builder.* *

* BlobStore connection = // get connection - * ListenableFuture> list = connection.list(maxResults(1000)); + * Future> list = connection.list(maxResults(1000)); * * * @author Adrian Cole diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/GetBlobsInListStrategy.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/GetBlobsInListStrategy.java index 800ce728df..e89026b65e 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/GetBlobsInListStrategy.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/GetBlobsInListStrategy.java @@ -18,8 +18,6 @@ */ package org.jclouds.blobstore.strategy; -import java.util.Set; - import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.strategy.internal.GetAllBlobsInListAndRetryOnFailure; @@ -34,6 +32,6 @@ import com.google.inject.ImplementedBy; @ImplementedBy(GetAllBlobsInListAndRetryOnFailure.class) public interface GetBlobsInListStrategy { - Set execute(String containerName, ListContainerOptions options); + Iterable execute(String containerName, ListContainerOptions options); } \ No newline at end of file diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobsInContainer.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobsInContainer.java index 34fcd3093e..02e1a5b440 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobsInContainer.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/ListBlobsInContainer.java @@ -18,6 +18,8 @@ */ package org.jclouds.blobstore.strategy; +import java.util.Set; + import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.strategy.internal.ListContainerAndRecurseThroughFolders; @@ -32,6 +34,6 @@ import com.google.inject.ImplementedBy; @ImplementedBy(ListContainerAndRecurseThroughFolders.class) public interface ListBlobsInContainer { - Iterable execute(String containerName, ListContainerOptions options); + Set execute(String containerName, ListContainerOptions options); } \ No newline at end of file diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java index 962fd1b220..402c208247 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/DeleteAllKeysInList.java @@ -43,7 +43,7 @@ import org.jclouds.logging.Logger; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; import com.google.inject.Inject; /** @@ -93,7 +93,7 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr Map exceptions = Maps.newHashMap(); Iterable toDelete = getResourcesToDelete(containerName, options); for (int i = 0; i < 3; i++) { // TODO parameterize - Map> responses = Maps.newHashMap(); + Map> responses = Maps.newHashMap(); try { for (final StorageMetadata md : toDelete) { String fullPath = parentIsFolder(options, md) ? options.getDir() + "/" diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/FetchBlobMetadata.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/FetchBlobMetadata.java index 099cea2e3e..68782f6407 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/FetchBlobMetadata.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/FetchBlobMetadata.java @@ -19,13 +19,10 @@ package org.jclouds.blobstore.strategy.internal; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; -import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion; +import static org.jclouds.concurrent.ConcurrentUtils.transformParallel; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import javax.annotation.Resource; import javax.annotation.concurrent.NotThreadSafe; @@ -33,20 +30,18 @@ import javax.inject.Named; import org.jclouds.Constants; import org.jclouds.blobstore.AsyncBlobStore; +import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.domain.StorageType; import org.jclouds.blobstore.domain.internal.PageSetImpl; -import org.jclouds.blobstore.internal.BlobRuntimeException; import org.jclouds.blobstore.reference.BlobStoreConstants; import org.jclouds.http.handlers.BackoffLimitedRetryHandler; import org.jclouds.logging.Logger; import com.google.common.base.Function; -import com.google.common.base.Throwables; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; import com.google.inject.Inject; /** @@ -55,8 +50,7 @@ import com.google.inject.Inject; * @author Adrian Cole */ @NotThreadSafe -public class FetchBlobMetadata implements - Function, PageSet> { +public class FetchBlobMetadata implements Function, PageSet> { protected final BackoffLimitedRetryHandler retryHandler; protected final AsyncBlobStore ablobstore; @@ -74,8 +68,8 @@ public class FetchBlobMetadata implements protected Long maxTime; @Inject - FetchBlobMetadata(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, - AsyncBlobStore ablobstore, BackoffLimitedRetryHandler retryHandler) { + FetchBlobMetadata(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, AsyncBlobStore ablobstore, + BackoffLimitedRetryHandler retryHandler) { this.userExecutor = userExecutor; this.ablobstore = ablobstore; this.retryHandler = retryHandler; @@ -88,35 +82,23 @@ public class FetchBlobMetadata implements public PageSet apply(PageSet in) { checkState(container != null, "container name should be initialized"); - Map exceptions = Maps.newHashMap(); - final Set metadata = Sets.newHashSet(); - Map> responses = Maps.newHashMap(); - for (StorageMetadata md : in) { - if (md.getType() == StorageType.BLOB) { - final ListenableFuture future = ablobstore.blobMetadata( - container, md.getName()); - future.addListener(new Runnable() { - @Override - public void run() { - try { - metadata.add(future.get()); - } catch (InterruptedException e) { - Throwables.propagate(e); - } catch (ExecutionException e) { - Throwables.propagate(e); - } - } - }, sameThreadExecutor()); - responses.put(md, future); - } else { - metadata.add(md); + + Iterable returnv = transformParallel(Iterables.filter(in, new Predicate() { + + @Override + public boolean apply(StorageMetadata input) { + return input.getType() == StorageType.BLOB; } - } - exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, String.format( - "getting metadata from containerName: %s", container)); - if (exceptions.size() > 0) - throw new BlobRuntimeException(String.format("errors getting from container %s: %s", - container, exceptions)); - return new PageSetImpl(metadata, in.getNextMarker()); + + }), new Function>() { + + @Override + public Future apply(StorageMetadata from) { + return ablobstore.blobMetadata(container, from.getName()); + } + + }, userExecutor, maxTime, logger, String.format("getting metadata from containerName: %s", container)); + + return new PageSetImpl(returnv, in.getNextMarker()); } } \ No newline at end of file diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java index ff3ca2579a..2b2eca49a8 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/GetAllBlobsInListAndRetryOnFailure.java @@ -18,13 +18,10 @@ */ package org.jclouds.blobstore.strategy.internal; -import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; -import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion; +import static org.jclouds.concurrent.ConcurrentUtils.transformParallel; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import javax.annotation.Resource; import javax.inject.Named; @@ -34,7 +31,6 @@ import org.jclouds.Constants; import org.jclouds.blobstore.AsyncBlobStore; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; -import org.jclouds.blobstore.internal.BlobRuntimeException; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.reference.BlobStoreConstants; import org.jclouds.blobstore.strategy.GetBlobsInListStrategy; @@ -42,10 +38,7 @@ import org.jclouds.blobstore.strategy.ListBlobsInContainer; import org.jclouds.http.handlers.BackoffLimitedRetryHandler; import org.jclouds.logging.Logger; -import com.google.common.base.Throwables; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.base.Function; import com.google.inject.Inject; /** @@ -72,52 +65,24 @@ public class GetAllBlobsInListAndRetryOnFailure implements GetBlobsInListStrateg protected Long maxTime; @Inject - GetAllBlobsInListAndRetryOnFailure( - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, - ListBlobsInContainer getAllBlobMetadata, AsyncBlobStore ablobstore, - BackoffLimitedRetryHandler retryHandler) { + GetAllBlobsInListAndRetryOnFailure(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, + ListBlobsInContainer getAllBlobMetadata, AsyncBlobStore ablobstore, BackoffLimitedRetryHandler retryHandler) { this.userExecutor = userExecutor; this.ablobstore = ablobstore; this.getAllBlobMetadata = getAllBlobMetadata; this.retryHandler = retryHandler; } - public Set execute(String container, ListContainerOptions options) { - Map exceptions = Maps.newHashMap(); - final Set objects = Sets.newHashSet(); - Iterable toGet = getAllBlobMetadata.execute(container, options); - for (int i = 0; i < 3; i++) { - Map> responses = Maps.newHashMap(); - for (BlobMetadata md : toGet) { - final ListenableFuture future = ablobstore.getBlob(container, md - .getName()); - future.addListener(new Runnable() { - @Override - public void run() { - try { - objects.add(future.get()); - } catch (InterruptedException e) { - Throwables.propagate(e); - } catch (ExecutionException e) { - Throwables.propagate(e); - } - } - }, sameThreadExecutor()); - responses.put(md, future); + public Iterable execute(final String container, ListContainerOptions options) { + Iterable list = getAllBlobMetadata.execute(container, options); + return transformParallel(list, new Function>() { + + @Override + public Future apply(BlobMetadata from) { + return ablobstore.getBlob(container, from.getName()); } - exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, String.format( - "getting from containerName: %s", container)); - if (exceptions.size() > 0) { - toGet = exceptions.keySet(); - retryHandler.imposeBackoffExponentialDelay(i + 1, String.format("blob %s/%s not found", - container, toGet)); - } else { - break; - } - } - if (exceptions.size() > 0) - throw new BlobRuntimeException(String.format("errors getting from container %s: %s", - container, exceptions)); - return objects; + + }, userExecutor, maxTime, logger, String.format("getting from containerName: %s", container), retryHandler, 3); + } } \ No newline at end of file diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListContainerAndRecurseThroughFolders.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListContainerAndRecurseThroughFolders.java index 6d3eef610f..27c4fff2d2 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListContainerAndRecurseThroughFolders.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/ListContainerAndRecurseThroughFolders.java @@ -18,7 +18,14 @@ */ package org.jclouds.blobstore.strategy.internal; +import static com.google.common.collect.Iterables.concat; +import static com.google.common.collect.Iterables.filter; +import static com.google.common.collect.Iterables.transform; +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Sets.newLinkedHashSet; + import java.util.List; +import java.util.Set; import javax.inject.Singleton; @@ -31,9 +38,6 @@ import org.jclouds.blobstore.strategy.ListContainerStrategy; import com.google.common.base.Function; import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import com.google.inject.Inject; /** @@ -52,22 +56,20 @@ public class ListContainerAndRecurseThroughFolders implements ListBlobsInContain } @Override - public Iterable execute(final String containerName, - final ListContainerOptions options) { - final List> lists = Lists.newArrayList(); + public Set execute(final String containerName, final ListContainerOptions options) { + final List> lists = newArrayList(); Iterable pwdList = lister.execute(containerName, options); - for (StorageMetadata md : Iterables.filter(pwdList, new Predicate() { + for (StorageMetadata md : filter(pwdList, new Predicate() { @Override public boolean apply(StorageMetadata input) { return (input.getType() == StorageType.FOLDER || input.getType() == StorageType.RELATIVE_PATH) && options.isRecursive(); } })) { - String directory = (options.getDir() != null) ? options.getDir() + "/" + md.getName() : md - .getName(); + String directory = (options.getDir() != null) ? options.getDir() + "/" + md.getName() : md.getName(); lists.add(execute(containerName, options.clone().inDirectory(directory))); } - lists.add(Iterables.transform(Iterables.filter(pwdList, new Predicate() { + lists.add(transform(filter(pwdList, new Predicate() { @Override public boolean apply(StorageMetadata input) { return input.getType() == StorageType.BLOB; @@ -78,6 +80,6 @@ public class ListContainerAndRecurseThroughFolders implements ListBlobsInContain return (BlobMetadata) from; } })); - return Sets.newHashSet(Iterables.concat(lists)); + return newLinkedHashSet(concat(lists)); } } \ No newline at end of file diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MarkersDeleteDirectoryStrategy.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MarkersDeleteDirectoryStrategy.java index 1cdc540077..5687b4be43 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MarkersDeleteDirectoryStrategy.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MarkersDeleteDirectoryStrategy.java @@ -38,7 +38,7 @@ import org.jclouds.logging.Logger; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; import com.google.inject.Inject; /** @@ -92,7 +92,7 @@ public class MarkersDeleteDirectoryStrategy implements DeleteDirectoryStrategy { for (String suffix : BlobStoreConstants.DIRECTORY_SUFFIXES) { names.add(directory + suffix); } - Map> responses = Maps.newHashMap(); + Map> responses = Maps.newHashMap(); for (String name : names) { responses.put(name, ablobstore.removeBlob(containerName, name)); } diff --git a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/PutBlobsStrategyImpl.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/PutBlobsStrategyImpl.java index 2259a8b45a..b908825eb0 100644 --- a/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/PutBlobsStrategyImpl.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/PutBlobsStrategyImpl.java @@ -36,7 +36,7 @@ import org.jclouds.blobstore.strategy.PutBlobsStrategy; import org.jclouds.logging.Logger; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; import com.google.inject.Inject; /** @@ -67,7 +67,7 @@ public class PutBlobsStrategyImpl implements PutBlobsStrategy { @Override public void execute(String containerName, Iterable blobs) { - Map> responses = Maps.newHashMap(); + Map> responses = Maps.newHashMap(); for (Blob blob : blobs) { responses.put(blob, ablobstore.putBlob(containerName, blob)); } diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java index 8be2accdcc..169f092b83 100755 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java @@ -38,6 +38,7 @@ import java.io.UnsupportedEncodingException; import java.security.NoSuchAlgorithmException; import java.util.Date; import java.util.Map; +import java.util.concurrent.Future; import java.util.zip.GZIPInputStream; import javax.ws.rs.core.MediaType; @@ -48,6 +49,7 @@ import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; import org.jclouds.blobstore.domain.StorageType; +import static org.jclouds.concurrent.ConcurrentUtils.*; import org.jclouds.encryption.EncryptionService; import org.jclouds.encryption.internal.JCEEncryptionService; import org.jclouds.http.BaseJettyTest; @@ -68,8 +70,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.io.ByteStreams; import com.google.common.io.InputSupplier; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; /** * @author Adrian Cole @@ -90,7 +90,7 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { @SuppressWarnings("unchecked") public static InputSupplier getTestDataSupplier() throws IOException { byte[] oneConstitution = ByteStreams.toByteArray(new GZIPInputStream(BaseJettyTest.class - .getResourceAsStream("/const.txt.gz"))); + .getResourceAsStream("/const.txt.gz"))); InputSupplier constitutionSupplier = ByteStreams.newInputStreamSupplier(oneConstitution); InputSupplier temp = ByteStreams.join(constitutionSupplier); @@ -108,23 +108,23 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { String key = "constitution.txt"; uploadConstitution(containerName, key); - Map> responses = Maps.newHashMap(); + Map> responses = Maps.newHashMap(); for (int i = 0; i < 10; i++) { - responses.put(i, Futures.compose(context.getAsyncBlobStore().getBlob(containerName, key), - new Function() { + responses.put(i, compose(context.getAsyncBlobStore().getBlob(containerName, key), + new Function() { - @Override - public Void apply(Blob from) { - assertEquals(context.utils().encryption().md5(from.getPayload().getInput()), - oneHundredOneConstitutionsMD5); - return null; - } + @Override + public Void apply(Blob from) { + assertEquals(context.utils().encryption().md5(from.getPayload().getInput()), + oneHundredOneConstitutionsMD5); + return null; + } - })); + }, this.exec)); } Map exceptions = awaitCompletion(responses, exec, 30000l, Logger.CONSOLE, - "get constitution"); + "get constitution"); assert exceptions.size() == 0 : exceptions; } finally { @@ -349,8 +349,8 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { @DataProvider(name = "delete") public Object[][] createData() { - return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" }, - { "colon:" }, { "asteri*k" }, { "quote\"" }, { "{greaten" }, { "p|pe" } }; + return new Object[][] { { "normal" }, { "sp ace" }, { "qu?stion" }, { "unic₪de" }, { "path/foo" }, { "colon:" }, + { "asteri*k" }, { "quote\"" }, { "{greaten" }, { "p|pe" } }; } @Test(groups = { "integration", "live" }, dataProvider = "delete") @@ -367,17 +367,17 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { private void assertContainerEmptyDeleting(String containerName, String key) { Iterable listing = Iterables.filter(context.getBlobStore().list(containerName), - new Predicate() { + new Predicate() { - @Override - public boolean apply(StorageMetadata input) { - return input.getType() == StorageType.BLOB; - } + @Override + public boolean apply(StorageMetadata input) { + return input.getType() == StorageType.BLOB; + } - }); + }); assertEquals(Iterables.size(listing), 0, String.format( - "deleting %s, we still have %s blobs left in container %s, using encoding %s", key, - Iterables.size(listing), containerName, LOCAL_ENCODING)); + "deleting %s, we still have %s blobs left in container %s, using encoding %s", key, Iterables + .size(listing), containerName, LOCAL_ENCODING)); } @Test(groups = { "integration", "live" }) @@ -397,13 +397,13 @@ public class BaseBlobIntegrationTest extends BaseBlobStoreIntegrationTest { String realObject = Utils.toStringAndClose(new FileInputStream("pom.xml")); return new Object[][] { { "file", "text/xml", new File("pom.xml"), realObject }, - { "string", "text/xml", realObject, realObject }, - { "bytes", "application/octet-stream", realObject.getBytes(), realObject } }; + { "string", "text/xml", realObject, realObject }, + { "bytes", "application/octet-stream", realObject.getBytes(), realObject } }; } @Test(groups = { "integration", "live" }, dataProvider = "putTests") public void testPutObject(String key, String type, Object content, Object realObject) throws InterruptedException, - IOException { + IOException { Blob blob = context.getBlobStore().newBlob(key); blob.getMetadata().setContentType(type); blob.setPayload(Payloads.newPayload(content)); diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseMapIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseMapIntegrationTest.java index 6a0d5342b3..8e8cdd7fa8 100755 --- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseMapIntegrationTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseMapIntegrationTest.java @@ -56,21 +56,17 @@ import com.google.common.io.Files; public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegrationTest { - public abstract void testPutAll() throws InterruptedException, ExecutionException, - TimeoutException; + public abstract void testPutAll() throws InterruptedException, ExecutionException, TimeoutException; - public abstract void testEntrySet() throws IOException, InterruptedException, - ExecutionException, TimeoutException; + public abstract void testEntrySet() throws IOException, InterruptedException, ExecutionException, TimeoutException; - public abstract void testValues() throws IOException, InterruptedException, ExecutionException, - TimeoutException; + public abstract void testValues() throws IOException, InterruptedException, ExecutionException, TimeoutException; - protected Map fiveBytes = Maps.transformValues(fiveStrings, - new Function() { - public byte[] apply(String from) { - return from.getBytes(); - } - }); + protected Map fiveBytes = Maps.transformValues(fiveStrings, new Function() { + public byte[] apply(String from) { + return from.getBytes(); + } + }); protected Map fiveInputs; protected Map fiveFiles; String tmpDirectory; @@ -86,13 +82,13 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration @BeforeClass(groups = { "integration", "live" }) @Parameters( { "basedir" }) - protected void setUpTempDir(@Optional String basedir) throws InterruptedException, - ExecutionException, FileNotFoundException, IOException, TimeoutException { + protected void setUpTempDir(@Optional String basedir) throws InterruptedException, ExecutionException, + FileNotFoundException, IOException, TimeoutException { if (basedir == null) { basedir = System.getProperty("java.io.tmpdir"); } - tmpDirectory = basedir + File.separator + "target" + File.separator + "testFiles" - + File.separator + getClass().getSimpleName(); + tmpDirectory = basedir + File.separator + "target" + File.separator + "testFiles" + File.separator + + getClass().getSimpleName(); new File(tmpDirectory).mkdirs(); fiveFiles = Maps.newHashMap(); for (Entry entry : fiveStrings.entrySet()) { @@ -123,8 +119,7 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration } @Test(groups = { "integration", "live" }) - public abstract void testRemove() throws IOException, InterruptedException, ExecutionException, - TimeoutException; + public abstract void testRemove() throws IOException, InterruptedException, ExecutionException, TimeoutException; @Test(groups = { "integration", "live" }) public void testKeySet() throws InterruptedException, ExecutionException, TimeoutException { @@ -140,8 +135,7 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration } } - protected void addTenObjectsUnderPrefix(String containerName, String prefix) - throws InterruptedException { + protected void addTenObjectsUnderPrefix(String containerName, String prefix) throws InterruptedException { for (int i = 0; i < 10; i++) { Blob blob = context.getBlobStore().newBlob(prefix + "/" + i); blob.setPayload(i + "content"); @@ -160,14 +154,12 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration @Test(groups = { "integration", "live" }) public void testDirectory() throws InterruptedException, UnsupportedEncodingException { String containerName = getContainerName(); + String directory = "apps"; + Map rootMap = createMap(context, containerName); + Map rootRecursiveMap = createMap(context, containerName, recursive()); + Map inDirectoryMap = createMap(context, containerName, inDirectory(directory)); + Map inDirectoryRecursiveMap = createMap(context, containerName, inDirectory(directory).recursive()); try { - String directory = "apps"; - - Map rootMap = createMap(context, containerName); - Map rootRecursiveMap = createMap(context, containerName, recursive()); - Map inDirectoryMap = createMap(context, containerName, inDirectory(directory)); - Map inDirectoryRecursiveMap = createMap(context, containerName, inDirectory( - directory).recursive()); context.getBlobStore().createDirectory(containerName, directory); addTenObjectsUnderRoot(containerName); @@ -206,24 +198,22 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration } - protected void assertConsistencyAwareKeySetEquals(final Map map, - final Set expected) throws InterruptedException { + protected void assertConsistencyAwareKeySetEquals(final Map map, final Set expected) + throws InterruptedException { assertConsistencyAware(new Runnable() { public void run() { Set toMatch = map.keySet(); Set shouldBeEmpty = Sets.difference(expected, toMatch); - assert shouldBeEmpty.size() == 0 : "toMatch has less keys than expected. missing: " - + shouldBeEmpty; + assert shouldBeEmpty.size() == 0 : "toMatch has less keys than expected. missing: " + shouldBeEmpty; shouldBeEmpty = Sets.difference(toMatch, expected); - assert shouldBeEmpty.size() == 0 : "toMatch has more keys than expected. extras: " - + shouldBeEmpty; + assert shouldBeEmpty.size() == 0 : "toMatch has more keys than expected. extras: " + shouldBeEmpty; assertEquals(Sets.newTreeSet(toMatch), Sets.newTreeSet(expected)); } }); } - protected void assertConsistencyAwareRemoveEquals(final Map map, final String key, - final Object equals) throws InterruptedException { + protected void assertConsistencyAwareRemoveEquals(final Map map, final String key, final Object equals) + throws InterruptedException { assertConsistencyAware(new Runnable() { public void run() { assertEquals(map.remove(key), equals); @@ -231,8 +221,8 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration }); } - protected void assertConsistencyAwareGetEquals(final Map map, final String key, - final Object equals) throws InterruptedException { + protected void assertConsistencyAwareGetEquals(final Map map, final String key, final Object equals) + throws InterruptedException { assertConsistencyAware(new Runnable() { public void run() { assertEquals(map.get(key), equals); @@ -240,8 +230,7 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration }); } - protected void assertConsistencyAwareKeySize(final Map map, final int size) - throws InterruptedException { + protected void assertConsistencyAwareKeySize(final Map map, final int size) throws InterruptedException { assertConsistencyAware(new Runnable() { public void run() { assertEquals(map.keySet().size(), size); @@ -275,8 +264,7 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration }); } - protected void assertConsistencyAwareContainsKey(final Map map) - throws InterruptedException { + protected void assertConsistencyAwareContainsKey(final Map map) throws InterruptedException { assertConsistencyAware(new Runnable() { public void run() { assert map.containsKey("one"); @@ -284,8 +272,7 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration }); } - protected void assertConsistencyAwareDoesntContainKey(final Map map) - throws InterruptedException { + protected void assertConsistencyAwareDoesntContainKey(final Map map) throws InterruptedException { assertConsistencyAware(new Runnable() { public void run() { assert !map.containsKey("one"); @@ -306,8 +293,7 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration } } - protected void assertConsistencyAwareNotEmpty(final Map map) - throws InterruptedException { + protected void assertConsistencyAwareNotEmpty(final Map map) throws InterruptedException { assertConsistencyAware(new Runnable() { public void run() { assert !map.isEmpty(); @@ -323,18 +309,16 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration }); } - abstract protected void putStringWithMD5(Map map, String key, String value) - throws InterruptedException, ExecutionException, TimeoutException; + abstract protected void putStringWithMD5(Map map, String key, String value) throws InterruptedException, + ExecutionException, TimeoutException; protected void fourLeftRemovingOne(Map map) throws InterruptedException { map.remove("one"); assertConsistencyAwareMapSize(map, 4); - assertConsistencyAwareKeySetEquals(map, new TreeSet(ImmutableSet.of("two", "three", - "four", "five"))); + assertConsistencyAwareKeySetEquals(map, new TreeSet(ImmutableSet.of("two", "three", "four", "five"))); } - protected void assertConsistencyAwareMapSize(final Map map, final int size) - throws InterruptedException { + protected void assertConsistencyAwareMapSize(final Map map, final int size) throws InterruptedException { assertConsistencyAware(new Runnable() { public void run() { assertEquals(map.size(), size); @@ -343,12 +327,10 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration } @Test(groups = { "integration", "live" }) - public abstract void testPut() throws IOException, InterruptedException, ExecutionException, - TimeoutException; + public abstract void testPut() throws IOException, InterruptedException, ExecutionException, TimeoutException; @Test(groups = { "integration", "live" }) - public void testListContainer() throws InterruptedException, ExecutionException, - TimeoutException { + public void testListContainer() throws InterruptedException, ExecutionException, TimeoutException { String containerNameName = getContainerName(); try { ListableMap map = (ListableMap) createMap(context, containerNameName); @@ -358,8 +340,8 @@ public abstract class BaseMapIntegrationTest extends BaseBlobStoreIntegration } } - protected void assertConsistencyAwareListContainer(final ListableMap map, - final String containerNameName) throws InterruptedException { + protected void assertConsistencyAwareListContainer(final ListableMap map, final String containerNameName) + throws InterruptedException { assertConsistencyAware(new Runnable() { public void run() { assertTrue(Iterables.size(map.list()) >= 0); diff --git a/boxdotnet/src/main/java/org/jclouds/boxdotnet/BoxDotNetClient.java b/boxdotnet/src/main/java/org/jclouds/boxdotnet/BoxDotNetClient.java index b1b76f4aed..bf3f864d1b 100644 --- a/boxdotnet/src/main/java/org/jclouds/boxdotnet/BoxDotNetClient.java +++ b/boxdotnet/src/main/java/org/jclouds/boxdotnet/BoxDotNetClient.java @@ -40,7 +40,7 @@ public interface BoxDotNetClient { /* * Note all these delegate to methods in BoxDotNetAsyncClient with a specified or inherited timeout. * The singatures should match those of BoxDotNetAsyncClient, except the returnvals should not be - * wrapped in a ListenableFuture + * wrapped in a Future */ String list(); diff --git a/chef/core/src/main/java/org/jclouds/chef/ChefService.java b/chef/core/src/main/java/org/jclouds/chef/ChefService.java index a07b66ab3e..2695f7ba3b 100644 --- a/chef/core/src/main/java/org/jclouds/chef/ChefService.java +++ b/chef/core/src/main/java/org/jclouds/chef/ChefService.java @@ -1,7 +1,5 @@ package org.jclouds.chef; -import java.util.Set; - import org.jclouds.chef.domain.Node; import org.jclouds.chef.internal.BaseChefService; @@ -26,11 +24,11 @@ public interface ChefService { void deleteAllClientsAndNodesInList(Iterable names); - Set getNodes(); + Iterable getNodes(); - Set getNodesWithNamesMatching(Predicate nodeNameSelector); + Iterable getNodesWithNamesMatching(Predicate nodeNameSelector); - Set getNodesNamed(Iterable names); + Iterable getNodesNamed(Iterable names); void updateAutomaticAttributesOnNode(String nodeName); } diff --git a/chef/core/src/main/java/org/jclouds/chef/internal/BaseChefService.java b/chef/core/src/main/java/org/jclouds/chef/internal/BaseChefService.java index e790c58910..c907e8ae8a 100644 --- a/chef/core/src/main/java/org/jclouds/chef/internal/BaseChefService.java +++ b/chef/core/src/main/java/org/jclouds/chef/internal/BaseChefService.java @@ -2,8 +2,6 @@ package org.jclouds.chef.internal; import static com.google.common.base.Preconditions.checkNotNull; -import java.util.Set; - import javax.annotation.Resource; import javax.inject.Inject; import javax.inject.Named; @@ -72,17 +70,17 @@ public class BaseChefService implements ChefService { } @Override - public Set getNodes() { + public Iterable getNodes() { return getNodes.execute(); } @Override - public Set getNodesWithNamesMatching(Predicate nodeNameSelector) { + public Iterable getNodesWithNamesMatching(Predicate nodeNameSelector) { return getNodes.execute(nodeNameSelector); } @Override - public Set getNodesNamed(Iterable names) { + public Iterable getNodesNamed(Iterable names) { return getNodes.execute(names); } diff --git a/chef/core/src/main/java/org/jclouds/chef/strategy/GetNodes.java b/chef/core/src/main/java/org/jclouds/chef/strategy/GetNodes.java index f5cdd4e231..695933ce58 100644 --- a/chef/core/src/main/java/org/jclouds/chef/strategy/GetNodes.java +++ b/chef/core/src/main/java/org/jclouds/chef/strategy/GetNodes.java @@ -18,8 +18,6 @@ */ package org.jclouds.chef.strategy; -import java.util.Set; - import org.jclouds.chef.domain.Node; import org.jclouds.chef.strategy.internal.GetNodesImpl; @@ -34,9 +32,9 @@ import com.google.inject.ImplementedBy; @ImplementedBy(GetNodesImpl.class) public interface GetNodes { - Set execute(); + Iterable execute(); - Set execute(Predicate nodeNameSelector); + Iterable execute(Predicate nodeNameSelector); - Set execute(Iterable toGet); + Iterable execute(Iterable toGet); } \ No newline at end of file diff --git a/chef/core/src/main/java/org/jclouds/chef/strategy/internal/CleanupStaleNodesAndClientsImpl.java b/chef/core/src/main/java/org/jclouds/chef/strategy/internal/CleanupStaleNodesAndClientsImpl.java index 6e136583d0..ccd619df7e 100644 --- a/chef/core/src/main/java/org/jclouds/chef/strategy/internal/CleanupStaleNodesAndClientsImpl.java +++ b/chef/core/src/main/java/org/jclouds/chef/strategy/internal/CleanupStaleNodesAndClientsImpl.java @@ -68,7 +68,7 @@ public class CleanupStaleNodesAndClientsImpl implements CleanupStaleNodesAndClie final Calendar expired = Calendar.getInstance(); expired.setTime(new Date()); expired.add(Calendar.SECOND, -secondsStale); - Iterable staleNodes = filter(getAllNodes.execute(new Predicate() { + Iterable staleNodes = filter(getAllNodes.execute(new Predicate() { @Override public boolean apply(String input) { diff --git a/chef/core/src/main/java/org/jclouds/chef/strategy/internal/DeleteAllClientsAndNodesInListImpl.java b/chef/core/src/main/java/org/jclouds/chef/strategy/internal/DeleteAllClientsAndNodesInListImpl.java index 9cb1c0080b..41f292c360 100644 --- a/chef/core/src/main/java/org/jclouds/chef/strategy/internal/DeleteAllClientsAndNodesInListImpl.java +++ b/chef/core/src/main/java/org/jclouds/chef/strategy/internal/DeleteAllClientsAndNodesInListImpl.java @@ -35,7 +35,7 @@ import org.jclouds.chef.reference.ChefConstants; import org.jclouds.chef.strategy.DeleteAllClientsAndNodesInList; import org.jclouds.logging.Logger; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; import com.google.inject.Inject; /** @@ -68,7 +68,7 @@ public class DeleteAllClientsAndNodesInListImpl implements DeleteAllClientsAndNo @Override public void execute(Iterable names) { Map exceptions = newHashMap(); - Map> responses = newHashMap(); + Map> responses = newHashMap(); for (String name : names) { responses.put(name, chefAsyncClient.deleteClient(name)); responses.put(name, chefAsyncClient.deleteNode(name)); diff --git a/chef/core/src/main/java/org/jclouds/chef/strategy/internal/GetNodesImpl.java b/chef/core/src/main/java/org/jclouds/chef/strategy/internal/GetNodesImpl.java index 370b3782bc..ab467b1248 100644 --- a/chef/core/src/main/java/org/jclouds/chef/strategy/internal/GetNodesImpl.java +++ b/chef/core/src/main/java/org/jclouds/chef/strategy/internal/GetNodesImpl.java @@ -18,17 +18,11 @@ */ package org.jclouds.chef.strategy.internal; -import static com.google.common.base.Throwables.propagate; import static com.google.common.collect.Iterables.filter; -import static com.google.common.collect.Maps.newHashMap; -import static com.google.common.collect.Sets.newHashSet; -import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; -import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion; +import static org.jclouds.concurrent.ConcurrentUtils.transformParallel; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import javax.annotation.Resource; import javax.inject.Named; @@ -42,8 +36,8 @@ import org.jclouds.chef.reference.ChefConstants; import org.jclouds.chef.strategy.GetNodes; import org.jclouds.logging.Logger; +import com.google.common.base.Function; import com.google.common.base.Predicate; -import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; /** @@ -67,46 +61,33 @@ public class GetNodesImpl implements GetNodes { @Inject GetNodesImpl(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userExecutor, ChefClient getAllNode, - ChefAsyncClient ablobstore) { + ChefAsyncClient ablobstore) { this.userExecutor = userExecutor; this.chefAsyncClient = ablobstore; this.chefClient = getAllNode; } @Override - public Set execute() { + public Iterable execute() { return execute(chefClient.listNodes()); } @Override - public Set execute(Predicate nodeNameSelector) { + public Iterable execute(Predicate nodeNameSelector) { return execute(filter(chefClient.listNodes(), nodeNameSelector)); } @Override - public Set execute(Iterable toGet) { - Map exceptions = newHashMap(); - final Set nodes = newHashSet(); - Map> responses = newHashMap(); - for (String nodeName : toGet) { - final ListenableFuture future = chefAsyncClient.getNode(nodeName); - future.addListener(new Runnable() { - @Override - public void run() { - try { - nodes.add(future.get()); - } catch (InterruptedException e) { - propagate(e); - } catch (ExecutionException e) { - propagate(e); - } - } - }, sameThreadExecutor()); - responses.put(nodeName, future); - } - exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, String.format("getting nodes: %s", toGet)); - if (exceptions.size() > 0) - throw new RuntimeException(String.format("errors getting nodes: %s: %s", toGet, exceptions)); - return nodes; + public Iterable execute(Iterable toGet) { + return transformParallel(toGet, new Function>() { + + @Override + public Future apply(String from) { + return chefAsyncClient.getNode(from); + } + + }, userExecutor, maxTime, logger, "getting nodes"); + } + } \ No newline at end of file diff --git a/chef/core/src/test/java/org/jclouds/chef/strategy/internal/GetNodesImplLiveTest.java b/chef/core/src/test/java/org/jclouds/chef/strategy/internal/GetNodesImplLiveTest.java index b16bf5e7f8..bd70150fa6 100644 --- a/chef/core/src/test/java/org/jclouds/chef/strategy/internal/GetNodesImplLiveTest.java +++ b/chef/core/src/test/java/org/jclouds/chef/strategy/internal/GetNodesImplLiveTest.java @@ -23,6 +23,7 @@ */ package org.jclouds.chef.strategy.internal; +import static com.google.common.collect.Iterables.size; import static org.testng.Assert.assertEquals; import java.io.IOException; @@ -69,24 +70,24 @@ public class GetNodesImplLiveTest extends BaseChefStrategyLiveTest { @Test public void testExecute() { - assert strategy.execute().size() > 0; + assert size(strategy.execute()) > 0; } @Test public void testExecutePredicateOfString() { - assertEquals(strategy.execute(new Predicate() { + assertEquals(size(strategy.execute(new Predicate() { @Override public boolean apply(String input) { return input.startsWith(prefix); } - }).size(), 2); + })), 2); } @Test public void testExecuteIterableOfString() { - assertEquals(strategy.execute(ImmutableSet.of(prefix, prefix + 1)).size(), 2); + assertEquals(size(strategy.execute(ImmutableSet.of(prefix, prefix + 1))), 2); } } diff --git a/chef/servlet/src/main/java/org/jclouds/chef/servlet/ChefRegistrationListener.java b/chef/servlet/src/main/java/org/jclouds/chef/servlet/ChefRegistrationListener.java index 3fc7c411a4..9d1877c354 100644 --- a/chef/servlet/src/main/java/org/jclouds/chef/servlet/ChefRegistrationListener.java +++ b/chef/servlet/src/main/java/org/jclouds/chef/servlet/ChefRegistrationListener.java @@ -27,7 +27,6 @@ import static org.jclouds.chef.reference.ChefConstants.CHEF_NODE; import static org.jclouds.chef.reference.ChefConstants.CHEF_ROLE; import static org.jclouds.chef.reference.ChefConstants.CHEF_SERVICE_CLIENT; -import java.util.Enumeration; import java.util.Properties; import java.util.Set; import java.util.Map.Entry; @@ -41,18 +40,18 @@ import org.jclouds.chef.ChefClient; import org.jclouds.chef.ChefContext; import org.jclouds.chef.ChefService; import org.jclouds.chef.reference.ChefConstants; +import org.jclouds.chef.servlet.functions.InitParamsToProperties; import org.jclouds.logging.Logger; import org.jclouds.logging.jdk.JDKLogger; import org.jclouds.rest.RestContextFactory; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; /** - * Registers a new node in Chef and binds its name to - * {@link ChefConstants.CHEF_NODE}, its role to {@link ChefConstants.CHEF_ROLE} - * and the {@link ChefService} for the client to - * {@link ChefConstants.CHEF_SERVICE_CLIENT} upon initialized. Deletes the node - * and client when the context is destroyed. + * Registers a new node in Chef and binds its name to {@link ChefConstants.CHEF_NODE}, its role to + * {@link ChefConstants.CHEF_ROLE} and the {@link ChefService} for the client to + * {@link ChefConstants.CHEF_SERVICE_CLIENT} upon initialized. Deletes the node and client when the + * context is destroyed. * * @author Adrian Cole */ @@ -60,18 +59,11 @@ public class ChefRegistrationListener implements ServletContextListener { private Logger logger = new JDKLogger.JDKLoggerFactory().getLogger(ChefRegistrationListener.class.getName()); - @SuppressWarnings("unchecked") @Override public void contextInitialized(ServletContextEvent servletContextEvent) { try { logger.debug("starting initialization"); - Properties overrides = new Properties(); - Enumeration e = servletContextEvent.getServletContext().getInitParameterNames(); - while (e.hasMoreElements()) { - String propertyName = e.nextElement(); - overrides.setProperty(propertyName, servletContextEvent.getServletContext().getInitParameter(propertyName)); - - } + Properties overrides = InitParamsToProperties.INSTANCE.apply(servletContextEvent); String role = getInitParam(servletContextEvent, CHEF_ROLE); logger.trace("creating validator connection"); @@ -105,8 +97,8 @@ public class ChefRegistrationListener implements ServletContextListener { } private String findNextClientAndNodeName(ChefService validatorService, String prefix) { - ListenableFuture> nodes = validatorService.getContext().getAsyncApi().listNodes(); - ListenableFuture> clients = validatorService.getContext().getAsyncApi().listClients(); + Future> nodes = validatorService.getContext().getAsyncApi().listNodes(); + Future> clients = validatorService.getContext().getAsyncApi().listClients(); try { String nodeName; Set names = newHashSet(concat(nodes.get(), clients.get())); @@ -156,7 +148,7 @@ public class ChefRegistrationListener implements ServletContextListener { private ChefService createService(Properties props) { return ((ChefContext) new RestContextFactory(). createContext("chef", props)) - .getChefService(); + .getChefService(); } private static String getInitParam(ServletContextEvent servletContextEvent, String name) { diff --git a/chef/servlet/src/main/java/org/jclouds/chef/servlet/functions/InitParamsToProperties.java b/chef/servlet/src/main/java/org/jclouds/chef/servlet/functions/InitParamsToProperties.java new file mode 100644 index 0000000000..084448abba --- /dev/null +++ b/chef/servlet/src/main/java/org/jclouds/chef/servlet/functions/InitParamsToProperties.java @@ -0,0 +1,47 @@ +/** + * + * Copyright (C) 2010 Cloud Conscious, LLC. + * + * ==================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + */ +package org.jclouds.chef.servlet.functions; + +import java.util.Enumeration; +import java.util.Properties; + +import javax.inject.Singleton; +import javax.servlet.ServletContextEvent; + +import com.google.common.base.Function; + +/** + * + * @author Adrian Cole + */ +@Singleton +public class InitParamsToProperties implements Function { + + public static final InitParamsToProperties INSTANCE = new InitParamsToProperties(); + + public Properties apply(ServletContextEvent servletContextEvent) { + Properties overrides = new Properties(); + Enumeration e = servletContextEvent.getServletContext().getInitParameterNames(); + while (e.hasMoreElements()) { + String propertyName = e.nextElement().toString(); + overrides.setProperty(propertyName, servletContextEvent.getServletContext().getInitParameter(propertyName)); + } + return overrides; + } +} \ No newline at end of file diff --git a/compute/src/main/java/org/jclouds/compute/domain/TemplateBuilder.java b/compute/src/main/java/org/jclouds/compute/domain/TemplateBuilder.java index 2de5d41528..21b2c7d888 100644 --- a/compute/src/main/java/org/jclouds/compute/domain/TemplateBuilder.java +++ b/compute/src/main/java/org/jclouds/compute/domain/TemplateBuilder.java @@ -18,6 +18,8 @@ */ package org.jclouds.compute.domain; +import java.util.NoSuchElementException; + import org.jclouds.compute.internal.TemplateBuilderImpl; import org.jclouds.compute.options.TemplateOptions; @@ -73,6 +75,9 @@ public interface TemplateBuilder { /** * Configure this template to start in a specific location + * + * @throws NoSuchElementException + * if location matches the id specified */ TemplateBuilder locationId(String locationId); diff --git a/compute/src/main/java/org/jclouds/compute/internal/BaseComputeService.java b/compute/src/main/java/org/jclouds/compute/internal/BaseComputeService.java index 2952177acd..bbd318bae9 100755 --- a/compute/src/main/java/org/jclouds/compute/internal/BaseComputeService.java +++ b/compute/src/main/java/org/jclouds/compute/internal/BaseComputeService.java @@ -27,12 +27,13 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static org.jclouds.compute.util.ComputeServiceUtils.installNewCredentials; import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion; -import static org.jclouds.concurrent.ConcurrentUtils.makeListenable; +import static org.jclouds.concurrent.ConcurrentUtils.transformParallel; import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import javax.annotation.Nullable; import javax.annotation.Resource; @@ -75,7 +76,7 @@ import com.google.common.base.Predicates; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.Futures; /** * @@ -106,13 +107,14 @@ public class BaseComputeService implements ComputeService { @Inject protected BaseComputeService(ComputeServiceContext context, Provider> images, - Provider> sizes, Provider> locations, - ListNodesStrategy listNodesStrategy, GetNodeMetadataStrategy getNodeMetadataStrategy, - RunNodesAndAddToSetStrategy runNodesAndAddToSetStrategy, RebootNodeStrategy rebootNodeStrategy, - DestroyNodeStrategy destroyNodeStrategy, Provider templateBuilderProvider, - Provider templateOptionsProvider, @Named("NODE_RUNNING") Predicate nodeRunning, - @Named("NODE_TERMINATED") Predicate nodeTerminated, ComputeUtils utils, - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) { + Provider> sizes, Provider> locations, + ListNodesStrategy listNodesStrategy, GetNodeMetadataStrategy getNodeMetadataStrategy, + RunNodesAndAddToSetStrategy runNodesAndAddToSetStrategy, RebootNodeStrategy rebootNodeStrategy, + DestroyNodeStrategy destroyNodeStrategy, Provider templateBuilderProvider, + Provider templateOptionsProvider, + @Named("NODE_RUNNING") Predicate nodeRunning, + @Named("NODE_TERMINATED") Predicate nodeTerminated, ComputeUtils utils, + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) { this.context = checkNotNull(context, "context"); this.images = checkNotNull(images, "images"); this.sizes = checkNotNull(sizes, "sizes"); @@ -143,16 +145,15 @@ public class BaseComputeService implements ComputeService { */ @Override public Set runNodesWithTag(String tag, int count, Template template) - throws RunNodesException { + throws RunNodesException { checkArgument(tag.indexOf('-') == -1, "tag cannot contain hyphens"); checkNotNull(template.getLocation(), "location"); logger.debug(">> running %d node%s tag(%s) location(%s) image(%s) size(%s) options(%s)", count, count > 1 ? "s" - : "", tag, template.getLocation().getId(), template.getImage().getProviderId(), template.getSize() - .getProviderId(), template.getOptions()); + : "", tag, template.getLocation().getId(), template.getImage().getProviderId(), template.getSize() + .getProviderId(), template.getOptions()); Set nodes = Sets.newHashSet(); Map badNodes = Maps.newLinkedHashMap(); - Map> responses = runNodesAndAddToSetStrategy.execute(tag, count, template, nodes, - badNodes); + Map> responses = runNodesAndAddToSetStrategy.execute(tag, count, template, nodes, badNodes); Map executionExceptions = awaitCompletion(responses, executor, null, logger, "starting nodes"); if (executionExceptions.size() > 0 || badNodes.size() > 0) { throw new RunNodesException(tag, count, template, nodes, executionExceptions, badNodes); @@ -165,7 +166,7 @@ public class BaseComputeService implements ComputeService { */ @Override public Set runNodesWithTag(String tag, int count, TemplateOptions templateOptions) - throws RunNodesException { + throws RunNodesException { return runNodesWithTag(tag, count, templateBuilder().any().options(templateOptions).build()); } @@ -195,21 +196,26 @@ public class BaseComputeService implements ComputeService { @Override public Set destroyNodesMatching(Predicate filter) { logger.debug(">> destroying nodes matching(%s)", filter); - Map> responses = Maps.newHashMap(); - final Set destroyedNodes = Sets.newLinkedHashSet(); - for (final NodeMetadata node : nodesMatchingFilterAndNotTerminated(filter)) { - responses.put(node, makeListenable(executor.submit(new Callable() { - @Override - public Void call() throws Exception { - destroyNode(node.getId()); - destroyedNodes.add(node); - return null; - } - }), executor)); - } - awaitCompletion(responses, executor, null, logger, "destroying nodes"); - logger.debug("<< destroyed"); - return destroyedNodes; + Set set = Sets.newLinkedHashSet(transformParallel(nodesMatchingFilterAndNotTerminated(filter), + new Function>() { + + // TODO make an async interface instead of re-wrapping + @Override + public Future apply(final NodeMetadata from) { + return executor.submit(new Callable() { + + @Override + public NodeMetadata call() throws Exception { + destroyNode(from.getId()); + return from; + } + + }); + } + + }, executor, null, logger, "destroying nodes")); + logger.debug("<< destroyed(%d)", set.size()); + return set; } private Iterable nodesMatchingFilterAndNotTerminated(Predicate filter) { @@ -220,9 +226,9 @@ public class BaseComputeService implements ComputeService { * {@inheritDoc} */ @Override - public Set listNodes() { + public Set listNodes() { logger.debug(">> listing nodes"); - Set set = Sets.newLinkedHashSet(listNodesStrategy.list()); + Set set = Sets.newLinkedHashSet(listNodesStrategy.list()); logger.debug("<< list(%d)", set.size()); return set; } @@ -234,7 +240,7 @@ public class BaseComputeService implements ComputeService { public Set listNodesDetailsMatching(Predicate filter) { checkNotNull(filter, "filter"); logger.debug(">> listing node details matching(%s)", filter); - Set set = Sets.newLinkedHashSet(listNodesStrategy.listDetailsOnNodesMatching(filter)); + Set set = Sets.newLinkedHashSet(listNodesStrategy.listDetailsOnNodesMatching(filter)); logger.debug("<< list(%d)", set.size()); return set; } @@ -298,18 +304,15 @@ public class BaseComputeService implements ComputeService { @Override public void rebootNodesMatching(Predicate filter) { logger.debug(">> rebooting nodes matching(%s)", filter); + transformParallel(nodesMatchingFilterAndNotTerminated(filter), new Function>() { + // TODO use native async + @Override + public Future apply(NodeMetadata from) { + rebootNode(from.getId()); + return Futures.immediateFuture(null); + } - Map> responses = Maps.newHashMap(); - for (final NodeMetadata node : nodesMatchingFilterAndNotTerminated(filter)) { - responses.put(node, makeListenable(executor.submit(new Callable() { - @Override - public Void call() throws Exception { - rebootNode(node.getId()); - return null; - } - }), executor)); - } - awaitCompletion(responses, executor, null, logger, "rebooting nodes"); + }, executor, null, logger, "rebooting nodes"); logger.debug("<< rebooted"); } @@ -317,8 +320,8 @@ public class BaseComputeService implements ComputeService { * {@inheritDoc} */ @Override - public Map runScriptOnNodesMatching(Predicate filter, - Payload runScript) throws RunScriptOnNodesException { + public Map runScriptOnNodesMatching(Predicate filter, Payload runScript) + throws RunScriptOnNodesException { return runScriptOnNodesMatching(filter, runScript, RunScriptOptions.NONE); } @@ -327,19 +330,19 @@ public class BaseComputeService implements ComputeService { */ @Override public Map runScriptOnNodesMatching(Predicate filter, - final Payload runScript, @Nullable final RunScriptOptions options) throws RunScriptOnNodesException { - Iterable nodes = verifyParametersAndListNodes(filter, runScript, - (options != null) ? options : RunScriptOptions.NONE); + final Payload runScript, @Nullable final RunScriptOptions options) throws RunScriptOnNodesException { + Iterable nodes = verifyParametersAndListNodes(filter, runScript, (options != null) ? options + : RunScriptOptions.NONE); final Map execs = Maps.newHashMap(); final Map badNodes = Maps.newLinkedHashMap(); - Map> responses = Maps.newHashMap(); + Map> responses = Maps.newHashMap(); for (final NodeMetadata node : nodes) { - responses.put(node, makeListenable(executor.submit(new Callable() { + responses.put(node, executor.submit(new Callable() { @Override public Void call() throws Exception { try { @@ -363,7 +366,7 @@ public class BaseComputeService implements ComputeService { } return null; } - }), executor)); + })); } Map exceptions = awaitCompletion(responses, executor, null, logger, "starting nodes"); @@ -374,14 +377,14 @@ public class BaseComputeService implements ComputeService { } - private Iterable verifyParametersAndListNodes(Predicate filter, - Payload runScript, final RunScriptOptions options) { + private Iterable verifyParametersAndListNodes(Predicate filter, Payload runScript, + final RunScriptOptions options) { checkNotNull(filter, "Filter must be provided"); checkNotNull(runScript, "The script (represented by bytes array - use \"script\".getBytes() must be provided"); checkNotNull(options, "options"); Iterable nodes = Iterables.filter(detailsOnAllNodes(), filter); - + // TODO parallel return Iterables.transform(nodes, new Function() { @Override @@ -396,17 +399,17 @@ public class BaseComputeService implements ComputeService { // don't override checkNotNull(node.getCredentials(), "If the default credentials need to be used, they can't be null"); checkNotNull(node.getCredentials().identity, "Account name for ssh authentication must be " - + "specified. Try passing RunScriptOptions with new credentials"); + + "specified. Try passing RunScriptOptions with new credentials"); checkNotNull(node.getCredentials().credential, "Key or password for ssh authentication must be " - + "specified. Try passing RunScriptOptions with new credentials"); + + "specified. Try passing RunScriptOptions with new credentials"); } return node; } }); } - private Iterable detailsOnAllNodes() { - return listNodesStrategy.listDetailsOnNodesMatching(NodePredicates.all()); + private Set detailsOnAllNodes() { + return Sets.newLinkedHashSet(listNodesStrategy.listDetailsOnNodesMatching(NodePredicates.all())); } @Override diff --git a/compute/src/main/java/org/jclouds/compute/internal/TemplateBuilderImpl.java b/compute/src/main/java/org/jclouds/compute/internal/TemplateBuilderImpl.java index 856c9334c1..4c025df56a 100644 --- a/compute/src/main/java/org/jclouds/compute/internal/TemplateBuilderImpl.java +++ b/compute/src/main/java/org/jclouds/compute/internal/TemplateBuilderImpl.java @@ -21,7 +21,6 @@ package org.jclouds.compute.internal; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Predicates.and; import static com.google.common.collect.Iterables.filter; -import static com.google.common.collect.Iterables.find; import static com.google.common.collect.Lists.newArrayList; import static org.jclouds.util.Utils.multiMax; @@ -79,7 +78,7 @@ public class TemplateBuilderImpl implements TemplateBuilder { @VisibleForTesting protected Architecture arch; @VisibleForTesting - protected String locationId; + protected Location location; @VisibleForTesting protected String imageId; @VisibleForTesting @@ -105,8 +104,8 @@ public class TemplateBuilderImpl implements TemplateBuilder { @Inject protected TemplateBuilderImpl(Provider> locations, Provider> images, - Provider> sizes, Location defaultLocation, Provider optionsProvider, - @Named("DEFAULT") Provider defaultTemplateProvider) { + Provider> sizes, Location defaultLocation, Provider optionsProvider, + @Named("DEFAULT") Provider defaultTemplateProvider) { this.locations = locations; this.images = images; this.sizes = sizes; @@ -115,12 +114,20 @@ public class TemplateBuilderImpl implements TemplateBuilder { this.defaultTemplateProvider = defaultTemplateProvider; } + /** + * If the current location id is null, then we don't care where to launch a node. + * + * If the input location is null, then the data isn't location sensitive + * + * If the input location is a parent of the specified location, then we are ok. + */ private final Predicate locationPredicate = new Predicate() { @Override public boolean apply(ComputeMetadata input) { boolean returnVal = true; - if (locationId != null && input.getLocation() != null) - returnVal = locationId.equals(input.getLocation().getId()); + if (location != null && input.getLocation() != null) + returnVal = location.equals(input.getLocation()) || location.getParent() != null + && location.getParent().equals(input.getLocation()); return returnVal; } }; @@ -172,7 +179,7 @@ public class TemplateBuilderImpl implements TemplateBuilder { returnVal = false; else returnVal = input.getOsDescription().contains(osDescription) - || input.getOsDescription().matches(osDescription); + || input.getOsDescription().matches(osDescription); } return returnVal; } @@ -212,8 +219,8 @@ public class TemplateBuilderImpl implements TemplateBuilder { returnVal = false; else returnVal = input.getDescription().equals(imageDescription) - || input.getDescription().contains(imageDescription) - || input.getDescription().matches(imageDescription); + || input.getDescription().contains(imageDescription) + || input.getDescription().matches(imageDescription); } return returnVal; } @@ -247,12 +254,12 @@ public class TemplateBuilderImpl implements TemplateBuilder { } }; private final Predicate sizePredicate = and(sizeIdPredicate, locationPredicate, sizeCoresPredicate, - sizeRamPredicate); + sizeRamPredicate); static final Ordering DEFAULT_SIZE_ORDERING = new Ordering() { public int compare(Size left, Size right) { return ComparisonChain.start().compare(left.getCores(), right.getCores()).compare(left.getRam(), - right.getRam()).compare(left.getDisk(), right.getDisk()).result(); + right.getRam()).compare(left.getDisk(), right.getDisk()).result(); } }; static final Ordering BY_CORES_ORDERING = new Ordering() { @@ -263,10 +270,10 @@ public class TemplateBuilderImpl implements TemplateBuilder { static final Ordering DEFAULT_IMAGE_ORDERING = new Ordering() { public int compare(Image left, Image right) { return ComparisonChain.start().compare(left.getName(), right.getName(), - Ordering. natural().nullsLast()).compare(left.getVersion(), right.getVersion(), - Ordering. natural().nullsLast()).compare(left.getOsDescription(), right.getOsDescription(), - Ordering. natural().nullsLast()).compare(left.getArchitecture(), right.getArchitecture()) - .result(); + Ordering. natural().nullsLast()).compare(left.getVersion(), right.getVersion(), + Ordering. natural().nullsLast()).compare(left.getOsDescription(), right.getOsDescription(), + Ordering. natural().nullsLast()).compare(left.getArchitecture(), right.getArchitecture()) + .result(); } }; @@ -286,7 +293,7 @@ public class TemplateBuilderImpl implements TemplateBuilder { @Override public TemplateBuilder fromSize(Size size) { if (size.getLocation() != null) - this.locationId = size.getLocation().getId(); + this.location = size.getLocation(); this.minCores = size.getCores(); this.minRam = size.getRam(); return this; @@ -298,7 +305,7 @@ public class TemplateBuilderImpl implements TemplateBuilder { @Override public TemplateBuilder fromImage(Image image) { if (image.getLocation() != null) - this.locationId = image.getLocation().getId(); + this.location = image.getLocation(); if (image.getOsFamily() != null) this.os = image.getOsFamily(); if (image.getName() != null) @@ -346,7 +353,14 @@ public class TemplateBuilderImpl implements TemplateBuilder { */ @Override public TemplateBuilder locationId(final String locationId) { - this.locationId = locationId; + this.location = Iterables.find(locations.get(), new Predicate() { + + @Override + public boolean apply(Location input) { + return input.getId().equals(locationId); + } + + }); return this; } @@ -379,12 +393,11 @@ public class TemplateBuilderImpl implements TemplateBuilder { defaultTemplate.options(options); return defaultTemplate.build(); } - if (locationId == null) - locationId = defaultLocation.getId(); + if (location == null) + location = defaultLocation; if (options == null) options = optionsProvider.get(); logger.debug(">> searching params(%s)", this); - Location location = resolveLocation(); Size size = resolveSize(sizeSorter(), getImages()); Image image = resolveImage(size); logger.debug("<< matched image(%s)", image); @@ -394,19 +407,6 @@ public class TemplateBuilderImpl implements TemplateBuilder { return new TemplateImpl(image, size, location, options); } - protected Location resolveLocation() { - Location location = find(locations.get(), new Predicate() { - - @Override - public boolean apply(Location input) { - return input.getId().equals(locationId); - } - - }); - logger.debug("<< matched location(%s)", location); - return location; - } - protected Size resolveSize(Ordering sizeOrdering, final Iterable images) { Size size; try { @@ -429,7 +429,8 @@ public class TemplateBuilderImpl implements TemplateBuilder { }); size = sizeOrdering.max(filter(sizesThatAreCompatibleWithOurImages, sizePredicate)); } catch (NoSuchElementException exception) { - throw new NoSuchElementException("size didn't match: " + toString() + "\n" + sizes.get()); + throw new NoSuchElementException("sizes don't support any images: " + toString() + "\n" + sizes.get() + "\n" + + images); } logger.debug("<< matched size(%s)", size); return size; @@ -468,7 +469,8 @@ public class TemplateBuilderImpl implements TemplateBuilder { logger.trace("<< best images(%s)", maxImages); return maxImages.get(maxImages.size() - 1); } catch (NoSuchElementException exception) { - throw new NoSuchElementException("image didn't match: " + toString() + "\n" + getImages()); + Set images = getImages(); + throw new NoSuchElementException("image didn't match: " + toString() + "\n" + images); } } @@ -608,9 +610,9 @@ public class TemplateBuilderImpl implements TemplateBuilder { @VisibleForTesting boolean nothingChangedExceptOptions() { - return os == null && arch == null && locationId == null && imageId == null && sizeId == null - && osDescription == null && imageVersion == null && imageName == null && imageDescription == null - && minCores == 0 && minRam == 0 && !biggest && !fastest; + return os == null && arch == null && location == null && imageId == null && sizeId == null + && osDescription == null && imageVersion == null && imageName == null && imageDescription == null + && minCores == 0 && minRam == 0 && !biggest && !fastest; } /** @@ -624,9 +626,9 @@ public class TemplateBuilderImpl implements TemplateBuilder { @Override public String toString() { return "[arch=" + arch + ", biggest=" + biggest + ", fastest=" + fastest + ", imageName=" + imageName - + ", imageDescription=" + imageDescription + ", imageId=" + imageId + ", imageVersion=" + imageVersion - + ", location=" + locationId + ", minCores=" + minCores + ", minRam=" + minRam + ", os=" + os - + ", osDescription=" + osDescription + ", sizeId=" + sizeId + "]"; + + ", imageDescription=" + imageDescription + ", imageId=" + imageId + ", imageVersion=" + imageVersion + + ", location=" + location + ", minCores=" + minCores + ", minRam=" + minRam + ", os=" + os + + ", osDescription=" + osDescription + ", sizeId=" + sizeId + "]"; } } diff --git a/compute/src/main/java/org/jclouds/compute/strategy/RunNodesAndAddToSetStrategy.java b/compute/src/main/java/org/jclouds/compute/strategy/RunNodesAndAddToSetStrategy.java index 1a681667b6..1433edcf18 100644 --- a/compute/src/main/java/org/jclouds/compute/strategy/RunNodesAndAddToSetStrategy.java +++ b/compute/src/main/java/org/jclouds/compute/strategy/RunNodesAndAddToSetStrategy.java @@ -26,7 +26,7 @@ import org.jclouds.compute.domain.NodeMetadata; import org.jclouds.compute.domain.Template; import org.jclouds.compute.strategy.impl.EncodeTagIntoNameRunNodesAndAddToSetStrategy; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; import com.google.inject.ImplementedBy; /** @@ -37,6 +37,6 @@ import com.google.inject.ImplementedBy; @ImplementedBy(EncodeTagIntoNameRunNodesAndAddToSetStrategy.class) public interface RunNodesAndAddToSetStrategy { - Map> execute(String tag, int count, Template template, + Map> execute(String tag, int count, Template template, Set nodes, Map badNodes); } \ No newline at end of file diff --git a/compute/src/main/java/org/jclouds/compute/strategy/impl/EncodeTagIntoNameRunNodesAndAddToSetStrategy.java b/compute/src/main/java/org/jclouds/compute/strategy/impl/EncodeTagIntoNameRunNodesAndAddToSetStrategy.java index b37aa57461..eb6ad6ac4b 100755 --- a/compute/src/main/java/org/jclouds/compute/strategy/impl/EncodeTagIntoNameRunNodesAndAddToSetStrategy.java +++ b/compute/src/main/java/org/jclouds/compute/strategy/impl/EncodeTagIntoNameRunNodesAndAddToSetStrategy.java @@ -19,13 +19,12 @@ package org.jclouds.compute.strategy.impl; -import static org.jclouds.concurrent.ConcurrentUtils.makeListenable; - import java.security.SecureRandom; import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import javax.annotation.Resource; import javax.inject.Inject; @@ -47,7 +46,6 @@ import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.ListenableFuture; /** * creates futures that correlate to @@ -55,8 +53,7 @@ import com.google.common.util.concurrent.ListenableFuture; * @author Adrian Cole */ @Singleton -public class EncodeTagIntoNameRunNodesAndAddToSetStrategy implements - RunNodesAndAddToSetStrategy { +public class EncodeTagIntoNameRunNodesAndAddToSetStrategy implements RunNodesAndAddToSetStrategy { @Resource @Named(ComputeServiceConstants.COMPUTE_LOGGER) protected Logger logger = Logger.NULL; @@ -67,12 +64,9 @@ public class EncodeTagIntoNameRunNodesAndAddToSetStrategy implements protected final ExecutorService executor; @Inject - protected EncodeTagIntoNameRunNodesAndAddToSetStrategy( - AddNodeWithTagStrategy addNodeWithTagStrategy, - ListNodesStrategy listNodesStrategy, - @Named("NAMING_CONVENTION") String nodeNamingConvention, - ComputeUtils utils, - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) { + protected EncodeTagIntoNameRunNodesAndAddToSetStrategy(AddNodeWithTagStrategy addNodeWithTagStrategy, + ListNodesStrategy listNodesStrategy, @Named("NAMING_CONVENTION") String nodeNamingConvention, + ComputeUtils utils, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) { this.addNodeWithTagStrategy = addNodeWithTagStrategy; this.listNodesStrategy = listNodesStrategy; this.nodeNamingConvention = nodeNamingConvention; @@ -81,51 +75,43 @@ public class EncodeTagIntoNameRunNodesAndAddToSetStrategy implements } /** - * This implementation gets a list of acceptable node names to encode the tag - * into, then it simultaneously runs the nodes and applies options to them. + * This implementation gets a list of acceptable node names to encode the tag into, then it + * simultaneously runs the nodes and applies options to them. */ @Override - public Map> execute(final String tag, - final int count, final Template template, - final Set nodes, - final Map badNodes) { - Map> responses = Maps.newHashMap(); + public Map> execute(final String tag, final int count, final Template template, + final Set nodes, final Map badNodes) { + Map> responses = Maps.newHashMap(); for (final String name : getNextNames(tag, template, count)) { - responses.put(name, makeListenable(executor - .submit(new Callable() { - @Override - public Void call() throws Exception { - NodeMetadata node = null; - logger.debug(">> starting node(%s) tag(%s)", name, tag); - node = addNodeWithTagStrategy.execute(tag, name, template); - logger.debug("<< %s node(%s)", node.getState(), node - .getId()); - utils - .runOptionsOnNodeAndAddToGoodSetOrPutExceptionIntoBadMap( - node, badNodes, nodes, template.getOptions()) - .call(); - return null; - } - }), executor)); + responses.put(name, executor.submit(new Callable() { + @Override + public Void call() throws Exception { + NodeMetadata node = null; + logger.debug(">> starting node(%s) tag(%s)", name, tag); + node = addNodeWithTagStrategy.execute(tag, name, template); + logger.debug("<< %s node(%s)", node.getState(), node.getId()); + utils.runOptionsOnNodeAndAddToGoodSetOrPutExceptionIntoBadMap(node, badNodes, nodes, + template.getOptions()).call(); + return null; + } + })); } return responses; } /** - * Find the next node names that can be used. These will be derived from the - * tag and the template. We will pre-allocate a specified quantity, and - * attempt to verify that there is no name conflict with the current service. + * Find the next node names that can be used. These will be derived from the tag and the + * template. We will pre-allocate a specified quantity, and attempt to verify that there is no + * name conflict with the current service. * * @param tag * @param count * @param template * @return */ - protected Set getNextNames(final String tag, - final Template template, int count) { + protected Set getNextNames(final String tag, final Template template, int count) { Set names = Sets.newHashSet(); - Iterable currentNodes = listNodesStrategy - .list(); + Iterable currentNodes = listNodesStrategy.list(); int maxTries = 100; int currentTries = 0; while (names.size() < count && currentTries++ < maxTries) { @@ -145,16 +131,14 @@ public class EncodeTagIntoNameRunNodesAndAddToSetStrategy implements } /** - * Get a name using a random mechanism that still ties all nodes in a tag - * together. + * Get a name using a random mechanism that still ties all nodes in a tag together. * - * This implementation will pass the tag and a hex formatted random number to - * the configured naming convention. + * This implementation will pass the tag and a hex formatted random number to the configured + * naming convention. * */ protected String getNextName(final String tag, final Template template) { - return String.format(nodeNamingConvention, tag, Integer - .toHexString(new SecureRandom().nextInt(4095))); + return String.format(nodeNamingConvention, tag, Integer.toHexString(new SecureRandom().nextInt(4095))); } } \ No newline at end of file diff --git a/compute/src/main/java/org/jclouds/compute/stub/config/StubComputeServiceContextModule.java b/compute/src/main/java/org/jclouds/compute/stub/config/StubComputeServiceContextModule.java index 0ea1b81688..db856e51bb 100644 --- a/compute/src/main/java/org/jclouds/compute/stub/config/StubComputeServiceContextModule.java +++ b/compute/src/main/java/org/jclouds/compute/stub/config/StubComputeServiceContextModule.java @@ -180,12 +180,12 @@ public class StubComputeServiceContextModule extends AbstractModule { private NodeState state; private final ExecutorService service; - public StubNodeMetadata(String providerId, String name, String id, Location location, - URI uri, Map userMetadata, String tag, Image image, NodeState state, - Iterable publicAddresses, Iterable privateAddresses, - Map extra, Credentials credentials, ExecutorService service) { - super(providerId, name, id, location, uri, userMetadata, tag, image, state, - publicAddresses, privateAddresses, extra, credentials); + public StubNodeMetadata(String providerId, String name, String id, Location location, URI uri, + Map userMetadata, String tag, Image image, NodeState state, + Iterable publicAddresses, Iterable privateAddresses, Map extra, + Credentials credentials, ExecutorService service) { + super(providerId, name, id, location, uri, userMetadata, tag, image, state, publicAddresses, privateAddresses, + extra, credentials); this.setState(state, 0); this.service = service; } @@ -227,12 +227,10 @@ public class StubComputeServiceContextModule extends AbstractModule { private final String passwordPrefix; @Inject - public StubAddNodeWithTagStrategy(ConcurrentMap nodes, - Location location, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, - @Named("NODE_ID") Provider idProvider, - @Named("PUBLIC_IP_PREFIX") String publicIpPrefix, - @Named("PRIVATE_IP_PREFIX") String privateIpPrefix, - @Named("PASSWORD_PREFIX") String passwordPrefix) { + public StubAddNodeWithTagStrategy(ConcurrentMap nodes, Location location, + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService service, + @Named("NODE_ID") Provider idProvider, @Named("PUBLIC_IP_PREFIX") String publicIpPrefix, + @Named("PRIVATE_IP_PREFIX") String privateIpPrefix, @Named("PASSWORD_PREFIX") String passwordPrefix) { this.nodes = nodes; this.location = location; this.service = Executors.newCachedThreadPool(); @@ -244,14 +242,12 @@ public class StubComputeServiceContextModule extends AbstractModule { @Override public NodeMetadata execute(String tag, String name, Template template) { - checkArgument(location.equals(template.getLocation()), "invalid location: " - + template.getLocation()); + checkArgument(location.equals(template.getLocation()), "invalid location: " + template.getLocation()); int id = idProvider.get(); - StubNodeMetadata node = new StubNodeMetadata(id + "", name, id + "", location, null, - ImmutableMap. of(), tag, template.getImage(), NodeState.PENDING, - ImmutableSet. of(publicIpPrefix + id), ImmutableSet - . of(privateIpPrefix + id), ImmutableMap. of(), - new Credentials("root", passwordPrefix + id), service); + StubNodeMetadata node = new StubNodeMetadata(id + "", name, id + "", location, null, ImmutableMap + . of(), tag, template.getImage(), NodeState.PENDING, ImmutableSet + . of(publicIpPrefix + id), ImmutableSet. of(privateIpPrefix + id), ImmutableMap + . of(), new Credentials("root", passwordPrefix + id), service); nodes.put(id, node); node.setState(NodeState.RUNNING, 100); return node; @@ -289,8 +285,7 @@ public class StubComputeServiceContextModule extends AbstractModule { } @Override - public Iterable listDetailsOnNodesMatching( - Predicate filter) { + public Iterable listDetailsOnNodesMatching(Predicate filter) { return Iterables.filter(nodes.values(), filter); } } @@ -363,18 +358,16 @@ public class StubComputeServiceContextModule extends AbstractModule { @Provides @Singleton protected Set provideSizes() { - return ImmutableSet.of(new StubSize("small", 1, 1740, 160, ImmutableSet - .of(Architecture.X86_32)), new StubSize("medium", 4, 7680, 850, ImmutableSet - .of(Architecture.X86_64)), new StubSize("large", 8, 15360, 1690, ImmutableSet - .of(Architecture.X86_64))); + return ImmutableSet. of(new StubSize("small", 1, 1740, 160, ImmutableSet.of(Architecture.X86_32)), + new StubSize("medium", 4, 7680, 850, ImmutableSet.of(Architecture.X86_64)), new StubSize("large", 8, + 15360, 1690, ImmutableSet.of(Architecture.X86_64))); } private static class StubSize extends org.jclouds.compute.domain.internal.SizeImpl { /** The serialVersionUID */ private static final long serialVersionUID = -1842135761654973637L; - StubSize(String type, int cores, int ram, int disk, - Iterable supportedArchitectures) { + StubSize(String type, int cores, int ram, int disk, Iterable supportedArchitectures) { super(type, type, type, null, null, ImmutableMap. of(), cores, ram, disk, architectureIn(supportedArchitectures)); } @@ -383,22 +376,23 @@ public class StubComputeServiceContextModule extends AbstractModule { @Provides @Singleton protected Set provideImages(Location defaultLocation) { - return ImmutableSet.of(new ImageImpl("1", OsFamily.UBUNTU.name(), "1", defaultLocation, null, - ImmutableMap. of(), "stub ubuntu 32", "", OsFamily.UBUNTU, - "ubuntu 64", Architecture.X86_64, new Credentials("root", null)), new ImageImpl("2", - OsFamily.UBUNTU.name(), "2", defaultLocation, null, ImmutableMap - . of(), "stub ubuntu 64", "", OsFamily.UBUNTU, "ubuntu 64", - Architecture.X86_64, new Credentials("root", null)), new ImageImpl("3", - OsFamily.CENTOS.name(), "3", defaultLocation, null, ImmutableMap - . of(), "stub centos 64", "", OsFamily.CENTOS, "centos 64", - Architecture.X86_64, new Credentials("root", null))); + String parentId = defaultLocation.getParent().getId(); + return ImmutableSet. of(new ImageImpl("1", OsFamily.UBUNTU.name(), parentId + "/1", defaultLocation + .getParent(), null, ImmutableMap. of(), "stub ubuntu 32", "", OsFamily.UBUNTU, + "ubuntu 64", Architecture.X86_64, new Credentials("root", null)), new ImageImpl("2", OsFamily.UBUNTU + .name(), parentId + "/2", defaultLocation.getParent(), null, ImmutableMap. of(), + "stub ubuntu 64", "", OsFamily.UBUNTU, "ubuntu 64", Architecture.X86_64, new Credentials("root", null)), + new ImageImpl("3", OsFamily.CENTOS.name(), parentId + "/3", defaultLocation.getParent(), null, + ImmutableMap. of(), "stub centos 64", "", OsFamily.CENTOS, "centos 64", + Architecture.X86_64, new Credentials("root", null))); } @Provides @Singleton Location getLocation(@org.jclouds.rest.annotations.Provider String providerName) { Location provider = new LocationImpl(LocationScope.PROVIDER, providerName, providerName, null); - return new LocationImpl(LocationScope.ZONE, providerName, providerName, provider); + Location region = new LocationImpl(LocationScope.REGION, providerName+"region", providerName+"region", provider); + return new LocationImpl(LocationScope.ZONE, providerName+"zone", providerName+"zone", region); } @Provides diff --git a/compute/src/main/java/org/jclouds/compute/util/ComputeUtils.java b/compute/src/main/java/org/jclouds/compute/util/ComputeUtils.java index 171bd78cb9..cb4010ad41 100644 --- a/compute/src/main/java/org/jclouds/compute/util/ComputeUtils.java +++ b/compute/src/main/java/org/jclouds/compute/util/ComputeUtils.java @@ -23,7 +23,6 @@ import static com.google.common.base.Preconditions.checkState; import static org.jclouds.compute.util.ComputeServiceUtils.installNewCredentials; import static org.jclouds.compute.util.ComputeServiceUtils.isKeyAuth; import static org.jclouds.concurrent.ConcurrentUtils.awaitCompletion; -import static org.jclouds.concurrent.ConcurrentUtils.makeListenable; import java.util.List; import java.util.Map; @@ -31,6 +30,7 @@ import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; @@ -49,7 +49,6 @@ import org.jclouds.compute.reference.ComputeServiceConstants; import org.jclouds.compute.reference.ComputeServiceConstants.Timeouts; import org.jclouds.compute.strategy.GetNodeMetadataStrategy; import org.jclouds.compute.util.ComputeServiceUtils.SshCallable; -import org.jclouds.concurrent.ConcurrentUtils; import org.jclouds.io.Payload; import org.jclouds.logging.Logger; import org.jclouds.net.IPSocket; @@ -61,7 +60,6 @@ import com.google.common.base.Throwables; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; /** @@ -84,9 +82,10 @@ public class ComputeUtils { @Inject public ComputeUtils(Predicate socketTester, - @Named("SCRIPT_COMPLETE") Predicate runScriptNotRunning, GetNodeMetadataStrategy getNode, - Timeouts timeouts, @Named("NODE_RUNNING") Predicate nodeRunning, - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) { + @Named("SCRIPT_COMPLETE") Predicate runScriptNotRunning, + GetNodeMetadataStrategy getNode, Timeouts timeouts, + @Named("NODE_RUNNING") Predicate nodeRunning, + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) { this.nodeRunning = nodeRunning; this.timeouts = timeouts; this.getNode = getNode; @@ -95,19 +94,20 @@ public class ComputeUtils { this.executor = executor; } - public Map> runOptionsOnNodesAndAddToGoodSetOrPutExceptionIntoBadMap( - final TemplateOptions options, Iterable runningNodes, final Set goodNodes, - final Map badNodes) { - Map> responses = Maps.newHashMap(); + public Map> runOptionsOnNodesAndAddToGoodSetOrPutExceptionIntoBadMap(final TemplateOptions options, + Iterable runningNodes, final Set goodNodes, + final Map badNodes) { + Map> responses = Maps.newHashMap(); for (final NodeMetadata node : runningNodes) { - responses.put(node, makeListenable(executor.submit(runOptionsOnNodeAndAddToGoodSetOrPutExceptionIntoBadMap( - node, badNodes, goodNodes, options)), executor)); + responses.put(node, executor.submit(runOptionsOnNodeAndAddToGoodSetOrPutExceptionIntoBadMap(node, badNodes, + goodNodes, options))); } return responses; } public Callable runOptionsOnNodeAndAddToGoodSetOrPutExceptionIntoBadMap(final NodeMetadata node, - final Map badNodes, final Set goodNodes, final TemplateOptions options) { + final Map badNodes, final Set goodNodes, + final TemplateOptions options) { return new Callable() { @Override public Void call() throws Exception { @@ -117,7 +117,7 @@ public class ComputeUtils { goodNodes.add(node1); } catch (Exception e) { logger.error(e, "<< problem applying options to node(%s): ", node.getId(), Throwables.getRootCause(e) - .getMessage()); + .getMessage()); badNodes.put(node, e); } return null; @@ -133,8 +133,8 @@ public class ComputeUtils { node = installNewCredentials(getNode.execute(node.getId()), node.getCredentials()); else throw new IllegalStateException(String.format( - "node didn't achieve the state running on node %s within %d seconds, final state: %s", node.getId(), - timeouts.nodeRunning / 1000, node.getState())); + "node didn't achieve the state running on node %s within %d seconds, final state: %s", node.getId(), + timeouts.nodeRunning / 1000, node.getState())); List> callables = Lists.newArrayList(); if (options.getRunScript() != null) { @@ -148,13 +148,13 @@ public class ComputeUtils { // fail. if (callables.size() > 0 || options.getPrivateKey() != null) { runCallablesOnNode(node, callables, options.getPrivateKey() != null ? installKeyOnNode(node, options - .getPrivateKey()) : null); + .getPrivateKey()) : null); } if (options.getPort() > 0) { checkNodeHasPublicIps(node); blockUntilPortIsListeningOnPublicIp(options.getPort(), options.getSeconds(), Iterables.get(node - .getPublicAddresses(), 0)); + .getPublicAddresses(), 0)); } return node; } @@ -190,11 +190,12 @@ public class ComputeUtils { return new RunScriptOnNode(runScriptNotRunning, node, scriptName, script, false); } - public Map, ?> runCallablesOnNode(NodeMetadata node, Iterable> parallel, - @Nullable SshCallable last) { + public Map, ?> runCallablesOnNode(NodeMetadata node, Iterable> parallel, + @Nullable SshCallable last) { checkState(this.sshFactory != null, "runScript requested, but no SshModule configured"); checkNodeHasPublicIps(node); - checkNotNull(node.getCredentials().credential, "credentials.key for node " + node.getId()); + checkNotNull(node.getCredentials(), "credentials for node " + node.getId()); + checkNotNull(node.getCredentials().credential, "credentials.credential for node " + node.getId()); SshClient ssh = createSshClientOncePortIsListeningOnNode(node); try { ssh.connect(); @@ -205,8 +206,8 @@ public class ComputeUtils { } } - private Map, ?> runTasksUsingSshClient(Iterable> parallel, - SshCallable last, SshClient ssh) { + private Map, ?> runTasksUsingSshClient(Iterable> parallel, SshCallable last, + SshClient ssh) { Map, Object> responses = Maps.newHashMap(); if (Iterables.size(parallel) > 0) { responses.putAll(runCallablesUsingSshClient(parallel, ssh)); @@ -226,18 +227,18 @@ public class ComputeUtils { IPSocket socket = new IPSocket(Iterables.get(node.getPublicAddresses(), 0), 22); socketTester.apply(socket); SshClient ssh = isKeyAuth(node) ? sshFactory.create(socket, node.getCredentials().identity, - node.getCredentials().credential.getBytes()) : sshFactory.create(socket, node.getCredentials().identity, - node.getCredentials().credential); + node.getCredentials().credential.getBytes()) : sshFactory.create(socket, node.getCredentials().identity, + node.getCredentials().credential); return ssh; } - private Map, Object> runCallablesUsingSshClient(Iterable> parallel, - SshClient ssh) { - Map, ListenableFuture> parallelResponses = Maps.newHashMap(); + // TODO refactor + private Map, Object> runCallablesUsingSshClient(Iterable> parallel, SshClient ssh) { + Map, Future> parallelResponses = Maps.newHashMap(); for (SshCallable callable : parallel) { callable.setConnection(ssh, logger); - parallelResponses.put(callable, ConcurrentUtils.makeListenable(executor.submit(callable), executor)); + parallelResponses.put(callable, executor.submit(callable)); } Map, Exception> exceptions = awaitCompletion(parallelResponses, executor, null, logger, "ssh"); @@ -248,9 +249,9 @@ public class ComputeUtils { } @SuppressWarnings("unchecked") - public Map, T> transform(Map, ListenableFuture> responses) { + public Map, T> transform(Map, Future> responses) { Map, T> actualResponses = Maps.newHashMap(); - for (Map.Entry, ListenableFuture> entry : responses.entrySet()) { + for (Map.Entry, Future> entry : responses.entrySet()) { try { actualResponses.put(entry.getKey(), (T) entry.getValue().get()); } catch (InterruptedException e) { diff --git a/compute/src/test/java/org/jclouds/compute/BaseComputeServiceLiveTest.java b/compute/src/test/java/org/jclouds/compute/BaseComputeServiceLiveTest.java index f0187df79b..fd5fc33a3f 100755 --- a/compute/src/test/java/org/jclouds/compute/BaseComputeServiceLiveTest.java +++ b/compute/src/test/java/org/jclouds/compute/BaseComputeServiceLiveTest.java @@ -26,7 +26,7 @@ import static com.google.common.collect.Iterables.concat; import static com.google.common.collect.Iterables.get; import static com.google.common.collect.Iterables.getOnlyElement; import static com.google.common.collect.Sets.filter; -import static com.google.common.collect.Sets.newHashSet; +import static com.google.common.collect.Sets.newLinkedHashSet; import static com.google.common.collect.Sets.newTreeSet; import static org.jclouds.compute.options.RunScriptOptions.Builder.overrideCredentialsWith; import static org.jclouds.compute.predicates.NodePredicates.TERMINATED; @@ -137,7 +137,7 @@ public abstract class BaseComputeServiceLiveTest { String secret = Files.toString(new File(secretKeyFile), Charsets.UTF_8); assert secret.startsWith("-----BEGIN RSA PRIVATE KEY-----") : "invalid key:\n" + secret; return ImmutableMap. of("private", secret, "public", Files.toString(new File(secretKeyFile - + ".pub"), Charsets.UTF_8)); + + ".pub"), Charsets.UTF_8)); } protected void setupCredentials() { @@ -153,7 +153,7 @@ public abstract class BaseComputeServiceLiveTest { if (context != null) context.close(); context = new ComputeServiceContextFactory().createContext(provider, identity, credential, ImmutableSet.of( - new Log4JLoggingModule(), getSshModule())); + new Log4JLoggingModule(), getSshModule())); client = context.getComputeService(); } @@ -170,7 +170,7 @@ public abstract class BaseComputeServiceLiveTest { @Test(enabled = true, expectedExceptions = AuthorizationException.class) public void testCorrectAuthException() throws Exception { new ComputeServiceContextFactory().createContext(provider, "MOMMA", "MIA", - ImmutableSet. of(new Log4JLoggingModule())).close(); + ImmutableSet. of(new Log4JLoggingModule())).close(); } @Test(enabled = true, dependsOnMethods = "testCorrectAuthException") @@ -197,13 +197,13 @@ public abstract class BaseComputeServiceLiveTest { try { Set nodes = client.runNodesWithTag(tag, 1, options); Credentials good = nodes.iterator().next().getCredentials(); - assert good.identity != null; - assert good.credential != null; + assert good.identity != null : nodes; + assert good.credential != null : nodes; Image image = get(nodes, 0).getImage(); try { Map responses = runScriptWithCreds(tag, image.getOsFamily(), - new Credentials(good.identity, "romeo")); + new Credentials(good.identity, "romeo")); assert false : "shouldn't pass with a bad password\n" + responses; } catch (RunScriptOnNodesException e) { assert getRootCause(e).getMessage().contains("Auth fail") : e; @@ -237,8 +237,8 @@ public abstract class BaseComputeServiceLiveTest { template = buildTemplate(client.templateBuilder()); template.getOptions().installPrivateKey(newStringPayload(keyPair.get("private"))).authorizePublicKey( - newStringPayload(keyPair.get("public"))).runScript( - newStringPayload(buildScript(template.getImage().getOsFamily()))); + newStringPayload(keyPair.get("public"))).runScript( + newStringPayload(buildScript(template.getImage().getOsFamily()))); try { nodes = newTreeSet(client.runNodesWithTag(tag, 2, template)); } catch (RunNodesException e) { @@ -281,10 +281,10 @@ public abstract class BaseComputeServiceLiveTest { } protected Map runScriptWithCreds(final String tag, OsFamily osFamily, - Credentials creds) throws RunScriptOnNodesException { + Credentials creds) throws RunScriptOnNodesException { try { return client.runScriptOnNodesMatching(runningWithTag(tag), newStringPayload(buildScript(osFamily)), - overrideCredentialsWith(creds)); + overrideCredentialsWith(creds)); } catch (SshException e) { if (getRootCause(e).getMessage().contains("Auth fail")) { // System.err.printf("bad credentials: %s:%s for %s%n", @@ -317,38 +317,39 @@ public abstract class BaseComputeServiceLiveTest { public static String buildScript(OsFamily osFamily) { switch (osFamily) { - case UBUNTU: - return new StringBuilder()// - .append("echo nameserver 208.67.222.222 >> /etc/resolv.conf\n")// - .append("cp /etc/apt/sources.list /etc/apt/sources.list.old\n")// - .append( - "sed 's~us.archive.ubuntu.com~mirror.anl.gov/pub~g' /etc/apt/sources.list.old >/etc/apt/sources.list\n")// - .append("apt-get update\n")// - .append("apt-get install -f -y --force-yes openjdk-6-jdk\n")// - .append("wget -qO/usr/bin/runurl run.alestic.com/runurl\n")// - .append("chmod 755 /usr/bin/runurl\n")// - .toString(); - case CENTOS: - case RHEL: - return new StringBuilder() - .append("echo nameserver 208.67.222.222 >> /etc/resolv.conf\n") - .append("echo \"[jdkrepo]\" >> /etc/yum.repos.d/CentOS-Base.repo\n") - .append("echo \"name=jdkrepository\" >> /etc/yum.repos.d/CentOS-Base.repo\n") - .append( - "echo \"baseurl=http://ec2-us-east-mirror.rightscale.com/epel/5/i386/\" >> /etc/yum.repos.d/CentOS-Base.repo\n") - .append("echo \"enabled=1\" >> /etc/yum.repos.d/CentOS-Base.repo\n").append( - "yum --nogpgcheck -y install java-1.6.0-openjdk\n").append( - "echo \"export PATH=\\\"/usr/lib/jvm/jre-1.6.0-openjdk/bin/:\\$PATH\\\"\" >> /root/.bashrc\n") - .toString(); - default: - throw new IllegalArgumentException(osFamily.toString()); + case UBUNTU: + return new StringBuilder()// + .append("echo nameserver 208.67.222.222 >> /etc/resolv.conf\n")// + .append("cp /etc/apt/sources.list /etc/apt/sources.list.old\n")// + .append( + "sed 's~us.archive.ubuntu.com~mirror.anl.gov/pub~g' /etc/apt/sources.list.old >/etc/apt/sources.list\n")// + .append("apt-get update\n")// + .append("apt-get install -f -y --force-yes openjdk-6-jdk\n")// + .append("wget -qO/usr/bin/runurl run.alestic.com/runurl\n")// + .append("chmod 755 /usr/bin/runurl\n")// + .toString(); + case CENTOS: + case RHEL: + return new StringBuilder() + .append("echo nameserver 208.67.222.222 >> /etc/resolv.conf\n") + .append("echo \"[jdkrepo]\" >> /etc/yum.repos.d/CentOS-Base.repo\n") + .append("echo \"name=jdkrepository\" >> /etc/yum.repos.d/CentOS-Base.repo\n") + .append( + "echo \"baseurl=http://ec2-us-east-mirror.rightscale.com/epel/5/i386/\" >> /etc/yum.repos.d/CentOS-Base.repo\n") + .append("echo \"enabled=1\" >> /etc/yum.repos.d/CentOS-Base.repo\n") + .append("yum --nogpgcheck -y install java-1.6.0-openjdk\n") + .append( + "echo \"export PATH=\\\"/usr/lib/jvm/jre-1.6.0-openjdk/bin/:\\$PATH\\\"\" >> /root/.bashrc\n") + .toString(); + default: + throw new IllegalArgumentException(osFamily.toString()); } } @Test(enabled = true, dependsOnMethods = "testCreateAnotherNodeWithANewContextToEnsureSharedMemIsntRequired") public void testGet() throws Exception { - Set metadataSet = newHashSet(filter(client.listNodesDetailsMatching(all()), and( - withTag(tag), not(TERMINATED)))); + Set nodes = client.listNodesDetailsMatching(all()); + Set metadataSet = newLinkedHashSet(filter(nodes, and(withTag(tag), not(TERMINATED)))); for (NodeMetadata node : nodes) { metadataSet.remove(node); NodeMetadata metadata = client.getNodeMetadata(node.getId()); @@ -365,7 +366,7 @@ public abstract class BaseComputeServiceLiveTest { protected void assertNodeZero(Set metadataSet) { assert metadataSet.size() == 0 : String.format("nodes left in set: [%s] which didn't match set: [%s]", - metadataSet, nodes); + metadataSet, nodes); } @Test(enabled = true, dependsOnMethods = "testGet") @@ -426,26 +427,26 @@ public abstract class BaseComputeServiceLiveTest { assert location != location.getParent() : location; assert location.getScope() != null : location; switch (location.getScope()) { - case PROVIDER: - assertProvider(location); - break; - case REGION: - assertProvider(location.getParent()); - break; - case ZONE: - Location provider = location.getParent().getParent(); - // zone can be a direct descendant of provider - if (provider == null) - provider = location.getParent(); - assertProvider(provider); - break; - case HOST: - Location provider2 = location.getParent().getParent().getParent(); - // zone can be a direct descendant of provider - if (provider2 == null) - provider2 = location.getParent().getParent(); - assertProvider(provider2); - break; + case PROVIDER: + assertProvider(location); + break; + case REGION: + assertProvider(location.getParent()); + break; + case ZONE: + Location provider = location.getParent().getParent(); + // zone can be a direct descendant of provider + if (provider == null) + provider = location.getParent(); + assertProvider(provider); + break; + case HOST: + Location provider2 = location.getParent().getParent().getParent(); + // zone can be a direct descendant of provider + if (provider2 == null) + provider2 = location.getParent().getParent(); + assertProvider(provider2); + break; } } } diff --git a/compute/src/test/java/org/jclouds/compute/StubComputeServiceIntegrationTest.java b/compute/src/test/java/org/jclouds/compute/StubComputeServiceIntegrationTest.java index 1c1f7bb36b..be7e55f832 100644 --- a/compute/src/test/java/org/jclouds/compute/StubComputeServiceIntegrationTest.java +++ b/compute/src/test/java/org/jclouds/compute/StubComputeServiceIntegrationTest.java @@ -32,6 +32,8 @@ import java.io.IOException; import java.io.Serializable; import java.util.Set; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import org.easymock.IArgumentMatcher; import org.jclouds.compute.domain.Architecture; @@ -86,7 +88,7 @@ public class StubComputeServiceIntegrationTest extends BaseComputeServiceLiveTes Template defaultTemplate = client.templateBuilder().build(); assertEquals(defaultTemplate.getImage().getArchitecture(), Architecture.X86_64); assertEquals(defaultTemplate.getImage().getOsFamily(), OsFamily.UBUNTU); - assertEquals(defaultTemplate.getLocation().getId(), provider); + assertEquals(defaultTemplate.getLocation().getId(), provider + "zone"); assertEquals(defaultTemplate.getSize().getCores(), 4.0d); } @@ -109,17 +111,17 @@ public class StubComputeServiceIntegrationTest extends BaseComputeServiceLiveTes expect(open.apply(new IPSocket("144.175.1.4", 22))).andReturn(true); expect( - factory.create(eq(new IPSocket("144.175.1.1", 22)), eq("root"), aryEq(keyPair.get("private") - .getBytes()))).andReturn(client1).atLeastOnce(); + factory.create(eq(new IPSocket("144.175.1.1", 22)), eq("root"), aryEq(keyPair.get("private") + .getBytes()))).andReturn(client1).atLeastOnce(); expect( - factory.create(eq(new IPSocket("144.175.1.2", 22)), eq("root"), aryEq(keyPair.get("private") - .getBytes()))).andReturn(client2).atLeastOnce(); + factory.create(eq(new IPSocket("144.175.1.2", 22)), eq("root"), aryEq(keyPair.get("private") + .getBytes()))).andReturn(client2).atLeastOnce(); expect( - factory.create(eq(new IPSocket("144.175.1.3", 22)), eq("root"), aryEq(keyPair.get("private") - .getBytes()))).andReturn(client3).atLeastOnce(); + factory.create(eq(new IPSocket("144.175.1.3", 22)), eq("root"), aryEq(keyPair.get("private") + .getBytes()))).andReturn(client3).atLeastOnce(); expect( - factory.create(eq(new IPSocket("144.175.1.4", 22)), eq("root"), aryEq(keyPair.get("private") - .getBytes()))).andReturn(client4).atLeastOnce(); + factory.create(eq(new IPSocket("144.175.1.4", 22)), eq("root"), aryEq(keyPair.get("private") + .getBytes()))).andReturn(client4).atLeastOnce(); helloAndJava(client1); helloAndJava(client2); @@ -163,20 +165,20 @@ public class StubComputeServiceIntegrationTest extends BaseComputeServiceLiveTes SshClient client4 = createMock(SshClient.class); expect(factory.create(new IPSocket("144.175.1.1", 22), "root", "romeo")).andThrow( - new SshException("Auth fail")); + new SshException("Auth fail")); expect(factory.create(new IPSocket("144.175.1.1", 22), "root", "password1")).andReturn(client1) - .atLeastOnce(); + .atLeastOnce(); client1.connect(); runScript(client1, "computeserv", 1); client1.disconnect(); expect(factory.create(new IPSocket("144.175.1.2", 22), "root", "password2")).andReturn(client2) - .atLeastOnce(); + .atLeastOnce(); expect(factory.create(new IPSocket("144.175.1.3", 22), "root", "password3")).andReturn(client3) - .atLeastOnce(); + .atLeastOnce(); expect(factory.create(new IPSocket("144.175.1.4", 22), "root", "password4")).andReturn(client4) - .atLeastOnce(); + .atLeastOnce(); runScriptAndInstallSsh(client2, "runscript", 2); runScriptAndInstallSsh(client3, "runscript", 3); @@ -240,9 +242,9 @@ public class StubComputeServiceIntegrationTest extends BaseComputeServiceLiveTes public static String initScript(String scriptName, String script) { return new InitBuilder(scriptName, "/tmp/" + scriptName, "/tmp/" + scriptName, - ImmutableMap. of(), Iterables.toArray(Splitter.on("\n").split( - new String(checkNotNull(script, "script"))), String.class)) - .build(org.jclouds.scriptbuilder.domain.OsFamily.UNIX); + ImmutableMap. of(), Iterables.toArray(Splitter.on("\n").split( + new String(checkNotNull(script, "script"))), String.class)) + .build(org.jclouds.scriptbuilder.domain.OsFamily.UNIX); } public static Payload payloadEq(String value) { @@ -253,7 +255,7 @@ public class StubComputeServiceIntegrationTest extends BaseComputeServiceLiveTes public void testAssignability() throws Exception { @SuppressWarnings("unused") RestContext, ConcurrentMap> stubContext = new ComputeServiceContextFactory() - .createContext(provider, identity, credential).getProviderSpecificContext(); + .createContext(provider, identity, credential).getProviderSpecificContext(); } private static class PayloadEquals implements IArgumentMatcher, Serializable { @@ -303,7 +305,7 @@ public class StubComputeServiceIntegrationTest extends BaseComputeServiceLiveTes return false; PayloadEquals other = (PayloadEquals) o; return this.expected == null && other.expected == null || this.expected != null - && this.expected.equals(other.expected); + && this.expected.equals(other.expected); } @Override @@ -361,4 +363,9 @@ public class StubComputeServiceIntegrationTest extends BaseComputeServiceLiveTes super.testTemplateMatch(); } + @Override + protected void cleanup() throws InterruptedException, ExecutionException, TimeoutException { + super.cleanup(); + } + } \ No newline at end of file diff --git a/compute/src/test/java/org/jclouds/compute/internal/TemplateBuilderImplTest.java b/compute/src/test/java/org/jclouds/compute/internal/TemplateBuilderImplTest.java index 68e09b218d..0114f06222 100644 --- a/compute/src/test/java/org/jclouds/compute/internal/TemplateBuilderImplTest.java +++ b/compute/src/test/java/org/jclouds/compute/internal/TemplateBuilderImplTest.java @@ -59,12 +59,12 @@ public class TemplateBuilderImplTest { Image image2 = createMock(Image.class); Size size = new SizeImpl("sizeId", null, "sizeId", defaultLocation, null, ImmutableMap. of(), - 1.0, 0, 0, ImagePredicates.any()); + 1.0, 0, 0, ImagePredicates.any()); Provider> locations = Providers.> of(ImmutableSet - . of(defaultLocation)); + . of(defaultLocation)); Provider> images = Providers.> of(ImmutableSet - . of(image, image2)); + . of(image, image2)); Provider> sizes = Providers.> of(ImmutableSet. of(size)); Provider optionsProvider = createMock(Provider.class); Provider templateBuilderProvider = createMock(Provider.class); @@ -87,7 +87,7 @@ public class TemplateBuilderImplTest { replay(templateBuilderProvider); TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes, defaultLocation, optionsProvider, - templateBuilderProvider); + templateBuilderProvider); assertEquals(template.resolveImage(size), image2); @@ -107,18 +107,17 @@ public class TemplateBuilderImplTest { Image image2 = createMock(Image.class); Size size = new SizeImpl("sizeId", null, "sizeId", defaultLocation, null, ImmutableMap. of(), - 1.0, 0, 0, ImagePredicates.any()); + 1.0, 0, 0, ImagePredicates.any()); Provider> locations = Providers.> of(ImmutableSet - . of(defaultLocation)); + . of(defaultLocation)); Provider> images = Providers.> of(ImmutableSet - . of(image, image2)); + . of(image, image2)); Provider> sizes = Providers.> of(ImmutableSet. of(size)); Provider optionsProvider = createMock(Provider.class); Provider templateBuilderProvider = createMock(Provider.class); TemplateBuilder defaultTemplate = createMock(TemplateBuilder.class); - expect(defaultLocation.getId()).andReturn("locationId").atLeastOnce(); expect(optionsProvider.get()).andReturn(new TemplateOptions()); expect(image.getLocation()).andReturn(defaultLocation).atLeastOnce(); @@ -135,7 +134,7 @@ public class TemplateBuilderImplTest { replay(templateBuilderProvider); TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes, defaultLocation, optionsProvider, - templateBuilderProvider); + templateBuilderProvider); assertEquals(template.smallest().architecture(Architecture.X86_32).build().getImage(), image); @@ -153,17 +152,16 @@ public class TemplateBuilderImplTest { Location defaultLocation = createMock(Location.class); Image image = createMock(Image.class); Size size = new SizeImpl("sizeId", null, "sizeId", defaultLocation, null, ImmutableMap. of(), 0, - 0, 0, ImagePredicates.idEquals("imageId")); + 0, 0, ImagePredicates.idEquals("imageId")); Provider> locations = Providers.> of(ImmutableSet - . of(defaultLocation)); + . of(defaultLocation)); Provider> images = Providers.> of(ImmutableSet. of(image)); Provider> sizes = Providers.> of(ImmutableSet. of(size)); Provider optionsProvider = createMock(Provider.class); Provider templateBuilderProvider = createMock(Provider.class); TemplateBuilder defaultTemplate = createMock(TemplateBuilder.class); - expect(defaultLocation.getId()).andReturn("locationId").atLeastOnce(); expect(optionsProvider.get()).andReturn(new TemplateOptions()); expect(image.getId()).andReturn("imageId").atLeastOnce(); expect(image.getLocation()).andReturn(defaultLocation).atLeastOnce(); @@ -181,7 +179,7 @@ public class TemplateBuilderImplTest { replay(templateBuilderProvider); TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes, defaultLocation, optionsProvider, - templateBuilderProvider); + templateBuilderProvider); template.imageId("imageId").build(); @@ -198,17 +196,16 @@ public class TemplateBuilderImplTest { Location defaultLocation = createMock(Location.class); Image image = createMock(Image.class); Size size = new SizeImpl("sizeId", null, "sizeId", defaultLocation, null, ImmutableMap. of(), 0, - 0, 0, ImagePredicates.idEquals("imageId")); + 0, 0, ImagePredicates.idEquals("imageId")); Provider> locations = Providers.> of(ImmutableSet - . of(defaultLocation)); + . of(defaultLocation)); Provider> images = Providers.> of(ImmutableSet. of(image)); Provider> sizes = Providers.> of(ImmutableSet. of(size)); Provider optionsProvider = createMock(Provider.class); Provider templateBuilderProvider = createMock(Provider.class); TemplateBuilder defaultTemplate = createMock(TemplateBuilder.class); - expect(defaultLocation.getId()).andReturn("locationId").atLeastOnce(); expect(optionsProvider.get()).andReturn(new TemplateOptions()); expect(image.getId()).andReturn("notImageId").atLeastOnce(); @@ -219,7 +216,7 @@ public class TemplateBuilderImplTest { replay(templateBuilderProvider); TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes, defaultLocation, optionsProvider, - templateBuilderProvider); + templateBuilderProvider); try { template.imageId("notImageId").build(); assert false; @@ -239,7 +236,7 @@ public class TemplateBuilderImplTest { TemplateOptions from = provideTemplateOptions(); Provider> locations = Providers - .> of(ImmutableSet. of()); + .> of(ImmutableSet. of()); Provider> images = Providers.> of(ImmutableSet. of()); Provider> sizes = Providers.> of(ImmutableSet. of()); Location defaultLocation = createMock(Location.class); @@ -258,7 +255,7 @@ public class TemplateBuilderImplTest { replay(templateBuilderProvider); TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes, defaultLocation, optionsProvider, - templateBuilderProvider); + templateBuilderProvider); template.options(options).build(); @@ -273,7 +270,7 @@ public class TemplateBuilderImplTest { public void testNothingUsesDefaultTemplateBuilder() { Provider> locations = Providers - .> of(ImmutableSet. of()); + .> of(ImmutableSet. of()); Provider> images = Providers.> of(ImmutableSet. of()); Provider> sizes = Providers.> of(ImmutableSet. of()); @@ -291,7 +288,7 @@ public class TemplateBuilderImplTest { replay(templateBuilderProvider); TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes, defaultLocation, optionsProvider, - templateBuilderProvider); + templateBuilderProvider); template.build(); @@ -302,10 +299,10 @@ public class TemplateBuilderImplTest { } protected TemplateBuilderImpl createTemplateBuilder(Provider> locations, - Provider> images, Provider> sizes, Location defaultLocation, - Provider optionsProvider, Provider templateBuilderProvider) { + Provider> images, Provider> sizes, Location defaultLocation, + Provider optionsProvider, Provider templateBuilderProvider) { TemplateBuilderImpl template = new TemplateBuilderImpl(locations, images, sizes, defaultLocation, - optionsProvider, templateBuilderProvider); + optionsProvider, templateBuilderProvider); return template; } @@ -313,7 +310,7 @@ public class TemplateBuilderImplTest { @Test public void testSuppliedLocationWithNoOptions() { Provider> locations = Providers - .> of(ImmutableSet. of()); + .> of(ImmutableSet. of()); Provider> images = Providers.> of(ImmutableSet. of()); Provider> sizes = Providers.> of(ImmutableSet. of()); Location defaultLocation = createMock(Location.class); @@ -321,15 +318,13 @@ public class TemplateBuilderImplTest { Provider templateBuilderProvider = createMock(Provider.class); TemplateOptions defaultOptions = createMock(TemplateOptions.class); - expect(optionsProvider.get()).andReturn(defaultOptions); - replay(defaultOptions); replay(defaultLocation); replay(optionsProvider); replay(templateBuilderProvider); TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes, defaultLocation, optionsProvider, - templateBuilderProvider); + templateBuilderProvider); try { template.imageId("foo").locationId("location").build(); @@ -350,7 +345,7 @@ public class TemplateBuilderImplTest { TemplateOptions from = provideTemplateOptions(); Provider> locations = Providers - .> of(ImmutableSet. of()); + .> of(ImmutableSet. of()); Provider> images = Providers.> of(ImmutableSet. of()); Provider> sizes = Providers.> of(ImmutableSet. of()); Location defaultLocation = createMock(Location.class); @@ -364,7 +359,7 @@ public class TemplateBuilderImplTest { replay(templateBuilderProvider); TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes, defaultLocation, optionsProvider, - templateBuilderProvider); + templateBuilderProvider); try { template.imageId("foo").options(provideTemplateOptions()).locationId("location").build(); @@ -382,7 +377,7 @@ public class TemplateBuilderImplTest { @Test public void testDefaultLocationWithNoOptionsNoSuchElement() { Provider> locations = Providers - .> of(ImmutableSet. of()); + .> of(ImmutableSet. of()); Provider> images = Providers.> of(ImmutableSet. of()); Provider> sizes = Providers.> of(ImmutableSet. of()); Location defaultLocation = createMock(Location.class); @@ -390,7 +385,6 @@ public class TemplateBuilderImplTest { Provider templateBuilderProvider = createMock(Provider.class); TemplateOptions defaultOptions = createMock(TemplateOptions.class); - expect(defaultLocation.getId()).andReturn("foo"); expect(optionsProvider.get()).andReturn(defaultOptions); replay(defaultOptions); @@ -399,10 +393,10 @@ public class TemplateBuilderImplTest { replay(templateBuilderProvider); TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes, defaultLocation, optionsProvider, - templateBuilderProvider); + templateBuilderProvider); try { - template.imageId("foo").build(); + template.imageId("region/ami").build(); assert false; } catch (NoSuchElementException e) { @@ -422,7 +416,7 @@ public class TemplateBuilderImplTest { @Test public void testDefaultLocationWithOptions() { Provider> locations = Providers - .> of(ImmutableSet. of()); + .> of(ImmutableSet. of()); Provider> images = Providers.> of(ImmutableSet. of()); Provider> sizes = Providers.> of(ImmutableSet. of()); Location defaultLocation = createMock(Location.class); @@ -430,28 +424,19 @@ public class TemplateBuilderImplTest { TemplateOptions from = provideTemplateOptions(); Provider templateBuilderProvider = createMock(Provider.class); - expect(defaultLocation.getId()).andReturn("foo"); - // expect(defaultLocation.getId()).andReturn("foo"); expect(optionsProvider.get()).andReturn(from); - // expect(optionsProvider.get()).andReturn(provideTemplateOptions()); expect(from.getInboundPorts()).andReturn(new int[] { 22 }); - // expect(from.getRunScript()).andReturn(null); - // expect(from.getPrivateKey()).andReturn(null); - // expect(from.getPublicKey()).andReturn(null); - // expect(from.getPort()).andReturn(null); - // expect(from.isIncludeMetadata()).andReturn(false); - // expect(from.shouldBlockUntilRunning()).andReturn(true); replay(defaultLocation); replay(optionsProvider); replay(templateBuilderProvider); TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes, defaultLocation, optionsProvider, - templateBuilderProvider); + templateBuilderProvider); try { - template.imageId("foo").options(provideTemplateOptions()).build(); + template.imageId("region/ami").options(provideTemplateOptions()).build(); assert false; } catch (NoSuchElementException e) { @@ -466,7 +451,7 @@ public class TemplateBuilderImplTest { @Test public void testImageIdNullsEverythingElse() { Provider> locations = Providers - .> of(ImmutableSet. of()); + .> of(ImmutableSet. of()); Provider> images = Providers.> of(ImmutableSet. of()); Provider> sizes = Providers.> of(ImmutableSet. of()); Location defaultLocation = createMock(Location.class); @@ -478,7 +463,7 @@ public class TemplateBuilderImplTest { replay(templateBuilderProvider); TemplateBuilderImpl template = createTemplateBuilder(locations, images, sizes, defaultLocation, optionsProvider, - templateBuilderProvider); + templateBuilderProvider); template.architecture(Architecture.X86_32); template.imageDescriptionMatches("imageDescriptionMatches"); diff --git a/core/src/main/java/org/jclouds/concurrent/ConcurrentUtils.java b/core/src/main/java/org/jclouds/concurrent/ConcurrentUtils.java index b24bc59aa2..c9ea54cad8 100644 --- a/core/src/main/java/org/jclouds/concurrent/ConcurrentUtils.java +++ b/core/src/main/java/org/jclouds/concurrent/ConcurrentUtils.java @@ -18,41 +18,111 @@ */ package org.jclouds.concurrent; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.Maps.newHashMap; + +import java.util.Collections; +import java.util.List; import java.util.Map; -import java.util.Map.Entry; +import java.util.concurrent.AbstractExecutorService; import java.util.concurrent.CancellationException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import javax.annotation.Nullable; +import javax.annotation.Resource; +import javax.inject.Named; import javax.inject.Singleton; +import org.jclouds.Constants; +import org.jclouds.http.handlers.BackoffLimitedRetryHandler; import org.jclouds.logging.Logger; +import com.google.common.base.Function; import com.google.common.base.Throwables; +import com.google.common.collect.ForwardingObject; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ExecutionList; import com.google.common.util.concurrent.ForwardingFuture; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.inject.Inject; /** - * Adapt things missing from Guava. + * Adapt things from Guava. * * @author Adrian Cole */ @Singleton public class ConcurrentUtils { + @Resource + private static Logger logger = Logger.CONSOLE; - public static Map awaitCompletion(Map> responses, - ExecutorService executor, @Nullable Long maxTime, final Logger logger, final String logPrefix) { + @Inject(optional = true) + @Named(Constants.PROPERTY_MAX_RETRIES) + private static int maxRetries = 5; + + @Inject(optional = true) + private static BackoffLimitedRetryHandler retryHandler = BackoffLimitedRetryHandler.INSTANCE; + + public static Iterable transformParallel(final Iterable fromIterable, + final Function> function) { + return transformParallel(fromIterable, function, sameThreadExecutor(), null); + } + + public static Iterable transformParallel(final Iterable fromIterable, + final Function> function, ExecutorService exec, @Nullable Long maxTime) { + return transformParallel(fromIterable, function, exec, maxTime, logger, "transforming"); + } + + public static Iterable transformParallel(final Iterable fromIterable, + final Function> function, ExecutorService exec, @Nullable Long maxTime, Logger logger, + String logPrefix) { + return transformParallel(fromIterable, function, exec, maxTime, logger, logPrefix, retryHandler, maxRetries); + } + + public static Iterable transformParallel(Iterable fromIterable, + Function> function, ExecutorService exec, @Nullable Long maxTime, Logger logger, + String logPrefix, BackoffLimitedRetryHandler retryHandler, int maxRetries) { + Map exceptions = newHashMap(); + Map> responses = newHashMap(); + for (int i = 0; i < maxRetries; i++) { + + for (F from : fromIterable) { + responses.put(from, function.apply(from)); + } + exceptions = awaitCompletion(responses, exec, maxTime, logger, logPrefix); + if (exceptions.size() > 0) { + fromIterable = exceptions.keySet(); + retryHandler.imposeBackoffExponentialDelay(i + 1, String.format("error %s: %s: %s", logPrefix, + fromIterable, exceptions)); + } else { + break; + } + } + if (exceptions.size() > 0) + throw new RuntimeException(String.format("error %s: %s: %s", logPrefix, fromIterable, exceptions)); + + return unwrap(responses.values()); + } + + public static Map awaitCompletion(Map> responses, ExecutorService exec, + @Nullable Long maxTime, final Logger logger, final String logPrefix) { if (responses.size() == 0) return ImmutableMap.of(); final int total = responses.size(); @@ -61,8 +131,8 @@ public class ConcurrentUtils { final AtomicInteger errors = new AtomicInteger(0); final long start = System.currentTimeMillis(); final Map errorMap = Maps.newHashMap(); - for (final Entry> future : responses.entrySet()) { - future.getValue().addListener(new Runnable() { + for (final java.util.Map.Entry> future : responses.entrySet()) { + makeListenable(future.getValue(), exec).addListener(new Runnable() { public void run() { try { future.getValue().get(); @@ -74,7 +144,7 @@ public class ConcurrentUtils { } doneSignal.countDown(); } - }, executor); + }, exec); } try { if (maxTime != null) @@ -99,6 +169,22 @@ public class ConcurrentUtils { return errorMap; } + public static Iterable unwrap(Iterable> values) { + return Iterables.transform(values, new Function, T>() { + @Override + public T apply(Future from) { + try { + return from.get(); + } catch (InterruptedException e) { + Throwables.propagate(e); + } catch (ExecutionException e) { + Throwables.propagate(e); + } + return null; + } + }); + } + private static void logException(Logger logger, String logPrefix, int total, int complete, int errors, long start, Exception e) { String message = message(logPrefix, total, complete, errors, start); @@ -123,49 +209,55 @@ public class ConcurrentUtils { if (future instanceof ListenableFuture) { return (ListenableFuture) future; } - return new ListenableFutureAdapter(executorService, future); + return ListenableFutureAdapter.create(future, executorService); } /** - * Just like {@code Futures#ListenableFutureAdapter} except that we pass in an executorService. - *

- * Temporary hack until http://code.google.com/p/guava-libraries/issues/detail?id=317 is fixed. + * Just like {@code Futures#compose} except that we check the type of the executorService before + * creating the Future. If we are single threaded, invoke the function lazy as opposed to + * chaining, so that we don't invoke get() early. */ - private static class ListenableFutureAdapter extends ForwardingFuture implements ListenableFuture { + public static ListenableFuture compose(Future future, final Function function, + ExecutorService executorService) { + if (future instanceof ListenableFutureAdapter) { + ListenableFutureAdapter lf = (ListenableFutureAdapter) future; + if (lf.futureListener.executor.getClass().isAnnotationPresent(SingleThreaded.class)) + return LazyListenableFutureFunctionAdapter.create(((ListenableFutureAdapter) future).futureListener, + function); + else + return Futures.compose(lf, function, executorService); + } else if (executorService.getClass().isAnnotationPresent(SingleThreaded.class)) { + return LazyListenableFutureFunctionAdapter.create(future, function, executorService); + } else { + return Futures.compose(makeListenable(future, executorService), function, executorService); + } + } - private final Executor adapterExecutor; - - // The execution list to hold our listeners. + public static class FutureListener { + private final Future future; + private final ExecutorService executor; private final ExecutionList executionList = new ExecutionList(); - - // This allows us to only start up a thread waiting on the delegate future - // when the first listener is added. private final AtomicBoolean hasListeners = new AtomicBoolean(false); - // The delegate future. - private final Future delegate; - - ListenableFutureAdapter(ExecutorService executorService, final Future delegate) { - this.adapterExecutor = executorService; - this.delegate = delegate; + static FutureListener create(Future future, ExecutorService executor) { + return new FutureListener(future, executor); } - @Override - protected Future delegate() { - return delegate; + private FutureListener(Future future, ExecutorService executor) { + this.future = checkNotNull(future, "future"); + this.executor = checkNotNull(executor, "executor"); } - /* @Override */ public void addListener(Runnable listener, Executor exec) { // When a listener is first added, we run a task that will wait for - // the delegate to finish, and when it is done will run the listeners. + // the future to finish, and when it is done will run the listeners. if (!hasListeners.get() && hasListeners.compareAndSet(false, true)) { - adapterExecutor.execute(new Runnable() { + executor.execute(new Runnable() { /* @Override */ public void run() { try { - delegate.get(); + future.get(); } catch (CancellationException e) { // The task was cancelled, so it is done, run the listeners. } catch (InterruptedException e) { @@ -181,5 +273,289 @@ public class ConcurrentUtils { } executionList.add(listener, exec); } + + Future getFuture() { + return future; + } + + ExecutorService getExecutor() { + return executor; + } } + + public static class ListenableFutureAdapter extends ForwardingFuture implements ListenableFuture { + private final FutureListener futureListener; + + static ListenableFutureAdapter create(Future future, ExecutorService executor) { + return new ListenableFutureAdapter(future, executor); + } + + private ListenableFutureAdapter(Future future, ExecutorService executor) { + this.futureListener = FutureListener.create(future, executor); + } + + @Override + protected Future delegate() { + return futureListener.getFuture(); + } + + @Override + public void addListener(Runnable listener, Executor exec) { + futureListener.addListener(listener, exec); + } + + } + + public static class LazyListenableFutureFunctionAdapter extends ForwardingObject implements + ListenableFuture { + private final FutureListener futureListener; + private final Function function; + + static LazyListenableFutureFunctionAdapter create(Future future, + Function function, ExecutorService executor) { + return new LazyListenableFutureFunctionAdapter(future, function, executor); + } + + static LazyListenableFutureFunctionAdapter create(FutureListener futureListener, + Function function) { + return new LazyListenableFutureFunctionAdapter(futureListener, function); + } + + private LazyListenableFutureFunctionAdapter(Future future, Function function, + ExecutorService executor) { + this(FutureListener.create(future, executor), function); + } + + private LazyListenableFutureFunctionAdapter(FutureListener futureListener, + Function function) { + this.futureListener = checkNotNull(futureListener, "futureListener"); + this.function = checkNotNull(function, "function"); + } + + /* + * Concurrency detail: + * + *

To preserve the idempotency of calls to this.get(*) calls to the function are only + * applied once. A lock is required to prevent multiple applications of the function. The + * calls to future.get(*) are performed outside the lock, as is required to prevent calls to + * get(long, TimeUnit) to persist beyond their timeout. + * + *

Calls to future.get(*) on every call to this.get(*) also provide the cancellation + * behavior for this. + * + *

(Consider: in thread A, call get(), in thread B call get(long, TimeUnit). Thread B may + * have to wait for Thread A to finish, which would be unacceptable.) + * + *

Note that each call to Future.get(*) results in a call to Future.get(*), but the + * function is only applied once, so Future.get(*) is assumed to be idempotent. + */ + + private final Object lock = new Object(); + private boolean set = false; + private O value = null; + + @Override + public O get() throws InterruptedException, ExecutionException { + return apply(futureListener.getFuture().get()); + } + + @Override + public O get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + return apply(futureListener.getFuture().get(timeout, unit)); + } + + private O apply(I raw) { + synchronized (lock) { + if (!set) { + value = function.apply(raw); + set = true; + } + return value; + } + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return futureListener.getFuture().cancel(mayInterruptIfRunning); + } + + @Override + public boolean isCancelled() { + return futureListener.getFuture().isCancelled(); + } + + @Override + public boolean isDone() { + return futureListener.getFuture().isDone(); + } + + @Override + public void addListener(Runnable listener, Executor exec) { + futureListener.addListener(listener, exec); + } + + @Override + protected Object delegate() { + return futureListener.getFuture(); + } + + } + + /** + * Taken from {@link MoreExecutors#sameThreadExecutor} as it was hidden and therefore incapable + * of instanceof checks. + * + * + * + * Creates an executor service that runs each task in the thread that invokes {@code + * execute/submit}, as in {@link CallerRunsPolicy} This applies both to individually submitted + * tasks and to collections of tasks submitted via {@code invokeAll} or {@code invokeAny}. In the + * latter case, tasks will run serially on the calling thread. Tasks are run to completion before + * a {@code Future} is returned to the caller (unless the executor has been shutdown). + * + *

+ * Although all tasks are immediately executed in the thread that submitted the task, this + * {@code ExecutorService} imposes a small locking overhead on each task submission in order to + * implement shutdown and termination behavior. + * + *

+ * The implementation deviates from the {@code ExecutorService} specification with regards to the + * {@code shutdownNow} method. First, "best-effort" with regards to canceling running tasks is + * implemented as "no-effort". No interrupts or other attempts are made to stop threads executing + * tasks. Second, the returned list will always be empty, as any submitted task is considered to + * have started execution. This applies also to tasks given to {@code invokeAll} or {@code + * invokeAny} which are pending serial execution, even the subset of the tasks that have not yet + * started execution. It is unclear from the {@code ExecutorService} specification if these + * should be included, and it's much easier to implement the interpretation that they not be. + * Finally, a call to {@code shutdown} or {@code shutdownNow} may result in concurrent calls to + * {@code invokeAll/invokeAny} throwing RejectedExecutionException, although a subset of the + * tasks may already have been executed. + */ + public static ExecutorService sameThreadExecutor() { + return new SameThreadExecutorService(); + } + + // See sameThreadExecutor javadoc for behavioral notes. + @SingleThreaded + public static class SameThreadExecutorService extends AbstractExecutorService { + /** + * Lock used whenever accessing the state variables (runningTasks, shutdown, + * terminationCondition) of the executor + */ + private final Lock lock = new ReentrantLock(); + + /** Signaled after the executor is shutdown and running tasks are done */ + private final Condition termination = lock.newCondition(); + + private SameThreadExecutorService() { + } + + /* + * Conceptually, these two variables describe the executor being in one of three states: - + * Active: shutdown == false - Shutdown: runningTasks > 0 and shutdown == true - Terminated: + * runningTasks == 0 and shutdown == true + */ + private int runningTasks = 0; + private boolean shutdown = false; + + @Override + public void execute(Runnable command) { + startTask(); + try { + command.run(); + } finally { + endTask(); + } + } + + @Override + public boolean isShutdown() { + lock.lock(); + try { + return shutdown; + } finally { + lock.unlock(); + } + } + + @Override + public void shutdown() { + lock.lock(); + try { + shutdown = true; + } finally { + lock.unlock(); + } + } + + // See sameThreadExecutor javadoc for unusual behavior of this method. + @Override + public List shutdownNow() { + shutdown(); + return Collections.emptyList(); + } + + @Override + public boolean isTerminated() { + lock.lock(); + try { + return shutdown && runningTasks == 0; + } finally { + lock.unlock(); + } + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + long nanos = unit.toNanos(timeout); + lock.lock(); + try { + for (;;) { + if (isTerminated()) { + return true; + } else if (nanos <= 0) { + return false; + } else { + nanos = termination.awaitNanos(nanos); + } + } + } finally { + lock.unlock(); + } + } + + /** + * Checks if the executor has been shut down and increments the running task count. + * + * @throws RejectedExecutionException + * if the executor has been previously shutdown + */ + private void startTask() { + lock.lock(); + try { + if (isShutdown()) { + throw new RejectedExecutionException("Executor already shutdown"); + } + runningTasks++; + } finally { + lock.unlock(); + } + } + + /** + * Decrements the running task count. + */ + private void endTask() { + lock.lock(); + try { + runningTasks--; + if (isTerminated()) { + termination.signalAll(); + } + } finally { + lock.unlock(); + } + } + } + } \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/concurrent/FutureExceptionParser.java b/core/src/main/java/org/jclouds/concurrent/ExceptionParsingListenableFuture.java similarity index 68% rename from core/src/main/java/org/jclouds/concurrent/FutureExceptionParser.java rename to core/src/main/java/org/jclouds/concurrent/ExceptionParsingListenableFuture.java index ffeb9876d4..f53768c261 100755 --- a/core/src/main/java/org/jclouds/concurrent/FutureExceptionParser.java +++ b/core/src/main/java/org/jclouds/concurrent/ExceptionParsingListenableFuture.java @@ -18,6 +18,8 @@ */ package org.jclouds.concurrent; +import static com.google.common.base.Preconditions.checkNotNull; + import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; @@ -27,30 +29,35 @@ import com.google.common.base.Function; import com.google.common.util.concurrent.ListenableFuture; /** - * Transforms the result of a future as soon as it is available. + * Transforms the exceptions in a future upon get * * Temporarily here until the following is resolved: guava issue 310 * * @author Adrian Cole */ -public class FutureExceptionParser implements ListenableFuture { +public class ExceptionParsingListenableFuture implements ListenableFuture { - private final ListenableFuture delegate; + private final ListenableFuture future; private final Function function; - public FutureExceptionParser(ListenableFuture delegate, Function function) { - this.delegate = delegate; - this.function = function; + public static ExceptionParsingListenableFuture create(ListenableFuture future, + Function function) { + return new ExceptionParsingListenableFuture(future, function); + } + + public ExceptionParsingListenableFuture(ListenableFuture future, Function function) { + this.future = checkNotNull(future); + this.function = checkNotNull(function); } public boolean cancel(boolean mayInterruptIfRunning) { - return delegate.cancel(mayInterruptIfRunning); + return future.cancel(mayInterruptIfRunning); } public T get() throws InterruptedException, ExecutionException { try { - return delegate.get(); + return future.get(); } catch (Exception e) { return attemptConvert(e); } @@ -60,26 +67,24 @@ public class FutureExceptionParser implements ListenableFuture { return function.apply(e instanceof ExecutionException ? (Exception) e.getCause() : e); } - public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, - TimeoutException { + public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { try { - return delegate.get(timeout, unit); + return future.get(timeout, unit); } catch (Exception e) { return attemptConvert(e); } } public boolean isCancelled() { - return delegate.isCancelled(); + return future.isCancelled(); } public boolean isDone() { - return delegate.isDone(); + return future.isDone(); } @Override public void addListener(Runnable listener, Executor exec) { - delegate.addListener(listener, exec); + future.addListener(listener, exec); } - } diff --git a/core/src/main/java/org/jclouds/concurrent/RetryOnTimeOutExceptionSupplier.java b/core/src/main/java/org/jclouds/concurrent/RetryOnTimeOutExceptionSupplier.java index f4c6a4e1b4..04372f85da 100644 --- a/core/src/main/java/org/jclouds/concurrent/RetryOnTimeOutExceptionSupplier.java +++ b/core/src/main/java/org/jclouds/concurrent/RetryOnTimeOutExceptionSupplier.java @@ -18,13 +18,12 @@ */ package org.jclouds.concurrent; +import static com.google.common.base.Throwables.propagate; +import static org.jclouds.util.Utils.getFirstThrowableOfType; + import java.util.concurrent.TimeoutException; -import org.jclouds.util.Utils; - import com.google.common.base.Supplier; -import com.google.common.base.Throwables; - /** * * @author Adrian Cole @@ -44,15 +43,15 @@ public class RetryOnTimeOutExceptionSupplier implements Supplier { ex = null; return delegate.get(); } catch (Exception e) { - if ((ex = Utils.getFirstThrowableOfType(e, TimeoutException.class)) != null) + if ((ex = getFirstThrowableOfType(e, TimeoutException.class)) != null) continue; - Throwables.propagate(e); + propagate(e); assert false; return null; } } if (ex != null) - Throwables.propagate(ex); + propagate(ex); assert false; return null; } diff --git a/core/src/main/java/org/jclouds/concurrent/config/ExecutorServiceModule.java b/core/src/main/java/org/jclouds/concurrent/config/ExecutorServiceModule.java index f3a26e2385..6a7decf723 100644 --- a/core/src/main/java/org/jclouds/concurrent/config/ExecutorServiceModule.java +++ b/core/src/main/java/org/jclouds/concurrent/config/ExecutorServiceModule.java @@ -32,6 +32,8 @@ import javax.inject.Named; import javax.inject.Singleton; import org.jclouds.Constants; +import static org.jclouds.concurrent.ConcurrentUtils.*; +import org.jclouds.concurrent.SingleThreaded; import org.jclouds.lifecycle.Closer; import org.jclouds.logging.Logger; @@ -65,8 +67,7 @@ public class ExecutorServiceModule extends AbstractModule { public void close() throws IOException { List runnables = service.shutdownNow(); if (runnables.size() > 0) - logger.warn("when shutting down executor %s, runnables outstanding: %s", service, - runnables); + logger.warn("when shutting down executor %s, runnables outstanding: %s", service, runnables); } } @@ -74,11 +75,22 @@ public class ExecutorServiceModule extends AbstractModule { private final ExecutorService ioExecutorFromConstructor; @Inject - public ExecutorServiceModule( - @Named(Constants.PROPERTY_USER_THREADS) ExecutorService userThreads, + public ExecutorServiceModule(@Named(Constants.PROPERTY_USER_THREADS) ExecutorService userThreads, @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioThreads) { - this.userExecutorFromConstructor = userThreads; - this.ioExecutorFromConstructor = ioThreads; + this.userExecutorFromConstructor = checkNotGuavaSameThreadExecutor(userThreads); + this.ioExecutorFromConstructor = checkNotGuavaSameThreadExecutor(ioThreads); + } + + private ExecutorService checkNotGuavaSameThreadExecutor(ExecutorService executor) { + // we detect behavior based on the class + if (executor != null && !(executor.getClass().isAnnotationPresent(SingleThreaded.class)) + && executor.getClass().getSimpleName().indexOf("SameThread") != -1) { + Logger.CONSOLE.warn( + "please switch from %s to %s or annotate your same threaded executor with @SingleThreaded", executor + .getClass().getName(), SameThreadExecutorService.class.getName()); + return sameThreadExecutor(); + } + return executor; } public ExecutorServiceModule() { @@ -92,8 +104,7 @@ public class ExecutorServiceModule extends AbstractModule { @Provides @Singleton @Named(Constants.PROPERTY_USER_THREADS) - ExecutorService provideExecutorService(@Named(Constants.PROPERTY_USER_THREADS) int count, - Closer closer) { + ExecutorService provideExecutorService(@Named(Constants.PROPERTY_USER_THREADS) int count, Closer closer) { if (userExecutorFromConstructor != null) return shutdownOnClose(userExecutorFromConstructor, closer); return shutdownOnClose(newThreadPoolNamed("user thread %d", count), closer); @@ -102,8 +113,7 @@ public class ExecutorServiceModule extends AbstractModule { @Provides @Singleton @Named(Constants.PROPERTY_IO_WORKER_THREADS) - ExecutorService provideIOExecutor(@Named(Constants.PROPERTY_IO_WORKER_THREADS) int count, - Closer closer) { + ExecutorService provideIOExecutor(@Named(Constants.PROPERTY_IO_WORKER_THREADS) int count, Closer closer) { if (ioExecutorFromConstructor != null) return shutdownOnClose(ioExecutorFromConstructor, closer); return shutdownOnClose(newThreadPoolNamed("i/o thread %d", count), closer); @@ -117,20 +127,19 @@ public class ExecutorServiceModule extends AbstractModule { @VisibleForTesting static ExecutorService newCachedThreadPoolNamed(String name) { - return Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat(name) - .setThreadFactory(Executors.defaultThreadFactory()).build()); + return Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat(name).setThreadFactory( + Executors.defaultThreadFactory()).build()); } @VisibleForTesting static ExecutorService newThreadPoolNamed(String name, int maxCount) { - return maxCount == 0 ? newCachedThreadPoolNamed(name) : newScalingThreadPoolNamed(name, - maxCount); + return maxCount == 0 ? newCachedThreadPoolNamed(name) : newScalingThreadPoolNamed(name, maxCount); } @VisibleForTesting static ExecutorService newScalingThreadPoolNamed(String name, int maxCount) { - return newScalingThreadPool(1, maxCount, 60L * 1000, new ThreadFactoryBuilder() - .setNameFormat(name).setThreadFactory(Executors.defaultThreadFactory()).build()); + return newScalingThreadPool(1, maxCount, 60L * 1000, new ThreadFactoryBuilder().setNameFormat(name) + .setThreadFactory(Executors.defaultThreadFactory()).build()); } } \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/http/HttpCommandExecutorService.java b/core/src/main/java/org/jclouds/http/HttpCommandExecutorService.java index cf66d1293d..b724da668d 100644 --- a/core/src/main/java/org/jclouds/http/HttpCommandExecutorService.java +++ b/core/src/main/java/org/jclouds/http/HttpCommandExecutorService.java @@ -18,7 +18,7 @@ */ package org.jclouds.http; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; /** * Capable of invoking http commands. @@ -33,7 +33,7 @@ public interface HttpCommandExecutorService { * * @param command * that generates requests - * @return {@link ListenableFuture} containing the response from the {@code endpoint} + * @return {@link Future} containing the response from the {@code endpoint} */ - ListenableFuture submit(HttpCommand command); + Future submit(HttpCommand command); } diff --git a/core/src/main/java/org/jclouds/http/HttpCommandRendezvous.java b/core/src/main/java/org/jclouds/http/HttpCommandRendezvous.java index ad81c40eed..6a533d4f19 100644 --- a/core/src/main/java/org/jclouds/http/HttpCommandRendezvous.java +++ b/core/src/main/java/org/jclouds/http/HttpCommandRendezvous.java @@ -20,7 +20,7 @@ package org.jclouds.http; import java.util.concurrent.SynchronousQueue; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; /** * Used for passing objects for response processing @@ -32,11 +32,11 @@ public class HttpCommandRendezvous { private final HttpCommand command; @SuppressWarnings("unchecked") private final SynchronousQueue rendezvous; - private final ListenableFuture future; + private final Future future; @SuppressWarnings("unchecked") public HttpCommandRendezvous(HttpCommand command, SynchronousQueue rendezvous, - ListenableFuture future) { + Future future) { this.command = command; this.rendezvous = rendezvous; this.future = future; @@ -53,14 +53,14 @@ public class HttpCommandRendezvous { } public void cancel() { - getListenableFuture().cancel(true); + getFuture().cancel(true); } public HttpCommand getCommand() { return command; } - public ListenableFuture getListenableFuture() { + public Future getFuture() { return future; } diff --git a/core/src/main/java/org/jclouds/http/TransformingHttpCommandExecutorService.java b/core/src/main/java/org/jclouds/http/TransformingHttpCommandExecutorService.java index a65ceacad1..1b2c6a0c11 100644 --- a/core/src/main/java/org/jclouds/http/TransformingHttpCommandExecutorService.java +++ b/core/src/main/java/org/jclouds/http/TransformingHttpCommandExecutorService.java @@ -18,6 +18,8 @@ */ package org.jclouds.http; +import java.util.concurrent.Future; + import com.google.common.base.Function; import com.google.common.util.concurrent.ListenableFuture; @@ -30,7 +32,7 @@ import com.google.common.util.concurrent.ListenableFuture; public interface TransformingHttpCommandExecutorService { /** * - * Submits the command and transforms the result before requested via {@link ListenableFuture#get()}. + * Submits the command and transforms the result before requested via {@link Future#get()}. * * @param * type that is required from the value. diff --git a/core/src/main/java/org/jclouds/http/TransformingHttpCommandExecutorServiceImpl.java b/core/src/main/java/org/jclouds/http/TransformingHttpCommandExecutorServiceImpl.java index 8e723bc6db..59daebb1d7 100644 --- a/core/src/main/java/org/jclouds/http/TransformingHttpCommandExecutorServiceImpl.java +++ b/core/src/main/java/org/jclouds/http/TransformingHttpCommandExecutorServiceImpl.java @@ -18,10 +18,13 @@ */ package org.jclouds.http; -import static com.google.common.util.concurrent.Futures.compose; -import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; +import java.util.concurrent.ExecutorService; import javax.inject.Inject; +import javax.inject.Named; + +import org.jclouds.Constants; +import static org.jclouds.concurrent.ConcurrentUtils.*; import com.google.common.base.Function; import com.google.common.util.concurrent.ListenableFuture; @@ -32,21 +35,22 @@ import com.google.common.util.concurrent.ListenableFuture; * * @author Adrian Cole */ -public class TransformingHttpCommandExecutorServiceImpl implements - TransformingHttpCommandExecutorService { +public class TransformingHttpCommandExecutorServiceImpl implements TransformingHttpCommandExecutorService { private final HttpCommandExecutorService client; + private final ExecutorService userThreads; @Inject - public TransformingHttpCommandExecutorServiceImpl(HttpCommandExecutorService client) { + public TransformingHttpCommandExecutorServiceImpl(HttpCommandExecutorService client, + @Named(Constants.PROPERTY_USER_THREADS) ExecutorService userThreads) { this.client = client; + this.userThreads = userThreads; } /** * {@inheritDoc} */ - public ListenableFuture submit(HttpCommand command, - Function responseTransformer) { - return compose(client.submit(command), responseTransformer, sameThreadExecutor()); + public ListenableFuture submit(HttpCommand command, Function responseTransformer) { + return compose(client.submit(command), responseTransformer, userThreads); } } diff --git a/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java b/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java index 24d72ed6b3..4b6f04e6ce 100644 --- a/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java +++ b/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java @@ -79,6 +79,9 @@ import com.google.inject.Inject; */ @Singleton public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOExceptionRetryHandler { + + public static BackoffLimitedRetryHandler INSTANCE = new BackoffLimitedRetryHandler(); + @Inject(optional = true) @Named(Constants.PROPERTY_MAX_RETRIES) private int retryCountLimit = 5; @@ -102,13 +105,11 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException logger.warn("Cannot retry after server error, command is not replayable: %1$s", command); return false; } else if (command.getFailureCount() > retryCountLimit) { - logger.warn( - "Cannot retry after server error, command has exceeded retry limit %1$d: %2$s", - retryCountLimit, command); + logger.warn("Cannot retry after server error, command has exceeded retry limit %1$d: %2$s", retryCountLimit, + command); return false; } else { - imposeBackoffExponentialDelay(command.getFailureCount(), "server error: " - + command.toString()); + imposeBackoffExponentialDelay(command.getFailureCount(), "server error: " + command.toString()); return true; } } @@ -117,11 +118,9 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException imposeBackoffExponentialDelay(50L, 2, failureCount, retryCountLimit, commandDescription); } - public void imposeBackoffExponentialDelay(long period, int pow, int failureCount, int max, - String commandDescription) { + public void imposeBackoffExponentialDelay(long period, int pow, int failureCount, int max, String commandDescription) { long delayMs = (long) (period * Math.pow(failureCount, pow)); - logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, retryCountLimit, delayMs, - commandDescription); + logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, retryCountLimit, delayMs, commandDescription); try { Thread.sleep(delayMs); } catch (InterruptedException e) { diff --git a/core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java b/core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java index 012a088738..785c241beb 100644 --- a/core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java +++ b/core/src/main/java/org/jclouds/http/internal/BaseHttpCommandExecutorService.java @@ -19,7 +19,6 @@ package org.jclouds.http.internal; import static com.google.common.io.ByteStreams.copy; -import static org.jclouds.concurrent.ConcurrentUtils.makeListenable; import static org.jclouds.http.HttpUtils.checkRequestHasContentLengthOrChunkedEncoding; import static org.jclouds.http.HttpUtils.wirePayloadIfEnabled; import static org.jclouds.util.Utils.getFirstThrowableOfType; @@ -29,6 +28,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import javax.annotation.Resource; import javax.inject.Inject; @@ -49,7 +49,6 @@ import org.jclouds.http.handlers.DelegatingRetryHandler; import org.jclouds.logging.Logger; import com.google.common.io.NullOutputStream; -import com.google.common.util.concurrent.ListenableFuture; /** * @@ -74,9 +73,9 @@ public abstract class BaseHttpCommandExecutorService implements HttpCommandEx @Inject protected BaseHttpCommandExecutorService(HttpUtils utils, EncryptionService encryptionService, - @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor, - DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler, - DelegatingErrorHandler errorHandler, HttpWire wire) { + @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor, + DelegatingRetryHandler retryHandler, IOExceptionRetryHandler ioRetryHandler, + DelegatingErrorHandler errorHandler, HttpWire wire) { this.utils = utils; this.encryptionService = encryptionService; this.retryHandler = retryHandler; @@ -123,11 +122,12 @@ public abstract class BaseHttpCommandExecutorService implements HttpCommandEx } - public ListenableFuture submit(HttpCommand command) { + public Future submit(HttpCommand command) { HttpRequest request = command.getRequest(); checkRequestHasContentLengthOrChunkedEncoding(request, - "if the request has a payload, it must be set to chunked encoding or specify a content length: " + request); - return makeListenable(ioWorkerExecutor.submit(new HttpResponseCallable(command)), ioWorkerExecutor); + "if the request has a payload, it must be set to chunked encoding or specify a content length: " + + request); + return ioWorkerExecutor.submit(new HttpResponseCallable(command)); } public class HttpResponseCallable implements Callable { @@ -148,7 +148,7 @@ public abstract class BaseHttpCommandExecutorService implements HttpCommandEx filter.filter(request); } checkRequestHasContentLengthOrChunkedEncoding(request, - "After filtering, the request has niether chunked encoding nor content length: " + request); + "After filtering, the request has niether chunked encoding nor content length: " + request); logger.debug("Sending request %s: %s", request.hashCode(), request.getRequestLine()); wirePayloadIfEnabled(wire, request); nativeRequest = convert(request); @@ -174,7 +174,7 @@ public abstract class BaseHttpCommandExecutorService implements HttpCommandEx continue; } else { command.setException(new HttpResponseException(e.getMessage() + " connecting to " - + command.getRequest().getRequestLine(), command, null, e)); + + command.getRequest().getRequestLine(), command, null, e)); break; } } finally { diff --git a/core/src/main/java/org/jclouds/json/Json.java b/core/src/main/java/org/jclouds/json/Json.java index 9709140b33..cf0ef032d8 100644 --- a/core/src/main/java/org/jclouds/json/Json.java +++ b/core/src/main/java/org/jclouds/json/Json.java @@ -25,13 +25,20 @@ import java.lang.reflect.Type; */ public interface Json { /** - * Serialize the object into json. + * Serialize the object into json. If the object is a generic type, use + * {@link #toJson(Object, Type)} */ String toJson(Object src); /** - * Deserialize the generic object from json. If the object is not a generic - * type, use {@link #fromJson(Object, Class)} + * Serialize the generic object into json. If the object is not a generic, use + * {@link #toJson(Object, Type)} + */ + String toJson(Object src, Type type); + + /** + * Deserialize the generic object from json. If the object is not a generic type, use + * {@link #fromJson(Object, Class)} */ T fromJson(String json, Type type); diff --git a/core/src/main/java/org/jclouds/json/internal/GsonWrapper.java b/core/src/main/java/org/jclouds/json/internal/GsonWrapper.java index 553c563576..397f8e2ddd 100644 --- a/core/src/main/java/org/jclouds/json/internal/GsonWrapper.java +++ b/core/src/main/java/org/jclouds/json/internal/GsonWrapper.java @@ -56,4 +56,9 @@ public class GsonWrapper implements Json { return gson.toJson(src); } + @Override + public String toJson(Object src, Type type) { + return gson.toJson(src, type); + } + } \ No newline at end of file diff --git a/core/src/main/java/org/jclouds/rest/RestContext.java b/core/src/main/java/org/jclouds/rest/RestContext.java index be6299f31d..5b48c30fe4 100755 --- a/core/src/main/java/org/jclouds/rest/RestContext.java +++ b/core/src/main/java/org/jclouds/rest/RestContext.java @@ -22,7 +22,7 @@ import java.net.URI; import org.jclouds.rest.internal.RestContextImpl; -import com.google.common.util.concurrent.ListenableFuture; +import java.util.concurrent.Future; import com.google.inject.ImplementedBy; /** @@ -41,7 +41,7 @@ public interface RestContext { /** * low-level api to the cloud. Threadsafe implementations will return a singleton. * - * @return a connection to the cloud where all methods return {@link ListenableFuture}s + * @return a connection to the cloud where all methods return {@link Future}s */ A getAsyncApi(); diff --git a/core/src/main/java/org/jclouds/rest/RestContextBuilder.java b/core/src/main/java/org/jclouds/rest/RestContextBuilder.java index 484d5a36fe..eaaec6fc50 100755 --- a/core/src/main/java/org/jclouds/rest/RestContextBuilder.java +++ b/core/src/main/java/org/jclouds/rest/RestContextBuilder.java @@ -19,7 +19,7 @@ package org.jclouds.rest; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; +import static org.jclouds.concurrent.ConcurrentUtils.sameThreadExecutor; import static org.jclouds.Constants.PROPERTY_API; import static org.jclouds.Constants.PROPERTY_API_VERSION; import static org.jclouds.Constants.PROPERTY_CREDENTIAL; diff --git a/core/src/main/java/org/jclouds/rest/RestContextFactory.java b/core/src/main/java/org/jclouds/rest/RestContextFactory.java index 6fe2caef5b..8e4375715d 100644 --- a/core/src/main/java/org/jclouds/rest/RestContextFactory.java +++ b/core/src/main/java/org/jclouds/rest/RestContextFactory.java @@ -20,16 +20,17 @@ package org.jclouds.rest; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Splitter.on; import static com.google.common.base.Throwables.propagate; import static com.google.common.collect.Iterables.concat; import static com.google.common.collect.Iterables.size; import static com.google.common.collect.Iterables.transform; import static java.util.Collections.EMPTY_LIST; import static org.jclouds.util.Utils.initContextBuilder; +import static org.jclouds.util.Utils.modulesForProviderInProperties; import static org.jclouds.util.Utils.propagateAuthorizationOrOriginalException; import static org.jclouds.util.Utils.resolveContextBuilderClass; import static org.jclouds.util.Utils.resolvePropertiesBuilderClass; +import static org.jclouds.util.Utils.toStringAndClose; import java.io.File; import java.io.IOException; @@ -43,14 +44,12 @@ import org.jclouds.PropertiesBuilder; import com.google.common.base.Charsets; import com.google.common.base.Function; import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.io.Files; import com.google.inject.Module; /** - * Helper class to instantiate {@code RestContext} instances. - * "blobstore.properties" + * Helper class to instantiate {@code RestContext} instances. "blobstore.properties" * * At least one property is needed needed per context: *