JCLOUDS-458 Fix some obvious bugs in google-cloud-storage.

This commit is contained in:
Adrian Cole 2014-11-03 20:48:12 -08:00 committed by Adrian Cole
parent d02767ece5
commit b77ddb3351
6 changed files with 55 additions and 65 deletions

View File

@ -29,30 +29,35 @@ import org.jclouds.io.Payloads;
import org.jclouds.io.payloads.MultipartForm; import org.jclouds.io.payloads.MultipartForm;
import org.jclouds.io.payloads.Part; import org.jclouds.io.payloads.Part;
import org.jclouds.io.payloads.StringPayload; import org.jclouds.io.payloads.StringPayload;
import org.jclouds.json.Json;
import org.jclouds.rest.MapBinder; import org.jclouds.rest.MapBinder;
import com.google.gson.Gson; import com.google.inject.Inject;
public class MultipartUploadBinder implements MapBinder {
public final class MultipartUploadBinder implements MapBinder {
private static final String BOUNDARY_HEADER = "multipart_boundary"; private static final String BOUNDARY_HEADER = "multipart_boundary";
private final Json json;
@Inject MultipartUploadBinder(Json json){
this.json = json;
}
@Override public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) { @Override public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) {
ObjectTemplate template = (ObjectTemplate) postParams.get("template"); ObjectTemplate template = (ObjectTemplate) postParams.get("template");
Payload payload = (Payload) postParams.get("payload"); Payload payload = (Payload) postParams.get("payload");
String contentType = checkNotNull(template.cacheControl(), "contentType"); String contentType = checkNotNull(template.contentType(), "contentType");
Long length = checkNotNull(template.size(), "contentLength"); Long length = checkNotNull(template.size(), "contentLength");
StringPayload jsonPayload = Payloads.newStringPayload(new Gson().toJson(template)); StringPayload jsonPayload = Payloads.newStringPayload(json.toJson(template));
payload.getContentMetadata().setContentLength(length); payload.getContentMetadata().setContentLength(length);
Part jsonPart = Part.create("Metadata", jsonPayload, new Part.PartOptions().contentType(APPLICATION_JSON)); Part jsonPart = Part.create("Metadata", jsonPayload, new Part.PartOptions().contentType(APPLICATION_JSON));
Part mediaPart = Part.create(template.name(), payload, new Part.PartOptions().contentType(contentType)); Part mediaPart = Part.create(template.name(), payload, new Part.PartOptions().contentType(contentType));
MultipartForm compPayload = new MultipartForm(BOUNDARY_HEADER, jsonPart, mediaPart); request.setPayload(new MultipartForm(BOUNDARY_HEADER, jsonPart, mediaPart));
request.setPayload(compPayload);
// HeaderPart // HeaderPart
request.toBuilder().replaceHeader(CONTENT_TYPE, "Multipart/related; boundary= " + BOUNDARY_HEADER).build(); request.toBuilder().replaceHeader(CONTENT_TYPE, "Multipart/related; boundary= " + BOUNDARY_HEADER).build();
return request; return request;

View File

@ -18,12 +18,13 @@ package org.jclouds.googlecloudstorage.blobstore;
import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.io.BaseEncoding.base64; import static com.google.common.io.BaseEncoding.base64;
import static org.jclouds.googlecloudstorage.domain.DomainResourceReferences.ObjectRole.READER;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.Set; import java.util.Set;
import javax.inject.Singleton; import javax.inject.Inject;
import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.Blob;
@ -35,6 +36,7 @@ import org.jclouds.blobstore.domain.internal.BlobImpl;
import org.jclouds.blobstore.domain.internal.PageSetImpl; import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.internal.BaseBlobStore; import org.jclouds.blobstore.internal.BaseBlobStore;
import org.jclouds.blobstore.options.CreateContainerOptions; import org.jclouds.blobstore.options.CreateContainerOptions;
import org.jclouds.blobstore.options.GetOptions;
import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.blobstore.options.PutOptions; import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata; import org.jclouds.blobstore.strategy.internal.FetchBlobMetadata;
@ -50,7 +52,7 @@ import org.jclouds.googlecloudstorage.blobstore.functions.ObjectToBlobMetadata;
import org.jclouds.googlecloudstorage.blobstore.strategy.internal.MultipartUploadStrategy; import org.jclouds.googlecloudstorage.blobstore.strategy.internal.MultipartUploadStrategy;
import org.jclouds.googlecloudstorage.config.UserProject; import org.jclouds.googlecloudstorage.config.UserProject;
import org.jclouds.googlecloudstorage.domain.Bucket; import org.jclouds.googlecloudstorage.domain.Bucket;
import org.jclouds.googlecloudstorage.domain.DomainResourceReferences.ObjectRole; import org.jclouds.googlecloudstorage.domain.DomainResourceReferences;
import org.jclouds.googlecloudstorage.domain.GCSObject; import org.jclouds.googlecloudstorage.domain.GCSObject;
import org.jclouds.googlecloudstorage.domain.ListPage; import org.jclouds.googlecloudstorage.domain.ListPage;
import org.jclouds.googlecloudstorage.domain.templates.BucketTemplate; import org.jclouds.googlecloudstorage.domain.templates.BucketTemplate;
@ -66,24 +68,21 @@ import com.google.common.base.Supplier;
import com.google.common.base.Throwables; import com.google.common.base.Throwables;
import com.google.common.collect.Iterables; import com.google.common.collect.Iterables;
import com.google.common.hash.HashCode; import com.google.common.hash.HashCode;
import com.google.inject.Inject;
import com.google.inject.Provider; import com.google.inject.Provider;
@Singleton public final class GCSBlobStore extends BaseBlobStore {
public class GCSBlobStore extends BaseBlobStore {
GoogleCloudStorageApi api; private final GoogleCloudStorageApi api;
BucketToStorageMetadata bucketToStorageMetadata; private final BucketToStorageMetadata bucketToStorageMetadata;
ObjectToBlobMetadata objectToBlobMetadata; private final ObjectToBlobMetadata objectToBlobMetadata;
ObjectListToStorageMetadata objectListToStorageMetadata; private final ObjectListToStorageMetadata objectListToStorageMetadata;
Provider<FetchBlobMetadata> fetchBlobMetadataProvider; private final Provider<FetchBlobMetadata> fetchBlobMetadataProvider;
BlobMetadataToObjectTemplate blobMetadataToObjectTemplate; private final BlobMetadataToObjectTemplate blobMetadataToObjectTemplate;
BlobStoreListContainerOptionsToListObjectOptions listContainerOptionsToListObjectOptions; private final BlobStoreListContainerOptionsToListObjectOptions listContainerOptionsToListObjectOptions;
MultipartUploadStrategy multipartUploadStrategy; private final MultipartUploadStrategy multipartUploadStrategy;
Supplier<String> projectId; private final Supplier<String> projectId;
@Inject @Inject GCSBlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
protected GCSBlobStore(BlobStoreContext context, BlobUtils blobUtils, Supplier<Location> defaultLocation,
@Memoized Supplier<Set<? extends Location>> locations, GoogleCloudStorageApi api, @Memoized Supplier<Set<? extends Location>> locations, GoogleCloudStorageApi api,
BucketToStorageMetadata bucketToStorageMetadata, ObjectToBlobMetadata objectToBlobMetadata, BucketToStorageMetadata bucketToStorageMetadata, ObjectToBlobMetadata objectToBlobMetadata,
ObjectListToStorageMetadata objectListToStorageMetadata, ObjectListToStorageMetadata objectListToStorageMetadata,
@ -105,9 +104,10 @@ public class GCSBlobStore extends BaseBlobStore {
@Override @Override
public PageSet<? extends StorageMetadata> list() { public PageSet<? extends StorageMetadata> list() {
return new Function<ListPage<Bucket>, org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata>>() { return new Function<ListPage<Bucket>, PageSet<? extends StorageMetadata>>() {
public org.jclouds.blobstore.domain.PageSet<? extends StorageMetadata> apply(ListPage<Bucket> from) { public PageSet<? extends StorageMetadata> apply(ListPage<Bucket> from) {
return new PageSetImpl<StorageMetadata>(Iterables.transform(from, bucketToStorageMetadata), null); return new PageSetImpl<StorageMetadata>(Iterables.transform(from, bucketToStorageMetadata),
from.nextPageToken());
} }
}.apply(api.getBucketApi().listBucket(projectId.get())); }.apply(api.getBucketApi().listBucket(projectId.get()));
} }
@ -121,8 +121,7 @@ public class GCSBlobStore extends BaseBlobStore {
public boolean createContainerInLocation(Location location, String container) { public boolean createContainerInLocation(Location location, String container) {
BucketTemplate template = new BucketTemplate().name(container); BucketTemplate template = new BucketTemplate().name(container);
if (location != null) { if (location != null) {
org.jclouds.googlecloudstorage.domain.DomainResourceReferences.Location gcsLocation = org.jclouds.googlecloudstorage.domain.DomainResourceReferences.Location DomainResourceReferences.Location gcsLocation = DomainResourceReferences.Location.fromValue(location.getId());
.fromValue(location.getId());
template = template.location(gcsLocation); template = template.location(gcsLocation);
} }
return api.getBucketApi().createBucket(projectId.get(), template) != null; return api.getBucketApi().createBucket(projectId.get(), template) != null;
@ -132,15 +131,13 @@ public class GCSBlobStore extends BaseBlobStore {
public boolean createContainerInLocation(Location location, String container, CreateContainerOptions options) { public boolean createContainerInLocation(Location location, String container, CreateContainerOptions options) {
BucketTemplate template = new BucketTemplate().name(container); BucketTemplate template = new BucketTemplate().name(container);
if (location != null) { if (location != null) {
org.jclouds.googlecloudstorage.domain.DomainResourceReferences.Location gcsLocation = org.jclouds.googlecloudstorage.domain.DomainResourceReferences.Location DomainResourceReferences.Location gcsLocation = DomainResourceReferences.Location.fromValue(location.getId());
.fromValue(location.getId());
template = template.location(gcsLocation); template = template.location(gcsLocation);
} }
Bucket bucket = api.getBucketApi().createBucket(projectId.get(), template); Bucket bucket = api.getBucketApi().createBucket(projectId.get(), template);
if (options.isPublicRead()) { if (options.isPublicRead()) {
try { try {
ObjectAccessControlsTemplate doAclTemplate = ObjectAccessControlsTemplate ObjectAccessControlsTemplate doAclTemplate = ObjectAccessControlsTemplate.create("allUsers", READER);
.create("allUsers", ObjectRole.READER);
api.getDefaultObjectAccessControlsApi().createDefaultObjectAccessControls(container, doAclTemplate); api.getDefaultObjectAccessControlsApi().createDefaultObjectAccessControls(container, doAclTemplate);
} catch (HttpResponseException e) { } catch (HttpResponseException e) {
// If DefaultObjectAccessControls operation fail, Reverse create operation the operation. // If DefaultObjectAccessControls operation fail, Reverse create operation the operation.
@ -162,7 +159,6 @@ public class GCSBlobStore extends BaseBlobStore {
@Override @Override
public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) { public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) {
if (options != null && options != ListContainerOptions.NONE) { if (options != null && options != ListContainerOptions.NONE) {
ListObjectOptions listOptions = listContainerOptionsToListObjectOptions.apply(options); ListObjectOptions listOptions = listContainerOptionsToListObjectOptions.apply(options);
ListPage<GCSObject> gcsList = api.getObjectApi().listObjects(container, listOptions); ListPage<GCSObject> gcsList = api.getObjectApi().listObjects(container, listOptions);
@ -218,7 +214,7 @@ public class GCSBlobStore extends BaseBlobStore {
} }
@Override @Override
public Blob getBlob(String container, String name, org.jclouds.blobstore.options.GetOptions options) { public Blob getBlob(String container, String name, GetOptions options) {
GCSObject gcsObject = api.getObjectApi().getObject(container, name); GCSObject gcsObject = api.getObjectApi().getObject(container, name);
if (gcsObject == null) { if (gcsObject == null) {
return null; return null;

View File

@ -58,6 +58,5 @@ public class ObjectListToStorageMetadata implements Function<ListPage<GCSObject>
return input; return input;
} }
}), from.nextPageToken()); }), from.nextPageToken());
} }
} }

View File

@ -20,13 +20,10 @@ import static com.google.common.base.Preconditions.checkNotNull;
import java.util.List; import java.util.List;
import javax.annotation.Resource;
import javax.inject.Named;
import javax.inject.Provider; import javax.inject.Provider;
import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobBuilder; import org.jclouds.blobstore.domain.BlobBuilder;
import org.jclouds.blobstore.reference.BlobStoreConstants;
import org.jclouds.googlecloudstorage.GoogleCloudStorageApi; import org.jclouds.googlecloudstorage.GoogleCloudStorageApi;
import org.jclouds.googlecloudstorage.blobstore.functions.BlobMetadataToObjectTemplate; import org.jclouds.googlecloudstorage.blobstore.functions.BlobMetadataToObjectTemplate;
import org.jclouds.googlecloudstorage.domain.GCSObject; import org.jclouds.googlecloudstorage.domain.GCSObject;
@ -34,16 +31,11 @@ import org.jclouds.googlecloudstorage.domain.templates.ComposeObjectTemplate;
import org.jclouds.googlecloudstorage.domain.templates.ObjectTemplate; import org.jclouds.googlecloudstorage.domain.templates.ObjectTemplate;
import org.jclouds.io.Payload; import org.jclouds.io.Payload;
import org.jclouds.io.PayloadSlicer; import org.jclouds.io.PayloadSlicer;
import org.jclouds.logging.Logger;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.inject.Inject; import com.google.inject.Inject;
public class SequentialMultipartUploadStrategy extends MultipartUploadStrategy { public final class SequentialMultipartUploadStrategy extends MultipartUploadStrategy {
@Resource
@Named(BlobStoreConstants.BLOBSTORE_LOGGER)
private Logger logger = Logger.NULL;
private final GoogleCloudStorageApi api; private final GoogleCloudStorageApi api;
private final Provider<BlobBuilder> blobBuilders; private final Provider<BlobBuilder> blobBuilders;
@ -52,16 +44,15 @@ public class SequentialMultipartUploadStrategy extends MultipartUploadStrategy {
private final PayloadSlicer slicer; private final PayloadSlicer slicer;
private final MultipartNamingStrategy namingStrategy; private final MultipartNamingStrategy namingStrategy;
@Inject @Inject SequentialMultipartUploadStrategy(GoogleCloudStorageApi api, Provider<BlobBuilder> blobBuilders,
public SequentialMultipartUploadStrategy(GoogleCloudStorageApi api, Provider<BlobBuilder> blobBuilders,
BlobMetadataToObjectTemplate blob2ObjectTemplate, MultipartUploadSlicingAlgorithm algorithm, BlobMetadataToObjectTemplate blob2ObjectTemplate, MultipartUploadSlicingAlgorithm algorithm,
PayloadSlicer slicer, MultipartNamingStrategy namingStrategy) { PayloadSlicer slicer, MultipartNamingStrategy namingStrategy) {
this.api = checkNotNull(api, "api"); this.api = api;
this.blobBuilders = checkNotNull(blobBuilders, "blobBuilders"); this.blobBuilders = blobBuilders;
this.blob2ObjectTemplate = checkNotNull(blob2ObjectTemplate, "blob2Object"); this.blob2ObjectTemplate = blob2ObjectTemplate;
this.algorithm = checkNotNull(algorithm, "algorithm"); this.algorithm = algorithm;
this.slicer = checkNotNull(slicer, "slicer"); this.slicer = slicer;
this.namingStrategy = checkNotNull(namingStrategy, "namingStrategy"); this.namingStrategy = namingStrategy;
} }
@Override @Override

View File

@ -44,31 +44,31 @@ public abstract class GCSObject {
public abstract String etag(); public abstract String etag();
public abstract String name(); public abstract String name();
public abstract String bucket(); public abstract String bucket();
public abstract Long generation(); public abstract long generation();
public abstract Long metageneration(); public abstract long metageneration();
public abstract String contentType(); public abstract String contentType();
public abstract Date updated(); public abstract Date updated();
public abstract Date timeDeleted(); @Nullable public abstract Date timeDeleted();
public abstract StorageClass storageClass(); public abstract StorageClass storageClass();
public abstract Long size(); public abstract long size();
@Nullable public abstract String md5Hash(); @Nullable public abstract String md5Hash();
public abstract URI mediaLink(); public abstract URI mediaLink();
public abstract Map<String, String> metadata(); public abstract Map<String, String> metadata();
public abstract String contentEncoding(); @Nullable public abstract String contentEncoding();
public abstract String contentDisposition(); @Nullable public abstract String contentDisposition();
public abstract String contentLanguage(); @Nullable public abstract String contentLanguage();
public abstract String cacheControl(); @Nullable public abstract String cacheControl();
public abstract List<ObjectAccessControls> acl(); public abstract List<ObjectAccessControls> acl();
public abstract Owner owner(); public abstract Owner owner();
@Nullable public abstract String crc32c(); @Nullable public abstract String crc32c();
public abstract Integer componentCount(); @Nullable public abstract Integer componentCount();
@SerializedNames( @SerializedNames(
{ "id", "selfLink", "etag", "name", "bucket", "generation", "metageneration", "contentType", "updated", { "id", "selfLink", "etag", "name", "bucket", "generation", "metageneration", "contentType", "updated",
"timeDeleted", "storageClass", "size", "md5Hash", "mediaLink", "metadata", "contentEncoding", "timeDeleted", "storageClass", "size", "md5Hash", "mediaLink", "metadata", "contentEncoding",
"contentDisposition", "contentLanguage", "cacheControl", "acl", "owner", "crc32c", "componentCount" }) "contentDisposition", "contentLanguage", "cacheControl", "acl", "owner", "crc32c", "componentCount" })
public static GCSObject create(String id, URI selfLink, String etag, String name, String bucket, Long generation, public static GCSObject create(String id, URI selfLink, String etag, String name, String bucket, long generation,
Long metageneration, String contentType, Date updated, Date timeDeleted, StorageClass storageClass, Long size, long metageneration, String contentType, Date updated, Date timeDeleted, StorageClass storageClass, long size,
String md5Hash, URI mediaLink, Map<String, String> metadata, String contentEncoding, String contentDisposition, String md5Hash, URI mediaLink, Map<String, String> metadata, String contentEncoding, String contentDisposition,
String contentLanguage, String cacheControl, List<ObjectAccessControls> acl, Owner owner, String crc32c, String contentLanguage, String cacheControl, List<ObjectAccessControls> acl, Owner owner, String crc32c,
Integer componentCount) { Integer componentCount) {

View File

@ -474,7 +474,6 @@ public interface ObjectApi {
* *
* @return a {@link GCSObject} * @return a {@link GCSObject}
*/ */
@Named("Object:multipartUpload") @Named("Object:multipartUpload")
@POST @POST
@QueryParams(keys = "uploadType", values = "multipart") @QueryParams(keys = "uploadType", values = "multipart")
@ -483,6 +482,6 @@ public interface ObjectApi {
@OAuthScopes(STORAGE_FULLCONTROL_SCOPE) @OAuthScopes(STORAGE_FULLCONTROL_SCOPE)
@MapBinder(MultipartUploadBinder.class) @MapBinder(MultipartUploadBinder.class)
GCSObject multipartUpload(@PathParam("bucket") String bucketName, GCSObject multipartUpload(@PathParam("bucket") String bucketName,
@BinderParam(BindToJsonPayload.class) ObjectTemplate objectTemplate, @PayloadParam("template") ObjectTemplate objectTemplate,
@PayloadParam("payload") Payload payload); @PayloadParam("payload") Payload payload);
} }