From f679dc707af1ed9583dbc6036f3f22f9a2a134b1 Mon Sep 17 00:00:00 2001 From: hsbhathiya Date: Sat, 30 Aug 2014 02:27:06 +0530 Subject: [PATCH] JCLOUDS-458: Resumable Upload with live tests --- .../GoogleCloudStorageApi.java | 8 + .../binders/ResumableUploadBinder.java | 44 ++++ .../domain/DomainUtils.java | 4 + .../domain/ResumableUpload.java | 133 ++++++++++++ .../features/ResumableUploadApi.java | 197 ++++++++++++++++++ .../GoogleCloudStorageErrorHandler.java | 2 + ...oogleCloudStorageRedirectRetryHandler.java | 47 +++++ .../parser/ParseToResumableUpload.java | 66 ++++++ .../features/ResumableUploadApiLiveTest.java | 174 ++++++++++++++++ 9 files changed, 675 insertions(+) create mode 100644 providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/binders/ResumableUploadBinder.java create mode 100644 providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/domain/ResumableUpload.java create mode 100644 providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/features/ResumableUploadApi.java create mode 100644 providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageRedirectRetryHandler.java create mode 100644 providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/parser/ParseToResumableUpload.java create mode 100644 providers/google-cloud-storage/src/test/java/org/jclouds/googlecloudstorage/features/ResumableUploadApiLiveTest.java diff --git a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/GoogleCloudStorageApi.java b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/GoogleCloudStorageApi.java index e4d3e6d9b3..eb930df820 100644 --- a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/GoogleCloudStorageApi.java +++ b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/GoogleCloudStorageApi.java @@ -25,6 +25,7 @@ import org.jclouds.googlecloudstorage.features.BucketApi; import org.jclouds.googlecloudstorage.features.DefaultObjectAccessControlsApi; import org.jclouds.googlecloudstorage.features.ObjectAccessControlsApi; import org.jclouds.googlecloudstorage.features.ObjectApi; +import org.jclouds.googlecloudstorage.features.ResumableUploadApi; import org.jclouds.rest.annotations.Delegate; /** @@ -68,4 +69,11 @@ public interface GoogleCloudStorageApi extends Closeable { @Delegate @Path("") ObjectApi getObjectApi(); + + /** + * Provides access to Google Cloud Storage ResumableUpload features + */ + @Delegate + @Path("") + ResumableUploadApi getResumableUploadApi(); } diff --git a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/binders/ResumableUploadBinder.java b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/binders/ResumableUploadBinder.java new file mode 100644 index 0000000000..bfa29c9b88 --- /dev/null +++ b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/binders/ResumableUploadBinder.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.googlecloudstorage.binders; + +import java.util.Map; + +import javax.inject.Inject; + +import org.jclouds.googlecloudstorage.domain.templates.ObjectTemplate; +import org.jclouds.http.HttpRequest; +import org.jclouds.rest.MapBinder; +import org.jclouds.rest.binders.BindToJsonPayload; + +public class ResumableUploadBinder implements MapBinder { + + @Inject + private BindToJsonPayload jsonBinder; + + @Override + public R bindToRequest(R request, Map postParams) + throws IllegalArgumentException { + ObjectTemplate template = (ObjectTemplate) postParams.get("template"); + return bindToRequest(request, template); + } + + @Override + public R bindToRequest(R request, Object input) { + return jsonBinder.bindToRequest(request, input); + } +} diff --git a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/domain/DomainUtils.java b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/domain/DomainUtils.java index d028e7b7cb..c175691400 100644 --- a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/domain/DomainUtils.java +++ b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/domain/DomainUtils.java @@ -31,4 +31,8 @@ public final class DomainUtils { List reversedList = Lists.reverse(hashByte); return Bytes.toArray(reversedList); } + + public static String generateContentRange(Long lowerLimit, Long upperLimit, Long totalSize) { + return "bytes " + lowerLimit + "-" + upperLimit + "/" + totalSize; + } } diff --git a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/domain/ResumableUpload.java b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/domain/ResumableUpload.java new file mode 100644 index 0000000000..caeda2bcbe --- /dev/null +++ b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/domain/ResumableUpload.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.googlecloudstorage.domain; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkArgument; + +import org.jclouds.javax.annotation.Nullable; + +import com.google.common.base.MoreObjects.ToStringHelper; + +/** + * Represents results of resumable upload response. + */ +public class ResumableUpload { + + protected final Integer statusCode; + protected final String uploadId; + protected final String contentLength; + protected final Long rangeUpperValue; + protected final Long rangeLowerValue; + + private ResumableUpload(Integer statusCode, @Nullable String uploadId, @Nullable String contentLength, + @Nullable Long rangeLowerValue, @Nullable Long rangeUpperValue) { + if (rangeLowerValue != null && rangeUpperValue != null) { + checkArgument(rangeLowerValue < rangeUpperValue, "lower range must less than upper range, was: %s - %s", + rangeLowerValue, rangeUpperValue); + } + this.statusCode = checkNotNull(statusCode, "statusCode"); + this.uploadId = uploadId; + this.contentLength = contentLength; + this.rangeUpperValue = rangeUpperValue; + this.rangeLowerValue = rangeLowerValue; + } + + public String getUploadId() { + return uploadId; + } + + public Integer getStatusCode() { + return statusCode; + } + + public String getContentLength() { + return contentLength; + } + + public Long getRangeUpperValue() { + return rangeUpperValue; + } + + public Long getRangeLowerValue() { + return rangeLowerValue; + } + + protected ToStringHelper string() { + return toStringHelper(this).add("statusCode", statusCode).add("uploadId", uploadId) + .add("contentLength", contentLength).add("rangeUpperValue", rangeUpperValue) + .add("rangeLowerValue", rangeLowerValue); + } + + @Override + public String toString() { + return string().toString(); + } + + public static Builder builder() { + return new Builder(); + } + + public Builder toBuilder() { + return new Builder().fromResumableUpload(this); + } + + public static final class Builder { + + protected String uploadId; + protected Integer statusCode; + protected String contentLength; + protected Long rangeUpperValue; + protected Long rangeLowerValue; + + public Builder uploadId(String uploadId) { + this.uploadId = uploadId; + return this; + } + + public Builder statusCode(Integer statusCode) { + this.statusCode = statusCode; + return this; + } + + public Builder contentLength(String contentLength) { + this.contentLength = contentLength; + return this; + } + + public Builder rangeUpperValue(Long rangeUpperValue) { + this.rangeUpperValue = rangeUpperValue; + return this; + } + + public Builder rangeLowerValue(Long rangeLowerValue) { + this.rangeLowerValue = rangeLowerValue; + return this; + } + + public ResumableUpload build() { + return new ResumableUpload(statusCode, uploadId, contentLength, rangeLowerValue, rangeUpperValue); + } + + public Builder fromResumableUpload(ResumableUpload in) { + return this.statusCode(in.getStatusCode()).uploadId(in.getUploadId()).contentLength(in.getContentLength()) + .rangeUpperValue(in.getRangeUpperValue()).rangeLowerValue(in.getRangeLowerValue()); + } + + } +} diff --git a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/features/ResumableUploadApi.java b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/features/ResumableUploadApi.java new file mode 100644 index 0000000000..91d822cdcd --- /dev/null +++ b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/features/ResumableUploadApi.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.googlecloudstorage.features; + +import static org.jclouds.googlecloudstorage.reference.GoogleCloudStorageConstants.STORAGE_FULLCONTROL_SCOPE; + +import javax.inject.Named; +import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.HeaderParam; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; + +import org.jclouds.googlecloudstorage.binders.ResumableUploadBinder; +import org.jclouds.googlecloudstorage.binders.UploadBinder; +import org.jclouds.googlecloudstorage.domain.ResumableUpload; +import org.jclouds.googlecloudstorage.domain.templates.ObjectTemplate; +import org.jclouds.googlecloudstorage.options.InsertObjectOptions; +import org.jclouds.googlecloudstorage.parser.ParseToResumableUpload; +import org.jclouds.io.Payload; +import org.jclouds.oauth.v2.config.OAuthScopes; +import org.jclouds.oauth.v2.filters.OAuthAuthenticator; +import org.jclouds.rest.annotations.MapBinder; +import org.jclouds.rest.annotations.PayloadParam; +import org.jclouds.rest.annotations.QueryParams; +import org.jclouds.rest.annotations.RequestFilters; +import org.jclouds.rest.annotations.ResponseParser; +import org.jclouds.rest.annotations.SkipEncoding; + +/** + * Provides Resumable Upload support via Rest API + * + * @see + * @see + */ + +@SkipEncoding({ '/', '=' }) +@RequestFilters(OAuthAuthenticator.class) +public interface ResumableUploadApi { + + /** + * initiate a Resumable Upload Session + * + * @see https://developers.google.com/storage/docs/json_api/v1/how-tos/upload#resumable + * + * @param bucketName + * Name of the bucket in which the object to be stored + * @param objectName + * Name of the object to upload + * @param contentType + * Content type of the uploaded data + * @param contentLength + * ContentLength of the uploaded object (Media part) + * + * @return a {@link ResumableUpload} + */ + @Named("Object:initResumableUpload") + @POST + @QueryParams(keys = "uploadType", values = "resumable") + @Consumes(MediaType.APPLICATION_JSON) + @Path("/upload/storage/v1/b/{bucket}/o") + @OAuthScopes(STORAGE_FULLCONTROL_SCOPE) + @ResponseParser(ParseToResumableUpload.class) + ResumableUpload initResumableUpload(@PathParam("bucket") String bucketName, @QueryParam("name") String objectName, + @HeaderParam("X-Upload-Content-Type") String contentType, + @HeaderParam("X-Upload-Content-Length") String contentLength); + + /** + * initiate a Resumable Upload Session + * + * @see https://developers.google.com/storage/docs/json_api/v1/how-tos/upload#simple + * + * @param bucketName + * Name of the bucket in which the object to be stored + * @param contentType + * Content type of the uploaded data (Media part) + * @param contentLength + * Content length of the uploaded data (Media part) + * @param metada + * Supply an {@link ObjectTemplate} + * + * @return a {@link ResumableUpload} + */ + @Named("Object:resumableUpload") + @POST + @QueryParams(keys = "uploadType", values = "resumable") + @Consumes(MediaType.APPLICATION_JSON) + @Path("/upload/storage/v1/b/{bucket}/o") + @OAuthScopes(STORAGE_FULLCONTROL_SCOPE) + @MapBinder(ResumableUploadBinder.class) + @ResponseParser(ParseToResumableUpload.class) + ResumableUpload initResumableUpload(@PathParam("bucket") String bucketName, + @HeaderParam("X-Upload-Content-Type") String contentType, + @HeaderParam("X-Upload-Content-Length") Long contentLength, + @PayloadParam("template") ObjectTemplate metadata); + + /** + * Stores a new object + * + * @see https://developers.google.com/storage/docs/json_api/v1/how-tos/upload#resumable + * + * @param bucketName + * Name of the bucket in which the object to be stored + * @param options + * Supply {@link InsertObjectOptions} with optional query parameters. 'name' is mandatory. + * + * @return If successful, this method returns a {@link GCSObject} resource. + */ + @Named("Object:resumableUpload") + @PUT + @Consumes(MediaType.APPLICATION_JSON) + @QueryParams(keys = "uploadType", values = "resumable") + @Path("/upload/storage/v1/b/{bucket}/o") + @OAuthScopes(STORAGE_FULLCONTROL_SCOPE) + @MapBinder(UploadBinder.class) + @ResponseParser(ParseToResumableUpload.class) + ResumableUpload upload(@PathParam("bucket") String bucketName, @QueryParam("upload_id") String uploadId, + @HeaderParam("Content-Type") String contentType, @HeaderParam("Content-Length") String contentLength, + @PayloadParam("payload") Payload payload); + + /** + * Facilitate to use resumable upload operation to upload files in chunks + * + * @see https://developers.google.com/storage/docs/json_api/v1/how-tos/upload#resumable + * + * @param bucketName + * Name of the bucket in which the object to be stored + * @param uploadId + * uploadId returned from initResumableUpload operation + * @param contentType + * Content type of the uploaded data + * @param contentLength + * Content length of the uploaded data + * @param contentRange + * Range in {bytes StartingByte - Endingbyte/Totalsize } format ex: bytes 0 - 1213/2000 + * @param payload + * a {@link Payload} with actual data to upload + * + * @return a {@link ResumableUpload} + */ + @Named("Object:Upload") + @PUT + @Consumes(MediaType.APPLICATION_JSON) + @QueryParams(keys = "uploadType", values = "resumable") + @Path("/upload/storage/v1/b/{bucket}/o") + @OAuthScopes(STORAGE_FULLCONTROL_SCOPE) + @MapBinder(UploadBinder.class) + @ResponseParser(ParseToResumableUpload.class) + ResumableUpload chunkUpload(@PathParam("bucket") String bucketName, @QueryParam("upload_id") String uploadId, + @HeaderParam("Content-Type") String contentType, @HeaderParam("Content-Length") Long contentLength, + @HeaderParam("Content-Range") String contentRange, @PayloadParam("payload") Payload payload); + + /** + * Check the status of a resumable upload + * + * @see https://developers.google.com/storage/docs/json_api/v1/how-tos/upload#resumable + * + * @param bucketName + * Name of the bucket in which the object to be stored + * @param uploadId + * uploadId returned from initResumableUpload operation + * @param contentRange + * Range in {bytes StartingByte - Endingbyte/Totalsize } format ex: bytes 0 - 1213/2000 + * + * @return a {@link ResumableUpload} + */ + + @Named("Object:Upload") + @PUT + @Consumes(MediaType.APPLICATION_JSON) + @DefaultValue("0") + @QueryParams(keys = "uploadType", values = "resumable") + @Path("/upload/storage/v1/b/{bucket}/o") + @OAuthScopes(STORAGE_FULLCONTROL_SCOPE) + @ResponseParser(ParseToResumableUpload.class) + ResumableUpload checkStatus(@PathParam("bucket") String bucketName, @QueryParam("upload_id") String uploadId, + @HeaderParam("Content-Range") String contentRange); + +} diff --git a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageErrorHandler.java b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageErrorHandler.java index f7e24e9f0e..9c6840e9c4 100644 --- a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageErrorHandler.java +++ b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageErrorHandler.java @@ -46,6 +46,8 @@ public class GoogleCloudStorageErrorHandler implements HttpErrorHandler { String message412 = "PreconditionFailed: At least one of the pre-conditions you specified did not hold.\n"; switch (response.getStatusCode()) { + case 308: + return; case 401: case 403: exception = new AuthorizationException(message, exception); diff --git a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageRedirectRetryHandler.java b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageRedirectRetryHandler.java new file mode 100644 index 0000000000..45e6924166 --- /dev/null +++ b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/handlers/GoogleCloudStorageRedirectRetryHandler.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.googlecloudstorage.handlers; + +import javax.inject.Singleton; + +import org.jclouds.http.HttpCommand; +import org.jclouds.http.HttpResponse; +import org.jclouds.http.handlers.BackoffLimitedRetryHandler; +import org.jclouds.http.handlers.RedirectionRetryHandler; +import com.google.inject.Inject; + +/** + * This will parse and set an appropriate exception on the command object. + */ +@Singleton +public class GoogleCloudStorageRedirectRetryHandler extends RedirectionRetryHandler { + + @Inject + protected GoogleCloudStorageRedirectRetryHandler(BackoffLimitedRetryHandler backoffHandler) { + super(backoffHandler); + } + + @Override + public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) { + if (response.getStatusCode() == 308) { + return false; + } else { + return super.shouldRetryRequest(command, response); + } + } + +} diff --git a/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/parser/ParseToResumableUpload.java b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/parser/ParseToResumableUpload.java new file mode 100644 index 0000000000..fa3c3181e3 --- /dev/null +++ b/providers/google-cloud-storage/src/main/java/org/jclouds/googlecloudstorage/parser/ParseToResumableUpload.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.googlecloudstorage.parser; + +import java.util.regex.Pattern; + +import org.jclouds.googlecloudstorage.domain.ResumableUpload; +import org.jclouds.http.HttpResponse; + +import com.google.common.base.Function; +import com.google.common.base.Splitter; + +public class ParseToResumableUpload implements Function { + + @Override + public ResumableUpload apply(HttpResponse response) { + + String contentLength = response.getFirstHeaderOrNull("Content-Length"); + String sessionUri = response.getFirstHeaderOrNull("Location"); + String uploadId = null; + if (sessionUri != null) { + uploadId = getUploadId(sessionUri); + } + String range = response.getFirstHeaderOrNull("Range"); + Long upperLimit = null; + Long lowerLimit = null; + if (range != null) { + upperLimit = getUpperLimitFromRange(range); + lowerLimit = getLowerLimitFromRange(range); + } + + return ResumableUpload.builder().statusCode(response.getStatusCode()).contentLength(contentLength) + .uploadId(uploadId).rangeUpperValue(upperLimit).rangeLowerValue(lowerLimit).build(); + } + + // Return the Id of the Upload + private String getUploadId(String sessionUri) { + return Splitter.on(Pattern.compile("\\&")).trimResults().omitEmptyStrings().withKeyValueSeparator("=") + .split(sessionUri).get("upload_id"); + } + + private long getUpperLimitFromRange(String range) { + String upperLimit = range.split("-")[1]; + return Long.parseLong(upperLimit); + } + + private long getLowerLimitFromRange(String range) { + String removeByte = range.split("=")[1]; + String lowerLimit = removeByte.split("-")[0]; + return Long.parseLong(lowerLimit); + } +} diff --git a/providers/google-cloud-storage/src/test/java/org/jclouds/googlecloudstorage/features/ResumableUploadApiLiveTest.java b/providers/google-cloud-storage/src/test/java/org/jclouds/googlecloudstorage/features/ResumableUploadApiLiveTest.java new file mode 100644 index 0000000000..a802032d76 --- /dev/null +++ b/providers/google-cloud-storage/src/test/java/org/jclouds/googlecloudstorage/features/ResumableUploadApiLiveTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.googlecloudstorage.features; + +import static javax.ws.rs.core.Response.Status.OK; +import static javax.ws.rs.core.Response.Status.CREATED; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNotEquals; +import static org.assertj.core.api.Assertions.assertThat; +import java.io.IOException; +import java.util.UUID; + +import org.jclouds.googlecloudstorage.domain.DomainResourceRefferences.ObjectRole; +import org.jclouds.googlecloudstorage.domain.Bucket; +import org.jclouds.googlecloudstorage.domain.DomainUtils; +import org.jclouds.googlecloudstorage.domain.ResumableUpload; +import org.jclouds.googlecloudstorage.domain.templates.BucketTemplate; +import org.jclouds.googlecloudstorage.domain.templates.ObjectTemplate; +import org.jclouds.googlecloudstorage.domain.ObjectAccessControls; +import org.jclouds.googlecloudstorage.internal.BaseGoogleCloudStorageApiLiveTest; +import org.jclouds.io.Payloads; +import org.jclouds.io.payloads.ByteSourcePayload; +import org.jclouds.utils.TestUtils; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import com.google.common.io.ByteSource; + +public class ResumableUploadApiLiveTest extends BaseGoogleCloudStorageApiLiveTest { + + private static final String BUCKET_NAME = "resumableuploadbucket" + UUID.randomUUID(); + private static final String UPLOAD_OBJECT_NAME = "jcloudslogo.jpg"; + private static final String CHUNKED_OBJECT_NAME = "jclouds.pdf"; + private static final int INCOMPLETE = 308; + private static final long MIN_CHUNK_SIZE = 256 * 1024; // Min allowed size for a chunk + + private ResumableUploadApi api() { + return api.getResumableUploadApi(); + } + + @BeforeClass + private void createBucket() { + BucketTemplate template = new BucketTemplate().name(BUCKET_NAME); + Bucket bucket = api.getBucketApi().createBucket(PROJECT_NUMBER, template); + assertNotNull(bucket); + } + + @Test(groups = "live") + public void testResumableJpegUpload() throws IOException { + + // Read Object + long contentLength = MIN_CHUNK_SIZE * 4; + ByteSource byteSource = TestUtils.randomByteSource().slice(0, contentLength); + + // Initialize resumableUpload with metadata. ObjectTemaplete must provide the name + ObjectAccessControls oacl = ObjectAccessControls.builder().bucket(BUCKET_NAME).entity("allUsers") + .role(ObjectRole.OWNER).build(); + + ObjectTemplate template = new ObjectTemplate(); + template.contentType("image/jpeg").addAcl(oacl).size(contentLength).name(UPLOAD_OBJECT_NAME) + .contentLanguage("en").contentDisposition("attachment"); + + ResumableUpload initResponse = api().initResumableUpload(BUCKET_NAME, "image/jpeg", contentLength, template); + + assertNotNull(initResponse); + assertEquals(initResponse.getStatusCode().intValue(), OK.getStatusCode()); + assertNotNull(initResponse.getUploadId()); + + String uploadId = initResponse.getUploadId(); + + // Upload the payload + ByteSourcePayload payload = Payloads.newByteSourcePayload(byteSource); + ResumableUpload uploadResponse = api().upload(BUCKET_NAME, uploadId, "image/jpeg", byteSource.read().length + "", + payload); + + assertEquals(uploadResponse.getStatusCode().intValue(), OK.getStatusCode()); + + // CheckStatus + ResumableUpload status = api().checkStatus(BUCKET_NAME, uploadId, "bytes */*"); + + int code = status.getStatusCode(); + assertNotEquals(code, INCOMPLETE); + } + + @Test(groups = "live") + public void testResumableChunkedUpload() throws IOException, InterruptedException { + + // Read Object + long contentLength = MIN_CHUNK_SIZE * 3; + ByteSource byteSource = TestUtils.randomByteSource().slice(0, contentLength); + + // Initialize resumableUpload with metadata. ObjectTemaplete must provide the name + ObjectAccessControls oacl = ObjectAccessControls.builder().bucket(BUCKET_NAME).entity("allUsers") + .role(ObjectRole.OWNER).build(); + + ObjectTemplate template = new ObjectTemplate(); + template.contentType("application/pdf").addAcl(oacl).size(contentLength).name(CHUNKED_OBJECT_NAME) + .contentLanguage("en").contentDisposition("attachment"); + + ResumableUpload initResponse = api().initResumableUpload(BUCKET_NAME, "application/pdf", contentLength, template); + + assertNotNull(initResponse); + assertEquals(initResponse.getStatusCode().intValue(), OK.getStatusCode()); + assertNotNull(initResponse.getUploadId()); + + // Get the upload_id for the session + String uploadId = initResponse.getUploadId(); + + // Check the status first + ResumableUpload status = api().checkStatus(BUCKET_NAME, uploadId, "bytes */*"); + int code = status.getStatusCode(); + assertEquals(code, INCOMPLETE); + + // Uploads in 2 chunks. + long totalSize = byteSource.read().length; + long offset = 0; + // Size of the first chunk + long chunkSize = MIN_CHUNK_SIZE * 2; + + // Uploading First chunk + ByteSourcePayload payload = Payloads.newByteSourcePayload(byteSource.slice(offset, chunkSize)); + long length = byteSource.slice(offset, chunkSize).size(); + String Content_Range = DomainUtils.generateContentRange(0L, length, totalSize); + ResumableUpload uploadResponse = api().chunkUpload(BUCKET_NAME, uploadId, "application/pdf", length, + Content_Range, payload); + + int code2 = uploadResponse.getStatusCode(); + assertEquals(code2, INCOMPLETE); + + // Read uploaded length + long lowerValue = uploadResponse.getRangeLowerValue(); + long uploaded = uploadResponse.getRangeUpperValue(); + + assertThat(lowerValue).isEqualTo(0); + assertThat(uploaded).isEqualTo(chunkSize - 1); // confirms chunk is totally uploaded + + long resumeLength = totalSize - (uploaded + 1); + + // 2nd chunk + ByteSourcePayload payload2 = Payloads.newByteSourcePayload(byteSource.slice(uploaded + 1, + byteSource.read().length - uploaded - 1)); + // Upload the 2nd chunk + String Content_Range2 = DomainUtils.generateContentRange(uploaded + 1, totalSize - 1, totalSize); + ResumableUpload resumeResponse = api().chunkUpload(BUCKET_NAME, uploadId, "application/pdf", resumeLength, + Content_Range2, payload2); + + int code3 = resumeResponse.getStatusCode(); + assertThat(code3).isIn(OK.getStatusCode(), CREATED.getStatusCode()); // 200 or 201 if upload succeeded + } + + @AfterClass + private void deleteObjectsandBucket() { + api.getObjectApi().deleteObject(BUCKET_NAME, UPLOAD_OBJECT_NAME); + api.getObjectApi().deleteObject(BUCKET_NAME, CHUNKED_OBJECT_NAME); + api.getBucketApi().deleteBucket(BUCKET_NAME); + } +}