Apache/jclouds
1
0
Fork 0
mirror of https://github.com/apache/jclouds.git synced 2025-02-28 13:29:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-1267: Allow B2 streaming uploads

Browse Source 
B2 now supports uploads without calculating the SHA-1 hash.  This
allows uploading without a repeatable payload.
This commit is contained in:
Andrew Gaul 2017-04-07 17:28:35 -07:00
parent 8408242279
commit 66caf6d954
6 changed files with 20 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
providers/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java
View File

@ -33,10 +33,15 @@ public final class UploadFileBinder implements MapBinder {
public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) {
UploadUrlResponse uploadUrl = (UploadUrlResponse) postParams.get("uploadUrl");
String fileName = (String) postParams.get("fileName");
String contentSha1 = (String) postParams.get("contentSha1");
if (contentSha1 == null) {
contentSha1 = "do_not_verify";
}
Map<String, String> fileInfo = (Map<String, String>) postParams.get("fileInfo");
HttpRequest.Builder builder = request.toBuilder()
.endpoint(uploadUrl.uploadUrl())
.replaceHeader(HttpHeaders.AUTHORIZATION, uploadUrl.authorizationToken())
.replaceHeader(B2Headers.CONTENT_SHA1, contentSha1)
.replaceHeader(B2Headers.FILE_NAME, escaper.escape(fileName));
for (Map.Entry<String, String> entry : fileInfo.entrySet()) {
builder.replaceHeader(B2Headers.FILE_INFO_PREFIX + entry.getKey(), escaper.escape(entry.getValue()));

6
providers/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java
View File

@ -20,6 +20,7 @@ import java.util.Map;
import org.jclouds.http.HttpRequest;
import org.jclouds.b2.domain.GetUploadPartResponse;
import org.jclouds.b2.reference.B2Headers;
import org.jclouds.rest.MapBinder;
import com.google.common.net.HttpHeaders;
@ -28,9 +29,14 @@ public final class UploadPartBinder implements MapBinder {
@Override
public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) {
GetUploadPartResponse uploadUrl = (GetUploadPartResponse) postParams.get("response");
String contentSha1 = (String) postParams.get("contentSha1");
if (contentSha1 == null) {
contentSha1 = "do_not_verify";
}
return (R) request.toBuilder()
.endpoint(uploadUrl.uploadUrl())
.replaceHeader(HttpHeaders.AUTHORIZATION, uploadUrl.authorizationToken())
.replaceHeader(B2Headers.CONTENT_SHA1, contentSha1)
.build();
}

32
providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java
View File

@ -16,7 +16,6 @@
*/
package org.jclouds.b2.blobstore;
import java.io.IOException;
import java.net.URI;
import java.util.Date;
import java.util.List;
@ -67,7 +66,6 @@ import org.jclouds.blobstore.options.PutOptions;
import org.jclouds.blobstore.util.BlobUtils;
import org.jclouds.collect.Memoized;
import org.jclouds.domain.Location;
import org.jclouds.io.ByteStreams2;
import org.jclouds.io.ContentMetadata;
import org.jclouds.io.ContentMetadataBuilder;
import org.jclouds.io.MutableContentMetadata;
@ -78,14 +76,12 @@ import org.jclouds.io.payloads.BaseMutableContentMetadata;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import com.google.common.hash.Hashing;
import com.google.common.net.HttpHeaders;
import com.google.common.util.concurrent.UncheckedExecutionException;
@ -254,8 +250,6 @@ public final class B2BlobStore extends BaseBlobStore {
@Override
public String putBlob(String container, Blob blob, PutOptions options) {
Preconditions.checkArgument(blob.getPayload().isRepeatable(), "B2 requires repeatable payload to calculate SHA1");
if (options.getBlobAccess() != BlobAccess.PRIVATE) {
throw new UnsupportedOperationException("B2 only supports private access blobs");
}
@ -263,13 +257,6 @@ public final class B2BlobStore extends BaseBlobStore {
if (options.isMultipart()) {
return putMultipartBlob(container, blob, options);
} else {
String contentSha1;
try {
contentSha1 = ByteStreams2.hashAndClose(blob.getPayload().openStream(), Hashing.sha1()).toString();
} catch (IOException ioe) {
throw Throwables.propagate(ioe);
}
String name = blob.getMetadata().getName();
// B2 versions all files so we store the original fileId to delete it after the upload succeeds
@ -277,7 +264,7 @@ public final class B2BlobStore extends BaseBlobStore {
Bucket bucket = getBucket(container);
UploadUrlResponse uploadUrl = api.getObjectApi().getUploadUrl(bucket.bucketId());
UploadFileResponse uploadFile = api.getObjectApi().uploadFile(uploadUrl, name, contentSha1, blob.getMetadata().getUserMetadata(), blob.getPayload());
UploadFileResponse uploadFile = api.getObjectApi().uploadFile(uploadUrl, name, null, blob.getMetadata().getUserMetadata(), blob.getPayload());
if (oldFileId != null) {
api.getObjectApi().deleteFileVersion(name, oldFileId);
@ -397,20 +384,15 @@ public final class B2BlobStore extends BaseBlobStore {
@Override
public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) {
Preconditions.checkArgument(payload.isRepeatable());
String contentSha1;
try {
contentSha1 = ByteStreams2.hashAndClose(payload.openStream(), Hashing.sha1()).toString();
} catch (IOException ioe) {
throw Throwables.propagate(ioe);
}
GetUploadPartResponse getUploadPart = api.getMultipartApi().getUploadPartUrl(mpu.id());
UploadPartResponse uploadPart = api.getMultipartApi().uploadPart(getUploadPart, partNumber, contentSha1, payload);
UploadPartResponse uploadPart = api.getMultipartApi().uploadPart(getUploadPart, partNumber, null, payload);
Date lastModified = null; // B2 does not return Last-Modified
return MultipartPart.create(uploadPart.partNumber(), uploadPart.contentLength(), uploadPart.contentSha1(), lastModified);
String contentSha1 = uploadPart.contentSha1();
if (contentSha1.startsWith("unverified:")) {
contentSha1 = contentSha1.substring("unverified:".length());
}
return MultipartPart.create(uploadPart.partNumber(), uploadPart.contentLength(), contentSha1, lastModified);
}
@Override

2
providers/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java
View File

@ -84,7 +84,7 @@ public interface MultipartApi {
@Named("b2_upload_part")
@POST
@MapBinder(UploadPartBinder.class)
UploadPartResponse uploadPart(@PayloadParam("response") GetUploadPartResponse response, @HeaderParam("X-Bz-Part-Number") int partNumber, @HeaderParam("X-Bz-Content-Sha1") String sha1, @PayloadParam("payload") Payload payload);
UploadPartResponse uploadPart(@PayloadParam("response") GetUploadPartResponse response, @HeaderParam("X-Bz-Part-Number") int partNumber, @Nullable @PayloadParam("contentSha1") String sha1, @PayloadParam("payload") Payload payload);
@Named("b2_list_parts")
@POST

3
providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java
View File

@ -24,7 +24,6 @@ import java.util.Map;
import javax.inject.Named;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.HeaderParam;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@ -68,7 +67,7 @@ public interface ObjectApi {
@POST
@MapBinder(UploadFileBinder.class)
@Consumes(APPLICATION_JSON)
UploadFileResponse uploadFile(@PayloadParam("uploadUrl") UploadUrlResponse uploadUrl, @PayloadParam("fileName") String fileName, @HeaderParam("X-Bz-Content-Sha1") String contentSha1, @PayloadParam("fileInfo") Map<String, String> fileInfo, Payload payload);
UploadFileResponse uploadFile(@PayloadParam("uploadUrl") UploadUrlResponse uploadUrl, @PayloadParam("fileName") String fileName, @Nullable @PayloadParam("contentSha1") String contentSha1, @PayloadParam("fileInfo") Map<String, String> fileInfo, Payload payload);
@Named("b2_delete_file_version")
@POST

20
providers/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java
View File

@ -122,26 +122,6 @@ public final class B2BlobIntegrationLiveTest extends BaseBlobIntegrationTest {
}
}
@Override
public void testPutInputStream() throws Exception {
try {
super.testPutInputStream();
failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
} catch (IllegalArgumentException iae) {
throw new SkipException("B2 requires repeatable payloads to calculate SHA1 hash", iae);
}
}
@Override
public void testPutMultipartInputStream() throws Exception {
try {
super.testPutMultipartInputStream();
failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
} catch (IllegalArgumentException iae) {
throw new SkipException("B2 requires repeatable payloads to calculate SHA1 hash", iae);
}
}
@Override
public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException {
try {