JCLOUDS-1267: Allow B2 streaming uploads

B2 now supports uploads without calculating the SHA-1 hash.  This
allows uploading without a repeatable payload.

providers/b2/src/main/java/org/jclouds/b2/binders/UploadFileBinder.java

@@ -33,10 +33,15 @@ public final class UploadFileBinder implements MapBinder {
    public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) {
       UploadUrlResponse uploadUrl = (UploadUrlResponse) postParams.get("uploadUrl");
       String fileName = (String) postParams.get("fileName");
+      String contentSha1 = (String) postParams.get("contentSha1");
+      if (contentSha1 == null) {
+         contentSha1 = "do_not_verify";
+      }
       Map<String, String> fileInfo = (Map<String, String>) postParams.get("fileInfo");
       HttpRequest.Builder builder = request.toBuilder()
             .endpoint(uploadUrl.uploadUrl())
             .replaceHeader(HttpHeaders.AUTHORIZATION, uploadUrl.authorizationToken())
+            .replaceHeader(B2Headers.CONTENT_SHA1, contentSha1)
             .replaceHeader(B2Headers.FILE_NAME, escaper.escape(fileName));
       for (Map.Entry<String, String> entry : fileInfo.entrySet()) {
          builder.replaceHeader(B2Headers.FILE_INFO_PREFIX + entry.getKey(), escaper.escape(entry.getValue()));

providers/b2/src/main/java/org/jclouds/b2/binders/UploadPartBinder.java

@@ -20,6 +20,7 @@ import java.util.Map;
 
 import org.jclouds.http.HttpRequest;
 import org.jclouds.b2.domain.GetUploadPartResponse;
+import org.jclouds.b2.reference.B2Headers;
 import org.jclouds.rest.MapBinder;
 
 import com.google.common.net.HttpHeaders;
@@ -28,9 +29,14 @@ public final class UploadPartBinder implements MapBinder {
    @Override
    public <R extends HttpRequest> R bindToRequest(R request, Map<String, Object> postParams) {
       GetUploadPartResponse uploadUrl = (GetUploadPartResponse) postParams.get("response");
+      String contentSha1 = (String) postParams.get("contentSha1");
+      if (contentSha1 == null) {
+         contentSha1 = "do_not_verify";
+      }
       return (R) request.toBuilder()
             .endpoint(uploadUrl.uploadUrl())
             .replaceHeader(HttpHeaders.AUTHORIZATION, uploadUrl.authorizationToken())
+            .replaceHeader(B2Headers.CONTENT_SHA1, contentSha1)
             .build();
    }
 

providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java

@@ -16,7 +16,6 @@
  */
 package org.jclouds.b2.blobstore;
 
-import java.io.IOException;
 import java.net.URI;
 import java.util.Date;
 import java.util.List;
@@ -67,7 +66,6 @@ import org.jclouds.blobstore.options.PutOptions;
 import org.jclouds.blobstore.util.BlobUtils;
 import org.jclouds.collect.Memoized;
 import org.jclouds.domain.Location;
-import org.jclouds.io.ByteStreams2;
 import org.jclouds.io.ContentMetadata;
 import org.jclouds.io.ContentMetadataBuilder;
 import org.jclouds.io.MutableContentMetadata;
@@ -78,14 +76,12 @@ import org.jclouds.io.payloads.BaseMutableContentMetadata;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.base.Supplier;
-import com.google.common.base.Throwables;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Sets;
-import com.google.common.hash.Hashing;
 import com.google.common.net.HttpHeaders;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 
@@ -254,8 +250,6 @@ public final class B2BlobStore extends BaseBlobStore {
 
    @Override
    public String putBlob(String container, Blob blob, PutOptions options) {
-      Preconditions.checkArgument(blob.getPayload().isRepeatable(), "B2 requires repeatable payload to calculate SHA1");
-
       if (options.getBlobAccess() != BlobAccess.PRIVATE) {
          throw new UnsupportedOperationException("B2 only supports private access blobs");
       }
@@ -263,13 +257,6 @@ public final class B2BlobStore extends BaseBlobStore {
       if (options.isMultipart()) {
          return putMultipartBlob(container, blob, options);
       } else {
-         String contentSha1;
-         try {
-            contentSha1 = ByteStreams2.hashAndClose(blob.getPayload().openStream(), Hashing.sha1()).toString();
-         } catch (IOException ioe) {
-            throw Throwables.propagate(ioe);
-         }
-
          String name = blob.getMetadata().getName();
 
          // B2 versions all files so we store the original fileId to delete it after the upload succeeds
@@ -277,7 +264,7 @@ public final class B2BlobStore extends BaseBlobStore {
 
          Bucket bucket = getBucket(container);
          UploadUrlResponse uploadUrl = api.getObjectApi().getUploadUrl(bucket.bucketId());
-         UploadFileResponse uploadFile = api.getObjectApi().uploadFile(uploadUrl, name, contentSha1, blob.getMetadata().getUserMetadata(), blob.getPayload());
+         UploadFileResponse uploadFile = api.getObjectApi().uploadFile(uploadUrl, name, null, blob.getMetadata().getUserMetadata(), blob.getPayload());
 
          if (oldFileId != null) {
             api.getObjectApi().deleteFileVersion(name, oldFileId);
@@ -397,20 +384,15 @@ public final class B2BlobStore extends BaseBlobStore {
 
    @Override
    public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) {
-      Preconditions.checkArgument(payload.isRepeatable());
-
-      String contentSha1;
-      try {
-         contentSha1 = ByteStreams2.hashAndClose(payload.openStream(), Hashing.sha1()).toString();
-      } catch (IOException ioe) {
-         throw Throwables.propagate(ioe);
-      }
-
       GetUploadPartResponse getUploadPart = api.getMultipartApi().getUploadPartUrl(mpu.id());
-      UploadPartResponse uploadPart = api.getMultipartApi().uploadPart(getUploadPart, partNumber, contentSha1, payload);
+      UploadPartResponse uploadPart = api.getMultipartApi().uploadPart(getUploadPart, partNumber, null, payload);
 
       Date lastModified = null;  // B2 does not return Last-Modified
-      return MultipartPart.create(uploadPart.partNumber(), uploadPart.contentLength(), uploadPart.contentSha1(), lastModified);
+      String contentSha1 = uploadPart.contentSha1();
+      if (contentSha1.startsWith("unverified:")) {
+          contentSha1 = contentSha1.substring("unverified:".length());
+      }
+      return MultipartPart.create(uploadPart.partNumber(), uploadPart.contentLength(), contentSha1, lastModified);
    }
 
    @Override

providers/b2/src/main/java/org/jclouds/b2/features/MultipartApi.java

@@ -84,7 +84,7 @@ public interface MultipartApi {
    @Named("b2_upload_part")
    @POST
    @MapBinder(UploadPartBinder.class)
-   UploadPartResponse uploadPart(@PayloadParam("response") GetUploadPartResponse response, @HeaderParam("X-Bz-Part-Number") int partNumber, @HeaderParam("X-Bz-Content-Sha1") String sha1, @PayloadParam("payload") Payload payload);
+   UploadPartResponse uploadPart(@PayloadParam("response") GetUploadPartResponse response, @HeaderParam("X-Bz-Part-Number") int partNumber, @Nullable @PayloadParam("contentSha1") String sha1, @PayloadParam("payload") Payload payload);
 
    @Named("b2_list_parts")
    @POST

providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java

@@ -24,7 +24,6 @@ import java.util.Map;
 import javax.inject.Named;
 import javax.ws.rs.Consumes;
 import javax.ws.rs.GET;
-import javax.ws.rs.HeaderParam;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
@@ -68,7 +67,7 @@ public interface ObjectApi {
    @POST
    @MapBinder(UploadFileBinder.class)
    @Consumes(APPLICATION_JSON)
-   UploadFileResponse uploadFile(@PayloadParam("uploadUrl") UploadUrlResponse uploadUrl, @PayloadParam("fileName") String fileName, @HeaderParam("X-Bz-Content-Sha1") String contentSha1, @PayloadParam("fileInfo") Map<String, String> fileInfo, Payload payload);
+   UploadFileResponse uploadFile(@PayloadParam("uploadUrl") UploadUrlResponse uploadUrl, @PayloadParam("fileName") String fileName, @Nullable @PayloadParam("contentSha1") String contentSha1, @PayloadParam("fileInfo") Map<String, String> fileInfo, Payload payload);
 
    @Named("b2_delete_file_version")
    @POST

providers/b2/src/test/java/org/jclouds/b2/blobstore/integration/B2BlobIntegrationLiveTest.java

@@ -122,26 +122,6 @@ public final class B2BlobIntegrationLiveTest extends BaseBlobIntegrationTest {
       }
    }
 
-   @Override
-   public void testPutInputStream() throws Exception {
-      try {
-         super.testPutInputStream();
-         failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
-      } catch (IllegalArgumentException iae) {
-         throw new SkipException("B2 requires repeatable payloads to calculate SHA1 hash", iae);
-      }
-   }
-
-   @Override
-   public void testPutMultipartInputStream() throws Exception {
-      try {
-         super.testPutMultipartInputStream();
-         failBecauseExceptionWasNotThrown(IllegalArgumentException.class);
-      } catch (IllegalArgumentException iae) {
-         throw new SkipException("B2 requires repeatable payloads to calculate SHA1 hash", iae);
-      }
-   }
-
    @Override
    public void testPutObjectStream() throws InterruptedException, IOException, ExecutionException {
       try {