Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-1368: Correct use of slicing algorithm 

MultipartUploadSlicingAlgorithm creates multiple equal-sized parts and
a remaining amount.  BaseBlobStore.putMultipartBlob used this
interface incorrectly, which could create more parts than intended
since the remaining size could be larger than the part size.  This
manifested with Google Cloud Storage which only allows 32 parts.
This commit is contained in:
Andrew Gaul 2018-01-04 14:52:10 -08:00
parent 52c92a9eb5
commit d05be89614
2 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
blobstore/src/main/java/org/jclouds/blobstore/internal/BaseBlobStore.java
View File

@ -354,14 +354,24 @@ public abstract class BaseBlobStore implements BlobStore {
MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides);
try {
long contentLength = blob.getMetadata().getContentMetadata().getContentLength();
// TODO: inject MultipartUploadSlicingAlgorithm to override default part size
MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts());
long partSize = algorithm.calculateChunkSize(contentLength);
int partNumber = 1;
for (Payload payload : slicer.slice(blob.getPayload(), partSize)) {
// TODO: for InputStream payloads, this buffers all parts in-memory!
while (partNumber < algorithm.getParts()) {
Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), partSize);
BlobUploader b =
new BlobUploader(mpu, partNumber++, payload);
parts.add(executor.submit(b));
algorithm.addCopied(partSize);
}
if (algorithm.getRemaining() != 0) {
Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), algorithm.getRemaining());
BlobUploader b =
new BlobUploader(mpu, partNumber, payload);
parts.add(executor.submit(b));
}
return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts)));
} catch (RuntimeException re) {

8
blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
View File

@ -132,8 +132,7 @@ public final class MultipartUploadSlicingAlgorithm {
this.copied = copied;
}
@VisibleForTesting
protected int getParts() {
public int getParts() {
return parts;
}
@ -141,7 +140,7 @@ public final class MultipartUploadSlicingAlgorithm {
return ++part;
}
protected void addCopied(long copied) {
public void addCopied(long copied) {
this.copied += copied;
}
@ -156,8 +155,7 @@ public final class MultipartUploadSlicingAlgorithm {
return chunkSize;
}
@VisibleForTesting
protected long getRemaining() {
public long getRemaining() {
return remaining;
}