Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-894: Move MultipartUploadSlicingAlgorithm to core

This commit is contained in:
Andrew Gaul 2015-06-06 14:50:13 -07:00
parent 97c160879e
commit 00734ee755
3 changed files with 65 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuGraphData.java
View File

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.blobstore.strategy.internal;
import org.jclouds.s3.blobstore.strategy.MultipartUpload;
/**
* Print out on the console some graph data regarding the partitioning algorithm.
*/
public class MpuGraphData {
private static void calculate(long length, MultipartUploadSlicingAlgorithm algorithm) {
System.out.println("" + length + " " + algorithm.getParts() + " "
+ algorithm.calculateChunkSize(length) + " " + algorithm.getRemaining());
}
private static void foreach(long from, long to1, long to2, long to3, MultipartUploadSlicingAlgorithm algorithm) {
long i = 0L;
long step = 1L;
System.out.println("=== {" + from + "," + to1 + "} ===");
for (; i < to1 - from; step += i, i += step) {
calculate(i + from, algorithm);
}
calculate(to1, algorithm);
System.out.println("=== {" + (to1 + 1) + "," + to2 + "} ===");
for (; i < to2 - to1; step += i / 20, i += step) {
calculate(i + from, algorithm);
}
calculate(to2, algorithm);
System.out.println("=== {" + (to2 + 1) + "," + to3 + "} ===");
for (; i < to3 - to2; step += i / 40, i += step) {
calculate(i + from, algorithm);
}
calculate(to3, algorithm);
}
public static void main(String[] args) {
MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm();
foreach(1L,
algorithm.defaultPartSize * algorithm.magnitudeBase,
MultipartUpload.MAX_PART_SIZE * algorithm.magnitudeBase,
MultipartUpload.MAX_PART_SIZE * MultipartUpload.MAX_NUMBER_OF_PARTS,
algorithm);
}
}

58
apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java → blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java
View File

@ -14,28 +14,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* MultipartUploadSlicingAlgorithm.java
*
*
* Created by: tibor
*
* History
*/
package org.jclouds.s3.blobstore.strategy.internal;
package org.jclouds.blobstore.strategy.internal;
import static com.google.common.base.Preconditions.checkArgument;
import javax.annotation.Resource;
import javax.inject.Named;
import org.jclouds.s3.blobstore.strategy.MultipartUpload;
import org.jclouds.blobstore.reference.BlobStoreConstants;
import org.jclouds.logging.Logger;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
public class MultipartUploadSlicingAlgorithm {
public final class MultipartUploadSlicingAlgorithm {
private final long minimumPartSize;
private final long maximumPartSize;
private final int maximumNumberOfParts;
@Resource
@Named(BlobStoreConstants.BLOBSTORE_LOGGER)
@ -43,15 +39,15 @@ public class MultipartUploadSlicingAlgorithm {
@VisibleForTesting
static final long DEFAULT_PART_SIZE = 33554432; // 32MB
@VisibleForTesting
static final int DEFAULT_MAGNITUDE_BASE = 100;
@Inject(optional = true)
@Named("jclouds.mpu.parts.size")
@VisibleForTesting
long defaultPartSize = DEFAULT_PART_SIZE;
@Inject(optional = true)
@Named("jclouds.mpu.parts.magnitude")
@VisibleForTesting
@ -67,35 +63,43 @@ public class MultipartUploadSlicingAlgorithm {
private volatile long chunkOffset;
private volatile long copied;
@VisibleForTesting
protected long calculateChunkSize(long length) {
public MultipartUploadSlicingAlgorithm(long minimumPartSize, long maximumPartSize, int maximumNumberOfParts) {
checkArgument(minimumPartSize > 0);
this.minimumPartSize = minimumPartSize;
checkArgument(maximumPartSize > 0);
this.maximumPartSize = maximumPartSize;
checkArgument(maximumNumberOfParts > 0);
this.maximumNumberOfParts = maximumNumberOfParts;
}
public long calculateChunkSize(long length) {
long unitPartSize = defaultPartSize; // first try with default part size
int parts = (int)(length / unitPartSize);
long partSize = unitPartSize;
int magnitude = parts / magnitudeBase;
if (magnitude > 0) {
partSize = magnitude * unitPartSize;
if (partSize > MultipartUpload.MAX_PART_SIZE) {
partSize = MultipartUpload.MAX_PART_SIZE;
unitPartSize = MultipartUpload.MAX_PART_SIZE;
if (partSize > maximumPartSize) {
partSize = maximumPartSize;
unitPartSize = maximumPartSize;
}
parts = (int)(length / partSize);
if (parts * partSize < length) {
partSize = (magnitude + 1) * unitPartSize;
if (partSize > MultipartUpload.MAX_PART_SIZE) {
partSize = MultipartUpload.MAX_PART_SIZE;
unitPartSize = MultipartUpload.MAX_PART_SIZE;
if (partSize > maximumPartSize) {
partSize = maximumPartSize;
unitPartSize = maximumPartSize;
}
parts = (int)(length / partSize);
}
}
if (parts > MultipartUpload.MAX_NUMBER_OF_PARTS) { // if splits in too many parts or
if (parts > maximumNumberOfParts) { // if splits in too many parts or
// cannot be split
unitPartSize = MultipartUpload.MIN_PART_SIZE; // take the minimum part size
unitPartSize = minimumPartSize; // take the minimum part size
parts = (int)(length / unitPartSize);
}
if (parts > MultipartUpload.MAX_NUMBER_OF_PARTS) { // if still splits in too many parts
parts = MultipartUpload.MAX_NUMBER_OF_PARTS - 1; // limit them. do not care about not
if (parts > maximumNumberOfParts) { // if still splits in too many parts
parts = maximumNumberOfParts - 1; // limit them. do not care about not
// covering
}
long remainder = length % unitPartSize;
@ -106,7 +110,7 @@ public class MultipartUploadSlicingAlgorithm {
this.parts = parts;
this.remaining = length - partSize * parts;
logger.debug(" %d bytes partitioned in %d parts of part size: %d, remaining: %d%s", length, parts, chunkSize,
remaining, remaining > MultipartUpload.MAX_PART_SIZE ? " overflow!" : "");
remaining, remaining > maximumPartSize ? " overflow!" : "");
return this.chunkSize;
}

62
apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java → blobstore/src/test/java/org/jclouds/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java
View File

@ -14,11 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.s3.blobstore.strategy.internal;
package org.jclouds.blobstore.strategy.internal;
import static org.testng.Assert.assertEquals;
import org.jclouds.s3.blobstore.strategy.MultipartUpload;
import org.testng.annotations.Test;
/**
@ -26,7 +25,10 @@ import org.testng.annotations.Test;
* partitioning algorithm
*/
@Test(groups = "unit")
public class MpuPartitioningAlgorithmTest {
public final class MpuPartitioningAlgorithmTest {
private final long MIN_PART_SIZE = 5L * 1024 * 1024;
private final long MAX_PART_SIZE = 5L * 1024 * 1024 * 1024;
private final int MAX_NUMBER_OF_PARTS = 10 * 1000;
/**
* Below 1 parts the MPU is not used.
@ -35,10 +37,11 @@ public class MpuPartitioningAlgorithmTest {
*/
@Test
public void testLowerLimitFromWhereMultipartBecomeActive() {
MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm();
MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(
MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS);
// exactly the MIN_PART_SIZE
long length = MultipartUpload.MIN_PART_SIZE;
long length = MIN_PART_SIZE;
long chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE);
assertEquals(strategy.getParts(), 0);
@ -59,17 +62,18 @@ public class MpuPartitioningAlgorithmTest {
assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE);
assertEquals(strategy.getParts(), 1);
assertEquals(strategy.getRemaining(), 1);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
}
/**
* Phase 1 of the algorithm.
* ChunkSize does not grow from a {@code MultipartUploadStrategy.DEFAULT_PART_SIZE}
* until we reach {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE} number of parts.
* ChunkSize does not grow from a {@code MultipartUploadStrategy.DEFAULT_PART_SIZE}
* until we reach {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE} number of parts.
*/
@Test
public void testWhenChunkSizeHasToStartGrowing() {
MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm();
MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(
MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS);
// upper limit while we still have exactly defaultPartSize chunkSize
long length = MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE * MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE;
long chunkSize = strategy.calculateChunkSize(length);
@ -86,54 +90,56 @@ public class MpuPartitioningAlgorithmTest {
assertEquals(strategy.getRemaining(), 1);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
}
/**
* Phase 2 of the algorithm.
* The number of parts does not grow from {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE}
* until we reach the {@code MultipartUploadStrategy.MAX_PART_SIZE}.
* The number of parts does not grow from {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE}
* until we reach the {@code MultipartUploadStrategy.MAX_PART_SIZE}.
*/
@Test
public void testWhenPartsHasToStartGrowingFromMagnitudeBase() {
MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm();
MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(
MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS);
// upper limit while we still have exactly MAGNITUDE_BASE parts (together with the remaining)
long length = MultipartUpload.MAX_PART_SIZE * MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE;
long length = MAX_PART_SIZE * MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE;
long chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, MultipartUpload.MAX_PART_SIZE);
assertEquals(chunkSize, MAX_PART_SIZE);
assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE - 1);
assertEquals(strategy.getRemaining(), MultipartUpload.MAX_PART_SIZE);
assertEquals(strategy.getRemaining(), MAX_PART_SIZE);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
// then the number of parts is increasing
length += 1;
chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, MultipartUpload.MAX_PART_SIZE);
assertEquals(chunkSize, MAX_PART_SIZE);
assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE);
assertEquals(strategy.getRemaining(), 1);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
}
/**
* Phase 3 of the algorithm.
* The number of parts are increasing until {@code MAX_NUMBER_OF_PARTS}
* while its size does not exceeds the {@code MultipartUploadStrategy.MAX_PART_SIZE}.
* while its size does not exceeds the {@code MultipartUploadStrategy.MAX_PART_SIZE}.
*/
@Test
public void testWhenPartsExceedsMaxNumberOfParts() {
MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm();
MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(
MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS);
// upper limit while we still have exactly MAX_NUMBER_OF_PARTS parts (together with the remaining)
long length = MultipartUpload.MAX_PART_SIZE * MultipartUpload.MAX_NUMBER_OF_PARTS;
long length = MAX_PART_SIZE * MAX_NUMBER_OF_PARTS;
long chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, MultipartUpload.MAX_PART_SIZE);
assertEquals(strategy.getParts(), MultipartUpload.MAX_NUMBER_OF_PARTS - 1);
assertEquals(strategy.getRemaining(), MultipartUpload.MAX_PART_SIZE);
assertEquals(chunkSize, MAX_PART_SIZE);
assertEquals(strategy.getParts(), MAX_NUMBER_OF_PARTS - 1);
assertEquals(strategy.getRemaining(), MAX_PART_SIZE);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
// then the number of parts is increasing
length += 1;
chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, MultipartUpload.MAX_PART_SIZE);
assertEquals(strategy.getParts(), MultipartUpload.MAX_NUMBER_OF_PARTS);
assertEquals(chunkSize, MAX_PART_SIZE);
assertEquals(strategy.getParts(), MAX_NUMBER_OF_PARTS);
assertEquals(strategy.getRemaining(), 1);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
}
}
}