diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuGraphData.java b/apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuGraphData.java deleted file mode 100644 index 1bf0fe7404..0000000000 --- a/apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuGraphData.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.jclouds.s3.blobstore.strategy.internal; - -import org.jclouds.s3.blobstore.strategy.MultipartUpload; - -/** - * Print out on the console some graph data regarding the partitioning algorithm. - */ -public class MpuGraphData { - - private static void calculate(long length, MultipartUploadSlicingAlgorithm algorithm) { - System.out.println("" + length + " " + algorithm.getParts() + " " - + algorithm.calculateChunkSize(length) + " " + algorithm.getRemaining()); - } - - private static void foreach(long from, long to1, long to2, long to3, MultipartUploadSlicingAlgorithm algorithm) { - long i = 0L; - long step = 1L; - System.out.println("=== {" + from + "," + to1 + "} ==="); - for (; i < to1 - from; step += i, i += step) { - calculate(i + from, algorithm); - } - calculate(to1, algorithm); - System.out.println("=== {" + (to1 + 1) + "," + to2 + "} ==="); - for (; i < to2 - to1; step += i / 20, i += step) { - calculate(i + from, algorithm); - } - calculate(to2, algorithm); - System.out.println("=== {" + (to2 + 1) + "," + to3 + "} ==="); - for (; i < to3 - to2; step += i / 40, i += step) { - calculate(i + from, algorithm); - } - calculate(to3, algorithm); - } - - public static void main(String[] args) { - MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(); - foreach(1L, - algorithm.defaultPartSize * algorithm.magnitudeBase, - MultipartUpload.MAX_PART_SIZE * algorithm.magnitudeBase, - MultipartUpload.MAX_PART_SIZE * MultipartUpload.MAX_NUMBER_OF_PARTS, - algorithm); - } - -} diff --git a/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java similarity index 72% rename from apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java rename to blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java index c9125ad3a8..db74fb49b7 100644 --- a/apis/s3/src/main/java/org/jclouds/s3/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java +++ b/blobstore/src/main/java/org/jclouds/blobstore/strategy/internal/MultipartUploadSlicingAlgorithm.java @@ -14,28 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * MultipartUploadSlicingAlgorithm.java - * - * - * Created by: tibor - * - * History - */ -package org.jclouds.s3.blobstore.strategy.internal; +package org.jclouds.blobstore.strategy.internal; + +import static com.google.common.base.Preconditions.checkArgument; import javax.annotation.Resource; import javax.inject.Named; -import org.jclouds.s3.blobstore.strategy.MultipartUpload; import org.jclouds.blobstore.reference.BlobStoreConstants; import org.jclouds.logging.Logger; import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; -public class MultipartUploadSlicingAlgorithm { +public final class MultipartUploadSlicingAlgorithm { + private final long minimumPartSize; + private final long maximumPartSize; + private final int maximumNumberOfParts; @Resource @Named(BlobStoreConstants.BLOBSTORE_LOGGER) @@ -43,15 +39,15 @@ public class MultipartUploadSlicingAlgorithm { @VisibleForTesting static final long DEFAULT_PART_SIZE = 33554432; // 32MB - + @VisibleForTesting static final int DEFAULT_MAGNITUDE_BASE = 100; - + @Inject(optional = true) @Named("jclouds.mpu.parts.size") @VisibleForTesting long defaultPartSize = DEFAULT_PART_SIZE; - + @Inject(optional = true) @Named("jclouds.mpu.parts.magnitude") @VisibleForTesting @@ -67,35 +63,43 @@ public class MultipartUploadSlicingAlgorithm { private volatile long chunkOffset; private volatile long copied; - @VisibleForTesting - protected long calculateChunkSize(long length) { + public MultipartUploadSlicingAlgorithm(long minimumPartSize, long maximumPartSize, int maximumNumberOfParts) { + checkArgument(minimumPartSize > 0); + this.minimumPartSize = minimumPartSize; + checkArgument(maximumPartSize > 0); + this.maximumPartSize = maximumPartSize; + checkArgument(maximumNumberOfParts > 0); + this.maximumNumberOfParts = maximumNumberOfParts; + } + + public long calculateChunkSize(long length) { long unitPartSize = defaultPartSize; // first try with default part size int parts = (int)(length / unitPartSize); long partSize = unitPartSize; int magnitude = parts / magnitudeBase; if (magnitude > 0) { partSize = magnitude * unitPartSize; - if (partSize > MultipartUpload.MAX_PART_SIZE) { - partSize = MultipartUpload.MAX_PART_SIZE; - unitPartSize = MultipartUpload.MAX_PART_SIZE; + if (partSize > maximumPartSize) { + partSize = maximumPartSize; + unitPartSize = maximumPartSize; } parts = (int)(length / partSize); if (parts * partSize < length) { partSize = (magnitude + 1) * unitPartSize; - if (partSize > MultipartUpload.MAX_PART_SIZE) { - partSize = MultipartUpload.MAX_PART_SIZE; - unitPartSize = MultipartUpload.MAX_PART_SIZE; + if (partSize > maximumPartSize) { + partSize = maximumPartSize; + unitPartSize = maximumPartSize; } parts = (int)(length / partSize); } } - if (parts > MultipartUpload.MAX_NUMBER_OF_PARTS) { // if splits in too many parts or + if (parts > maximumNumberOfParts) { // if splits in too many parts or // cannot be split - unitPartSize = MultipartUpload.MIN_PART_SIZE; // take the minimum part size + unitPartSize = minimumPartSize; // take the minimum part size parts = (int)(length / unitPartSize); } - if (parts > MultipartUpload.MAX_NUMBER_OF_PARTS) { // if still splits in too many parts - parts = MultipartUpload.MAX_NUMBER_OF_PARTS - 1; // limit them. do not care about not + if (parts > maximumNumberOfParts) { // if still splits in too many parts + parts = maximumNumberOfParts - 1; // limit them. do not care about not // covering } long remainder = length % unitPartSize; @@ -106,7 +110,7 @@ public class MultipartUploadSlicingAlgorithm { this.parts = parts; this.remaining = length - partSize * parts; logger.debug(" %d bytes partitioned in %d parts of part size: %d, remaining: %d%s", length, parts, chunkSize, - remaining, remaining > MultipartUpload.MAX_PART_SIZE ? " overflow!" : ""); + remaining, remaining > maximumPartSize ? " overflow!" : ""); return this.chunkSize; } diff --git a/apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java b/blobstore/src/test/java/org/jclouds/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java similarity index 80% rename from apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java rename to blobstore/src/test/java/org/jclouds/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java index ac6dc7c6d4..bcf29e9477 100644 --- a/apis/s3/src/test/java/org/jclouds/s3/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java +++ b/blobstore/src/test/java/org/jclouds/blobstore/strategy/internal/MpuPartitioningAlgorithmTest.java @@ -14,11 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.jclouds.s3.blobstore.strategy.internal; +package org.jclouds.blobstore.strategy.internal; import static org.testng.Assert.assertEquals; -import org.jclouds.s3.blobstore.strategy.MultipartUpload; import org.testng.annotations.Test; /** @@ -26,7 +25,10 @@ import org.testng.annotations.Test; * partitioning algorithm */ @Test(groups = "unit") -public class MpuPartitioningAlgorithmTest { +public final class MpuPartitioningAlgorithmTest { + private final long MIN_PART_SIZE = 5L * 1024 * 1024; + private final long MAX_PART_SIZE = 5L * 1024 * 1024 * 1024; + private final int MAX_NUMBER_OF_PARTS = 10 * 1000; /** * Below 1 parts the MPU is not used. @@ -35,10 +37,11 @@ public class MpuPartitioningAlgorithmTest { */ @Test public void testLowerLimitFromWhereMultipartBecomeActive() { - MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(); - + MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm( + MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS); + // exactly the MIN_PART_SIZE - long length = MultipartUpload.MIN_PART_SIZE; + long length = MIN_PART_SIZE; long chunkSize = strategy.calculateChunkSize(length); assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE); assertEquals(strategy.getParts(), 0); @@ -59,17 +62,18 @@ public class MpuPartitioningAlgorithmTest { assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE); assertEquals(strategy.getParts(), 1); assertEquals(strategy.getRemaining(), 1); - assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); + assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); } /** * Phase 1 of the algorithm. - * ChunkSize does not grow from a {@code MultipartUploadStrategy.DEFAULT_PART_SIZE} - * until we reach {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE} number of parts. + * ChunkSize does not grow from a {@code MultipartUploadStrategy.DEFAULT_PART_SIZE} + * until we reach {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE} number of parts. */ @Test public void testWhenChunkSizeHasToStartGrowing() { - MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(); + MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm( + MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS); // upper limit while we still have exactly defaultPartSize chunkSize long length = MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE * MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE; long chunkSize = strategy.calculateChunkSize(length); @@ -86,54 +90,56 @@ public class MpuPartitioningAlgorithmTest { assertEquals(strategy.getRemaining(), 1); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); } - + /** * Phase 2 of the algorithm. - * The number of parts does not grow from {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE} - * until we reach the {@code MultipartUploadStrategy.MAX_PART_SIZE}. + * The number of parts does not grow from {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE} + * until we reach the {@code MultipartUploadStrategy.MAX_PART_SIZE}. */ @Test public void testWhenPartsHasToStartGrowingFromMagnitudeBase() { - MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(); + MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm( + MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS); // upper limit while we still have exactly MAGNITUDE_BASE parts (together with the remaining) - long length = MultipartUpload.MAX_PART_SIZE * MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE; + long length = MAX_PART_SIZE * MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE; long chunkSize = strategy.calculateChunkSize(length); - assertEquals(chunkSize, MultipartUpload.MAX_PART_SIZE); + assertEquals(chunkSize, MAX_PART_SIZE); assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE - 1); - assertEquals(strategy.getRemaining(), MultipartUpload.MAX_PART_SIZE); + assertEquals(strategy.getRemaining(), MAX_PART_SIZE); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); // then the number of parts is increasing length += 1; chunkSize = strategy.calculateChunkSize(length); - assertEquals(chunkSize, MultipartUpload.MAX_PART_SIZE); + assertEquals(chunkSize, MAX_PART_SIZE); assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE); assertEquals(strategy.getRemaining(), 1); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); } - + /** * Phase 3 of the algorithm. * The number of parts are increasing until {@code MAX_NUMBER_OF_PARTS} - * while its size does not exceeds the {@code MultipartUploadStrategy.MAX_PART_SIZE}. + * while its size does not exceeds the {@code MultipartUploadStrategy.MAX_PART_SIZE}. */ @Test public void testWhenPartsExceedsMaxNumberOfParts() { - MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(); + MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm( + MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS); // upper limit while we still have exactly MAX_NUMBER_OF_PARTS parts (together with the remaining) - long length = MultipartUpload.MAX_PART_SIZE * MultipartUpload.MAX_NUMBER_OF_PARTS; + long length = MAX_PART_SIZE * MAX_NUMBER_OF_PARTS; long chunkSize = strategy.calculateChunkSize(length); - assertEquals(chunkSize, MultipartUpload.MAX_PART_SIZE); - assertEquals(strategy.getParts(), MultipartUpload.MAX_NUMBER_OF_PARTS - 1); - assertEquals(strategy.getRemaining(), MultipartUpload.MAX_PART_SIZE); + assertEquals(chunkSize, MAX_PART_SIZE); + assertEquals(strategy.getParts(), MAX_NUMBER_OF_PARTS - 1); + assertEquals(strategy.getRemaining(), MAX_PART_SIZE); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); // then the number of parts is increasing length += 1; chunkSize = strategy.calculateChunkSize(length); - assertEquals(chunkSize, MultipartUpload.MAX_PART_SIZE); - assertEquals(strategy.getParts(), MultipartUpload.MAX_NUMBER_OF_PARTS); + assertEquals(chunkSize, MAX_PART_SIZE); + assertEquals(strategy.getParts(), MAX_NUMBER_OF_PARTS); assertEquals(strategy.getRemaining(), 1); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); - } + } }