JCLOUDS-619: Introduce MultipartNamingStrategy to generate part names correctly.

apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategy.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.swift.blobstore.strategy.internal;
+
+import javax.inject.Singleton;
+
+@Singleton
+public class MultipartNamingStrategy {
+
+   private static final String PART_SEPARATOR = "/";
+
+   protected String getPartName(String key, int partNumber, int totalParts) {
+      int base = (int) Math.log10(totalParts) + 1;
+      return String.format("%s%s%0" + base + "d", key, PART_SEPARATOR, partNumber);
+   }
+}

apis/swift/src/main/java/org/jclouds/openstack/swift/blobstore/strategy/internal/SequentialMultipartUploadStrategy.java

@@ -34,7 +34,6 @@ import org.jclouds.openstack.swift.blobstore.functions.BlobToObject;
 import com.google.inject.Inject;
 
 public class SequentialMultipartUploadStrategy implements MultipartUploadStrategy {
-   private static final String PART_SEPARATOR = "/";
 
    @Resource
    @Named(BlobStoreConstants.BLOBSTORE_LOGGER)
@@ -45,15 +44,18 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
    private final BlobToObject blob2Object;
    private final MultipartUploadSlicingAlgorithm algorithm;
    private final PayloadSlicer slicer;
+   private final MultipartNamingStrategy namingStrategy;
 
    @Inject
    public SequentialMultipartUploadStrategy(CommonSwiftClient client, Provider<BlobBuilder> blobBuilders,
-         BlobToObject blob2Object, MultipartUploadSlicingAlgorithm algorithm, PayloadSlicer slicer) {
+         BlobToObject blob2Object, MultipartUploadSlicingAlgorithm algorithm, PayloadSlicer slicer,
+         MultipartNamingStrategy namingStrategy) {
       this.client = checkNotNull(client, "client");
       this.blobBuilders = checkNotNull(blobBuilders, "blobBuilders");
       this.blob2Object = checkNotNull(blob2Object, "blob2Object");
       this.algorithm = checkNotNull(algorithm, "algorithm");
       this.slicer = checkNotNull(slicer, "slicer");
+      this.namingStrategy = checkNotNull(namingStrategy, "namingStrategy");
    }
 
    @Override
@@ -68,10 +70,11 @@ public class SequentialMultipartUploadStrategy implements MultipartUploadStrateg
       if (partCount > 0) {
          for (Payload part : slicer.slice(payload, chunkSize)) {
             int partNum = algorithm.getNextPart();
+            String partName = namingStrategy.getPartName(key, partNum, partCount);
             Blob blobPart = blobBuilders.get()
-                                        .name(key + PART_SEPARATOR + partNum)
+                                        .name(partName)
                                         .payload(part)
-                                        .contentDisposition(key + PART_SEPARATOR + partNum)
+                                        .contentDisposition(partName)
                                         .build();
             client.putObject(container, blob2Object.apply(blobPart));
          }

apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftBlobIntegrationLiveTest.java

@@ -26,11 +26,14 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Properties;
+import java.util.Random;
 
 import org.jclouds.blobstore.BlobStore;
 import org.jclouds.blobstore.domain.Blob;
 import org.jclouds.blobstore.integration.internal.BaseBlobIntegrationTest;
 import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.io.ByteSources;
+import org.jclouds.io.Payload;
 import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties;
 import org.jclouds.openstack.swift.blobstore.strategy.MultipartUpload;
 import org.testng.ITestContext;
@@ -40,6 +43,7 @@ import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import com.google.common.hash.Hashing;
+import com.google.common.io.ByteSource;
 import com.google.common.io.ByteStreams;
 import com.google.common.io.Files;
 import com.google.common.io.InputSupplier;
@@ -83,18 +87,18 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
       throw new SkipException("not yet implemented");
    }
 
-    @BeforeClass(groups = { "integration", "live" }, dependsOnMethods = "setupContext")
+   @BeforeClass(groups = {"integration", "live"}, dependsOnMethods = "setupContext")
-    @Override
+   @Override
-    public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
+   public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
-        super.setUpResourcesOnThisThread(testContext);
+      super.setUpResourcesOnThisThread(testContext);
-        oneHundredOneConstitutions = getTestDataSupplier();
+      oneHundredOneConstitutions = getTestDataSupplier();
-    }
+   }
 
    @Override
    protected void checkContentDisposition(Blob blob, String contentDisposition) {
-     // This works for Swift Server 1.4.4/SWauth 1.0.3 but was null in previous versions.
+      // This works for Swift Server 1.4.4/SWauth 1.0.3 but was null in previous versions.
-     // TODO: Better testing for the different versions.
+      // TODO: Better testing for the different versions.
-     super.checkContentDisposition(blob,contentDisposition);
+      super.checkContentDisposition(blob, contentDisposition);
    }
 
    // not supported in swift
@@ -128,7 +132,40 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
                     "A multipart blob wasn't actually created - " +
                     "there was only 1 extra blob but there should be one manifest blob and multiple chunk blobs");
       } finally {
-          returnContainer(containerName);
+         returnContainer(containerName);
+      }
+   }
+
+   /**
+    * Checks that when there are more than 9 chunks the object names
+    * are set correctly so that the order of the object names matches
+    * the upload order.
+    * See issue https://issues.apache.org/jira/browse/JCLOUDS-619
+    */
+   @Test(groups = {"integration", "live"})
+   public void testMultipartChunkedFilenames() throws InterruptedException, IOException {
+      String containerName = getContainerName();
+      try {
+         BlobStore blobStore = view.getBlobStore();
+         String objectName = "object.txt";
+         long countBefore = blobStore.countBlobs(containerName);
+
+         // we want 11 parts
+         ByteSource inputSource = createByteSource(PART_SIZE * 11);
+         addMultipartBlobToContainer(containerName, objectName, inputSource);
+
+         // did we create enough parts?
+         long countAfter = blobStore.countBlobs(containerName);
+         assertNotEquals(countAfter, countBefore, "No blob was created");
+         assertEquals(countAfter, countBefore + 12,
+                 "12 parts (11 objects + 1 manifest) were expected.");
+
+         // download and check if correct
+         Blob read = blobStore.getBlob(containerName, objectName);
+         Payload readPayload = read.getPayload();
+         assertTrue(inputSource.contentEquals(ByteSources.asByteSource(readPayload.openStream())));
+      } finally {
+         returnContainer(containerName);
       }
    }
 
@@ -165,15 +202,28 @@ public class SwiftBlobIntegrationLiveTest extends BaseBlobIntegrationTest {
 
    protected void addMultipartBlobToContainer(String containerName, String key) throws IOException {
       File fileToUpload = createFileBiggerThan(PART_SIZE);
+      addMultipartBlobToContainer(containerName, key, Files.asByteSource(fileToUpload));
+   }
 
+   protected void addMultipartBlobToContainer(String containerName, String key, ByteSource byteSource) throws IOException {
       BlobStore blobStore = view.getBlobStore();
       blobStore.createContainerInLocation(null, containerName);
       Blob blob = blobStore.blobBuilder(key)
-         .payload(fileToUpload)
+              .payload(byteSource)
-         .build();
+              .contentLength(byteSource.size())
+              .build();
       blobStore.putBlob(containerName, blob, PutOptions.Builder.multipart());
    }
 
+   private ByteSource createByteSource(long size) throws IOException {
+      final Random random = new Random();
+      final byte[] randomBytes = new byte[(int) MultipartUpload.MIN_PART_SIZE];
+      random.nextBytes(randomBytes);
+      ByteSource byteSource = ByteSources.repeatingArrayByteSource(randomBytes).slice(0, size);
+      assertEquals(byteSource.size(), size);
+      return byteSource;
+   }
+
    @SuppressWarnings("unchecked")
    private File createFileBiggerThan(long partSize) throws IOException {
       long copiesNeeded = (partSize / getOneHundredOneConstitutionsLength()) + 1;

apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/strategy/internal/MultipartNamingStrategyTest.java

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.swift.blobstore.strategy.internal;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+@Test(testName = "MultipartNamingStrategyTest")
+public class MultipartNamingStrategyTest {
+
+   @Test
+   public void testGetPartNameFirstOneHundred() {
+      final MultipartNamingStrategy strategy = new MultipartNamingStrategy();
+      final String key = "file.txt";
+      final int numberParts = 100;
+
+      // check the first 100
+      for (int i = 0; i < numberParts; i++) {
+         String partName = strategy.getPartName(key, i + 1, numberParts);
+         assertEquals(String.format("file.txt/%03d", i + 1), partName);
+      }
+   }
+
+   @Test
+   public void testGetPartNameChoices() {
+      final MultipartNamingStrategy strategy = new MultipartNamingStrategy();
+      final String key = "file.txt";
+
+      // check less than 10 parts
+      assertEquals(strategy.getPartName(key, 1, 5), "file.txt/1");
+      assertEquals(strategy.getPartName(key, 2, 5), "file.txt/2");
+      assertEquals(strategy.getPartName(key, 5, 5), "file.txt/5");
+
+      // check <= 10 parts
+      assertEquals(strategy.getPartName(key, 1, 10), "file.txt/01");
+      assertEquals(strategy.getPartName(key, 2, 10), "file.txt/02");
+      assertEquals(strategy.getPartName(key, 10, 10), "file.txt/10");
+
+      // check <= 100 parts
+      assertEquals(strategy.getPartName(key, 1, 100), "file.txt/001");
+      assertEquals(strategy.getPartName(key, 9, 100), "file.txt/009");
+      assertEquals(strategy.getPartName(key, 10, 100), "file.txt/010");
+      assertEquals(strategy.getPartName(key, 99, 100), "file.txt/099");
+      assertEquals(strategy.getPartName(key, 100, 100), "file.txt/100");
+
+      // check <= 5000 parts
+      assertEquals(strategy.getPartName(key, 1, 5000), "file.txt/0001");
+      assertEquals(strategy.getPartName(key, 10, 5000), "file.txt/0010");
+      assertEquals(strategy.getPartName(key, 99, 5000), "file.txt/0099");
+      assertEquals(strategy.getPartName(key, 100, 5000), "file.txt/0100");
+      assertEquals(strategy.getPartName(key, 999, 5000), "file.txt/0999");
+      assertEquals(strategy.getPartName(key, 4999, 5000), "file.txt/4999");
+      assertEquals(strategy.getPartName(key, 5000, 500), "file.txt/5000");
+   }
+}
+