JCLOUDS-1281: Swift dynamic large objects

apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java

@@ -27,6 +27,7 @@ import org.jclouds.location.functions.RegionToEndpoint;
 import org.jclouds.openstack.swift.v1.features.AccountApi;
 import org.jclouds.openstack.swift.v1.features.BulkApi;
 import org.jclouds.openstack.swift.v1.features.ContainerApi;
+import org.jclouds.openstack.swift.v1.features.DynamicLargeObjectApi;
 import org.jclouds.openstack.swift.v1.features.ObjectApi;
 import org.jclouds.openstack.swift.v1.features.StaticLargeObjectApi;
 import org.jclouds.rest.annotations.Delegate;
@@ -67,4 +68,9 @@ public interface SwiftApi extends Closeable {
    @Path("/{containerName}")
    StaticLargeObjectApi getStaticLargeObjectApi(@EndpointParam(parser = RegionToEndpoint.class) String region,
          @PathParam("containerName") String containerName);
+
+   @Delegate
+   @Path("/{containerName}")
+   DynamicLargeObjectApi getDynamicLargeObjectApi(@EndpointParam(parser = RegionToEndpoint.class) String region,
+         @PathParam("containerName") String containerName);
 }

apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApi.java

@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+
+import java.util.Map;
+
+import javax.inject.Named;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+
+import org.jclouds.openstack.keystone.v2_0.filters.AuthenticateRequest;
+import org.jclouds.openstack.swift.v1.binders.BindMetadataToHeaders.BindObjectMetadataToHeaders;
+import org.jclouds.openstack.swift.v1.binders.BindToHeaders;
+import org.jclouds.openstack.swift.v1.domain.SwiftObject;
+import org.jclouds.openstack.swift.v1.functions.ETagHeader;
+import org.jclouds.rest.annotations.BinderParam;
+import org.jclouds.rest.annotations.Headers;
+import org.jclouds.rest.annotations.RequestFilters;
+import org.jclouds.rest.annotations.ResponseParser;
+
+import com.google.common.annotations.Beta;
+
+/**
+ * Provides access to the OpenStack Object Storage (Swift) Dynamic Large Object
+ * API features.
+ * <p/>
+ * This API is new to jclouds and hence is in Beta.
+ */
+@Beta
+@RequestFilters(AuthenticateRequest.class)
+@Consumes(APPLICATION_JSON)
+@Path("/{objectName}")
+public interface DynamicLargeObjectApi {
+   /**
+    * Creates or updates a dynamic large object's manifest.
+    *
+    * @param objectName
+    *           corresponds to {@link SwiftObject#getName()}.
+    * @param metadata
+    *           corresponds to {@link SwiftObject#getMetadata()}.
+    * @param headers
+    *           Binds the map to headers, without prefixing/escaping the header
+    *           name/key.
+    *
+    * @return {@link SwiftObject#getEtag()} of the object, which is the MD5
+    *         checksum of the concatenated ETag values of the {@code segments}.
+    *         
+    * @see {@code StaticLargeObjectApi}
+    */
+   @Deprecated
+   @Named("dynamicLargeObject:putManifest")
+   @PUT
+   @ResponseParser(ETagHeader.class)
+   @Headers(keys = "X-Object-Manifest", values = "{containerName}/{objectName}/")
+   String putManifest(@PathParam("objectName") String objectName,
+         @BinderParam(BindObjectMetadataToHeaders.class) Map<String, String> metadata,
+         @BinderParam(BindToHeaders.class) Map<String, String> headers);
+
+   /**
+    * Creates or updates a dynamic large object's manifest.
+    *
+    * @param objectName
+    *           corresponds to {@link SwiftObject#getName()}.
+    * @param metadata
+    *           corresponds to {@link SwiftObject#getMetadata()}.
+    *
+    * @return {@link SwiftObject#getEtag()} of the object, which is the etag
+    *         of 0 sized object.
+    *         
+    * @see {@code StaticLargeObjectApi}
+    */
+   @Deprecated
+   @Named("dynamicLargeObject:putManifest")
+   @PUT
+   @ResponseParser(ETagHeader.class)
+   @Headers(keys = "X-Object-Manifest", values = "{containerName}/{objectName}/")
+   String putManifest(@PathParam("objectName") String objectName,
+         @BinderParam(BindObjectMetadataToHeaders.class) Map<String, String> metadata);
+}

apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiLiveTest.java

@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jclouds.openstack.swift.v1.features;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.jclouds.io.Payloads.newByteSourcePayload;
+import static org.testng.Assert.assertNotNull;
+
+import java.io.IOException;
+
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.openstack.swift.v1.blobstore.RegionScopedBlobStoreContext;
+import org.jclouds.openstack.swift.v1.domain.ObjectList;
+import org.jclouds.openstack.swift.v1.domain.SwiftObject;
+import org.jclouds.openstack.swift.v1.internal.BaseSwiftApiLiveTest;
+import org.jclouds.openstack.swift.v1.options.ListContainerOptions;
+import org.jclouds.utils.TestUtils;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.ByteSource;
+
+@Test(groups = "live", testName = "DynamicLargeObjectApiLiveTest", singleThreaded = true)
+public class DynamicLargeObjectApiLiveTest extends BaseSwiftApiLiveTest {
+
+   private String defaultName = getClass().getSimpleName();
+   private static final ByteSource megOf1s = TestUtils.randomByteSource().slice(0, 1024 * 1024);
+   private static final ByteSource megOf2s = TestUtils.randomByteSource().slice(0, 1024 * 1024);
+   private String objectName = "myObject";
+
+   @Test
+   public void testReplaceManifest() throws Exception {
+      for (String regionId : regions) {
+         assertReplaceManifest(regionId, defaultName);
+         uploadLargeFile(regionId);
+      }
+   }
+
+   @SuppressWarnings("deprecation")
+   @Test
+   public void uploadLargeFile(String regionId) throws IOException, InterruptedException {
+      long total_size = 0;
+      RegionScopedBlobStoreContext ctx = RegionScopedBlobStoreContext.class.cast(view);
+      BlobStore blobStore = ctx.getBlobStore();
+      String defaultContainerName = getContainerName();
+      // configure the blobstore to use multipart uploading of the file
+      for (int partNumber = 0; partNumber < 3; partNumber++) {
+         String objName = String.format("%s/%s/%s", objectName, "dlo", partNumber);
+         String data = "data" + partNumber;
+         ByteSource payload = ByteSource.wrap(data.getBytes(Charsets.UTF_8));
+         Blob blob = blobStore.blobBuilder(objName)
+               .payload(payload)
+               .build();
+         String etag = blobStore.putBlob(defaultContainerName, blob);
+         assertNotNull(etag);
+         total_size += data.length();
+      }
+      getApi().getDynamicLargeObjectApi(regionId, defaultContainerName).putManifest(objectName,
+            ImmutableMap.of("myfoo", "Bar"));
+      SwiftObject bigObject = getApi().getObjectApi(regionId, defaultContainerName).get(objectName);
+      assertThat(bigObject.getETag()).isEqualTo("54bc1337d7a51660c40db39759cc1944");
+      assertThat(bigObject.getPayload().getContentMetadata().getContentLength()).isEqualTo(total_size);
+      assertThat(getApi().getContainerApi(regionId).get(defaultContainerName).getObjectCount()).isEqualTo(Long.valueOf(4));
+   }
+
+   @SuppressWarnings("deprecation")
+   protected void assertReplaceManifest(String regionId, String name) throws InterruptedException {
+      String containerName = getContainerName();
+      ObjectApi objectApi = getApi().getObjectApi(regionId, containerName);
+
+      String etag1s = objectApi.put(name + "/1", newByteSourcePayload(megOf1s));
+      awaitConsistency();
+      assertMegabyteAndETagMatches(regionId, containerName, name + "/1", etag1s);
+
+      String etag2s = objectApi.put(name + "/2", newByteSourcePayload(megOf2s));
+      awaitConsistency();
+      assertMegabyteAndETagMatches(regionId, containerName, name + "/2", etag2s);
+
+      awaitConsistency();
+      String etagOfEtags = getApi().getDynamicLargeObjectApi(regionId, containerName).putManifest(name,
+            ImmutableMap.of("myfoo", "Bar"));
+
+      assertNotNull(etagOfEtags);
+
+      awaitConsistency();
+
+      SwiftObject bigObject = getApi().getObjectApi(regionId, containerName).get(name);
+      assertThat(bigObject.getPayload().getContentMetadata().getContentLength()).isEqualTo(Long.valueOf(2 * 1024L * 1024L));
+      assertThat(bigObject.getMetadata()).isEqualTo(ImmutableMap.of("myfoo", "Bar"));
+      // segments are visible
+      assertThat(getApi().getContainerApi(regionId).get(containerName).getObjectCount()).isEqualTo(Long.valueOf(3));
+   }
+
+   protected void assertMegabyteAndETagMatches(String regionId, String containerName, String name, String etag1s) {
+      SwiftObject object1s = getApi().getObjectApi(regionId, containerName).get(name);
+      assertThat(object1s.getETag()).isEqualTo(etag1s);
+      assertThat(object1s.getPayload().getContentMetadata().getContentLength()).isEqualTo(Long.valueOf(1024L * 1024L));
+   }
+
+   protected void deleteAllObjectsInContainerDLO(String regionId, final String containerName) {
+       ObjectList objects = getApi().getObjectApi(regionId, containerName).list(new ListContainerOptions());
+      if (objects == null) {
+         return;
+      }
+      for (SwiftObject object : objects) {
+         String name = containerName + "/" + object.getName();
+         getApi().getObjectApi(regionId, containerName).delete(name);
+      }
+   }
+}

apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiMockTest.java

@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jclouds.openstack.swift.v1.features;
+
+import static com.google.common.net.HttpHeaders.ETAG;
+import static org.jclouds.openstack.swift.v1.reference.SwiftHeaders.OBJECT_METADATA_PREFIX;
+import static org.testng.Assert.assertEquals;
+
+import org.jclouds.io.Payloads;
+import org.jclouds.openstack.swift.v1.SwiftApi;
+import org.jclouds.openstack.v2_0.internal.BaseOpenStackMockTest;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.net.HttpHeaders;
+import com.squareup.okhttp.mockwebserver.MockResponse;
+import com.squareup.okhttp.mockwebserver.MockWebServer;
+import com.squareup.okhttp.mockwebserver.RecordedRequest;
+
+@Test(groups = "unit", testName = "DynamicLargeObjectApiMockTest")
+public final class DynamicLargeObjectApiMockTest extends BaseOpenStackMockTest<SwiftApi> {
+
+   String containerName = "myContainer";
+   String objectName = "myObjectTest";
+
+   @SuppressWarnings("deprecation")
+   @Test
+   public void uploadLargeFile() throws Exception {
+      MockWebServer server = mockOpenStackServer();
+      server.enqueue(addCommonHeaders(new MockResponse().setBody(stringFromResource("/access.json"))));
+      server.enqueue(new MockResponse().setBody("").addHeader(ETAG, "89d903bc35dede724fd52c51437ff5fd"));
+      server.enqueue(new MockResponse().setBody("").addHeader(ETAG, "d41d8cd98f00b204e9800998ecf8427e"));
+      server.enqueue(addCommonHeaders(new MockResponse().addHeader("X-Object-Manifest", "myContainer/myObject")));
+
+      try {
+         SwiftApi api = api(server.getUrl("/").toString(), "openstack-swift");
+         assertEquals(api.getObjectApi("DFW", containerName).put(objectName.concat("1"), Payloads.newPayload("data1")),
+               "89d903bc35dede724fd52c51437ff5fd");
+         assertEquals(api.getDynamicLargeObjectApi("DFW", containerName).putManifest(objectName,
+               ImmutableMap.of("MyFoo", "Bar"), ImmutableMap.of("MyFoo", "Bar")), "d41d8cd98f00b204e9800998ecf8427e");
+
+         assertEquals(server.getRequestCount(), 3);
+         assertAuthentication(server);
+
+         RecordedRequest uploadRequest = server.takeRequest();
+         assertEquals(uploadRequest.getRequestLine(),
+               "PUT /v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/myObjectTest1 HTTP/1.1");
+         assertEquals(new String(uploadRequest.getBody()), "data1");
+
+         RecordedRequest uploadRequestManifest = server.takeRequest();
+         assertRequest(uploadRequestManifest, "PUT",
+               "/v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/myObjectTest");
+         assertEquals(uploadRequestManifest.getHeader(OBJECT_METADATA_PREFIX + "MyFoo"), "Bar");
+
+      } finally {
+         server.shutdown();
+      }
+   }
+
+   @SuppressWarnings("deprecation")
+   public void testReplaceManifest() throws Exception {
+      MockWebServer server = mockOpenStackServer();
+      server.enqueue(addCommonHeaders(new MockResponse().setBody(stringFromResource("/access.json"))));
+      server.enqueue(addCommonHeaders(new MockResponse().addHeader(HttpHeaders.ETAG, "\"abcd\"")));
+      server.enqueue(addCommonHeaders(new MockResponse().addHeader("X-Object-Manifest", "myContainer/myObject")));
+
+      try {
+         SwiftApi api = api(server.getUrl("/").toString(), "openstack-swift");
+         assertEquals(api.getDynamicLargeObjectApi("DFW", "myContainer").putManifest("myObject",
+               ImmutableMap.of("MyFoo", "Bar"), ImmutableMap.of("MyFoo", "Bar")), "abcd");
+
+         assertEquals(server.getRequestCount(), 2);
+         assertAuthentication(server);
+
+         RecordedRequest replaceRequest = server.takeRequest();
+         assertRequest(replaceRequest, "PUT",
+               "/v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/myObject");
+         assertEquals(replaceRequest.getHeader(OBJECT_METADATA_PREFIX + "myfoo"), "Bar");
+      } finally {
+         server.shutdown();
+      }
+   }
+
+   @SuppressWarnings("deprecation")
+   public void testReplaceManifestUnicodeUTF8() throws Exception {
+      MockWebServer server = mockOpenStackServer();
+      server.enqueue(addCommonHeaders(new MockResponse().setBody(stringFromResource("/access.json"))));
+      server.enqueue(addCommonHeaders(new MockResponse().addHeader(HttpHeaders.ETAG, "\"abcd\"")));
+      server.enqueue(addCommonHeaders(new MockResponse().addHeader("X-Object-Manifest", "myContainer/myObject")));
+
+      try {
+         SwiftApi api = api(server.getUrl("/").toString(), "openstack-swift");
+         assertEquals(api.getDynamicLargeObjectApi("DFW", "myContainer").putManifest("unic₪de",
+               ImmutableMap.of("MyFoo", "Bar"), ImmutableMap.of("MyFoo", "Bar")), "abcd");
+
+         assertEquals(server.getRequestCount(), 2);
+         assertAuthentication(server);
+
+         RecordedRequest replaceRequest = server.takeRequest();
+         assertRequest(replaceRequest, "PUT",
+               "/v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/unic%E2%82%AAde");
+         assertEquals(replaceRequest.getHeader(OBJECT_METADATA_PREFIX + "myfoo"), "Bar");
+      } finally {
+         server.shutdown();
+      }
+   }
+}