From eaf3c779dce63ddc808d5ff8fe3ed9aade10b2f2 Mon Sep 17 00:00:00 2001 From: Archana Chinnaiah Date: Tue, 23 May 2017 17:14:16 +0530 Subject: [PATCH] JCLOUDS-1281: Swift dynamic large objects --- .../jclouds/openstack/swift/v1/SwiftApi.java | 6 + .../v1/features/DynamicLargeObjectApi.java | 97 ++++++++++++++ .../DynamicLargeObjectApiLiveTest.java | 126 ++++++++++++++++++ .../DynamicLargeObjectApiMockTest.java | 122 +++++++++++++++++ 4 files changed, 351 insertions(+) create mode 100644 apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApi.java create mode 100644 apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiLiveTest.java create mode 100644 apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiMockTest.java diff --git a/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java index 1d4d1d0ee5..76a8e9ffbf 100644 --- a/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java +++ b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/SwiftApi.java @@ -27,6 +27,7 @@ import org.jclouds.location.functions.RegionToEndpoint; import org.jclouds.openstack.swift.v1.features.AccountApi; import org.jclouds.openstack.swift.v1.features.BulkApi; import org.jclouds.openstack.swift.v1.features.ContainerApi; +import org.jclouds.openstack.swift.v1.features.DynamicLargeObjectApi; import org.jclouds.openstack.swift.v1.features.ObjectApi; import org.jclouds.openstack.swift.v1.features.StaticLargeObjectApi; import org.jclouds.rest.annotations.Delegate; @@ -67,4 +68,9 @@ public interface SwiftApi extends Closeable { @Path("/{containerName}") StaticLargeObjectApi getStaticLargeObjectApi(@EndpointParam(parser = RegionToEndpoint.class) String region, @PathParam("containerName") String containerName); + + @Delegate + @Path("/{containerName}") + DynamicLargeObjectApi getDynamicLargeObjectApi(@EndpointParam(parser = RegionToEndpoint.class) String region, + @PathParam("containerName") String containerName); } diff --git a/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApi.java b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApi.java new file mode 100644 index 0000000000..044e457393 --- /dev/null +++ b/apis/openstack-swift/src/main/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApi.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.v1.features; + +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; + +import java.util.Map; + +import javax.inject.Named; +import javax.ws.rs.Consumes; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; + +import org.jclouds.openstack.keystone.v2_0.filters.AuthenticateRequest; +import org.jclouds.openstack.swift.v1.binders.BindMetadataToHeaders.BindObjectMetadataToHeaders; +import org.jclouds.openstack.swift.v1.binders.BindToHeaders; +import org.jclouds.openstack.swift.v1.domain.SwiftObject; +import org.jclouds.openstack.swift.v1.functions.ETagHeader; +import org.jclouds.rest.annotations.BinderParam; +import org.jclouds.rest.annotations.Headers; +import org.jclouds.rest.annotations.RequestFilters; +import org.jclouds.rest.annotations.ResponseParser; + +import com.google.common.annotations.Beta; + +/** + * Provides access to the OpenStack Object Storage (Swift) Dynamic Large Object + * API features. + *

+ * This API is new to jclouds and hence is in Beta. + */ +@Beta +@RequestFilters(AuthenticateRequest.class) +@Consumes(APPLICATION_JSON) +@Path("/{objectName}") +public interface DynamicLargeObjectApi { + /** + * Creates or updates a dynamic large object's manifest. + * + * @param objectName + * corresponds to {@link SwiftObject#getName()}. + * @param metadata + * corresponds to {@link SwiftObject#getMetadata()}. + * @param headers + * Binds the map to headers, without prefixing/escaping the header + * name/key. + * + * @return {@link SwiftObject#getEtag()} of the object, which is the MD5 + * checksum of the concatenated ETag values of the {@code segments}. + * + * @see {@code StaticLargeObjectApi} + */ + @Deprecated + @Named("dynamicLargeObject:putManifest") + @PUT + @ResponseParser(ETagHeader.class) + @Headers(keys = "X-Object-Manifest", values = "{containerName}/{objectName}/") + String putManifest(@PathParam("objectName") String objectName, + @BinderParam(BindObjectMetadataToHeaders.class) Map metadata, + @BinderParam(BindToHeaders.class) Map headers); + + /** + * Creates or updates a dynamic large object's manifest. + * + * @param objectName + * corresponds to {@link SwiftObject#getName()}. + * @param metadata + * corresponds to {@link SwiftObject#getMetadata()}. + * + * @return {@link SwiftObject#getEtag()} of the object, which is the etag + * of 0 sized object. + * + * @see {@code StaticLargeObjectApi} + */ + @Deprecated + @Named("dynamicLargeObject:putManifest") + @PUT + @ResponseParser(ETagHeader.class) + @Headers(keys = "X-Object-Manifest", values = "{containerName}/{objectName}/") + String putManifest(@PathParam("objectName") String objectName, + @BinderParam(BindObjectMetadataToHeaders.class) Map metadata); +} diff --git a/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiLiveTest.java b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiLiveTest.java new file mode 100644 index 0000000000..aaf37d3b10 --- /dev/null +++ b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiLiveTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jclouds.openstack.swift.v1.features; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.jclouds.io.Payloads.newByteSourcePayload; +import static org.testng.Assert.assertNotNull; + +import java.io.IOException; + +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.openstack.swift.v1.blobstore.RegionScopedBlobStoreContext; +import org.jclouds.openstack.swift.v1.domain.ObjectList; +import org.jclouds.openstack.swift.v1.domain.SwiftObject; +import org.jclouds.openstack.swift.v1.internal.BaseSwiftApiLiveTest; +import org.jclouds.openstack.swift.v1.options.ListContainerOptions; +import org.jclouds.utils.TestUtils; +import org.testng.annotations.Test; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.ByteSource; + +@Test(groups = "live", testName = "DynamicLargeObjectApiLiveTest", singleThreaded = true) +public class DynamicLargeObjectApiLiveTest extends BaseSwiftApiLiveTest { + + private String defaultName = getClass().getSimpleName(); + private static final ByteSource megOf1s = TestUtils.randomByteSource().slice(0, 1024 * 1024); + private static final ByteSource megOf2s = TestUtils.randomByteSource().slice(0, 1024 * 1024); + private String objectName = "myObject"; + + @Test + public void testReplaceManifest() throws Exception { + for (String regionId : regions) { + assertReplaceManifest(regionId, defaultName); + uploadLargeFile(regionId); + } + } + + @SuppressWarnings("deprecation") + @Test + public void uploadLargeFile(String regionId) throws IOException, InterruptedException { + long total_size = 0; + RegionScopedBlobStoreContext ctx = RegionScopedBlobStoreContext.class.cast(view); + BlobStore blobStore = ctx.getBlobStore(); + String defaultContainerName = getContainerName(); + // configure the blobstore to use multipart uploading of the file + for (int partNumber = 0; partNumber < 3; partNumber++) { + String objName = String.format("%s/%s/%s", objectName, "dlo", partNumber); + String data = "data" + partNumber; + ByteSource payload = ByteSource.wrap(data.getBytes(Charsets.UTF_8)); + Blob blob = blobStore.blobBuilder(objName) + .payload(payload) + .build(); + String etag = blobStore.putBlob(defaultContainerName, blob); + assertNotNull(etag); + total_size += data.length(); + } + getApi().getDynamicLargeObjectApi(regionId, defaultContainerName).putManifest(objectName, + ImmutableMap.of("myfoo", "Bar")); + SwiftObject bigObject = getApi().getObjectApi(regionId, defaultContainerName).get(objectName); + assertThat(bigObject.getETag()).isEqualTo("54bc1337d7a51660c40db39759cc1944"); + assertThat(bigObject.getPayload().getContentMetadata().getContentLength()).isEqualTo(total_size); + assertThat(getApi().getContainerApi(regionId).get(defaultContainerName).getObjectCount()).isEqualTo(Long.valueOf(4)); + } + + @SuppressWarnings("deprecation") + protected void assertReplaceManifest(String regionId, String name) throws InterruptedException { + String containerName = getContainerName(); + ObjectApi objectApi = getApi().getObjectApi(regionId, containerName); + + String etag1s = objectApi.put(name + "/1", newByteSourcePayload(megOf1s)); + awaitConsistency(); + assertMegabyteAndETagMatches(regionId, containerName, name + "/1", etag1s); + + String etag2s = objectApi.put(name + "/2", newByteSourcePayload(megOf2s)); + awaitConsistency(); + assertMegabyteAndETagMatches(regionId, containerName, name + "/2", etag2s); + + awaitConsistency(); + String etagOfEtags = getApi().getDynamicLargeObjectApi(regionId, containerName).putManifest(name, + ImmutableMap.of("myfoo", "Bar")); + + assertNotNull(etagOfEtags); + + awaitConsistency(); + + SwiftObject bigObject = getApi().getObjectApi(regionId, containerName).get(name); + assertThat(bigObject.getPayload().getContentMetadata().getContentLength()).isEqualTo(Long.valueOf(2 * 1024L * 1024L)); + assertThat(bigObject.getMetadata()).isEqualTo(ImmutableMap.of("myfoo", "Bar")); + // segments are visible + assertThat(getApi().getContainerApi(regionId).get(containerName).getObjectCount()).isEqualTo(Long.valueOf(3)); + } + + protected void assertMegabyteAndETagMatches(String regionId, String containerName, String name, String etag1s) { + SwiftObject object1s = getApi().getObjectApi(regionId, containerName).get(name); + assertThat(object1s.getETag()).isEqualTo(etag1s); + assertThat(object1s.getPayload().getContentMetadata().getContentLength()).isEqualTo(Long.valueOf(1024L * 1024L)); + } + + protected void deleteAllObjectsInContainerDLO(String regionId, final String containerName) { + ObjectList objects = getApi().getObjectApi(regionId, containerName).list(new ListContainerOptions()); + if (objects == null) { + return; + } + for (SwiftObject object : objects) { + String name = containerName + "/" + object.getName(); + getApi().getObjectApi(regionId, containerName).delete(name); + } + } +} diff --git a/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiMockTest.java b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiMockTest.java new file mode 100644 index 0000000000..c0420ee648 --- /dev/null +++ b/apis/openstack-swift/src/test/java/org/jclouds/openstack/swift/v1/features/DynamicLargeObjectApiMockTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jclouds.openstack.swift.v1.features; + +import static com.google.common.net.HttpHeaders.ETAG; +import static org.jclouds.openstack.swift.v1.reference.SwiftHeaders.OBJECT_METADATA_PREFIX; +import static org.testng.Assert.assertEquals; + +import org.jclouds.io.Payloads; +import org.jclouds.openstack.swift.v1.SwiftApi; +import org.jclouds.openstack.v2_0.internal.BaseOpenStackMockTest; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableMap; +import com.google.common.net.HttpHeaders; +import com.squareup.okhttp.mockwebserver.MockResponse; +import com.squareup.okhttp.mockwebserver.MockWebServer; +import com.squareup.okhttp.mockwebserver.RecordedRequest; + +@Test(groups = "unit", testName = "DynamicLargeObjectApiMockTest") +public final class DynamicLargeObjectApiMockTest extends BaseOpenStackMockTest { + + String containerName = "myContainer"; + String objectName = "myObjectTest"; + + @SuppressWarnings("deprecation") + @Test + public void uploadLargeFile() throws Exception { + MockWebServer server = mockOpenStackServer(); + server.enqueue(addCommonHeaders(new MockResponse().setBody(stringFromResource("/access.json")))); + server.enqueue(new MockResponse().setBody("").addHeader(ETAG, "89d903bc35dede724fd52c51437ff5fd")); + server.enqueue(new MockResponse().setBody("").addHeader(ETAG, "d41d8cd98f00b204e9800998ecf8427e")); + server.enqueue(addCommonHeaders(new MockResponse().addHeader("X-Object-Manifest", "myContainer/myObject"))); + + try { + SwiftApi api = api(server.getUrl("/").toString(), "openstack-swift"); + assertEquals(api.getObjectApi("DFW", containerName).put(objectName.concat("1"), Payloads.newPayload("data1")), + "89d903bc35dede724fd52c51437ff5fd"); + assertEquals(api.getDynamicLargeObjectApi("DFW", containerName).putManifest(objectName, + ImmutableMap.of("MyFoo", "Bar"), ImmutableMap.of("MyFoo", "Bar")), "d41d8cd98f00b204e9800998ecf8427e"); + + assertEquals(server.getRequestCount(), 3); + assertAuthentication(server); + + RecordedRequest uploadRequest = server.takeRequest(); + assertEquals(uploadRequest.getRequestLine(), + "PUT /v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/myObjectTest1 HTTP/1.1"); + assertEquals(new String(uploadRequest.getBody()), "data1"); + + RecordedRequest uploadRequestManifest = server.takeRequest(); + assertRequest(uploadRequestManifest, "PUT", + "/v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/myObjectTest"); + assertEquals(uploadRequestManifest.getHeader(OBJECT_METADATA_PREFIX + "MyFoo"), "Bar"); + + } finally { + server.shutdown(); + } + } + + @SuppressWarnings("deprecation") + public void testReplaceManifest() throws Exception { + MockWebServer server = mockOpenStackServer(); + server.enqueue(addCommonHeaders(new MockResponse().setBody(stringFromResource("/access.json")))); + server.enqueue(addCommonHeaders(new MockResponse().addHeader(HttpHeaders.ETAG, "\"abcd\""))); + server.enqueue(addCommonHeaders(new MockResponse().addHeader("X-Object-Manifest", "myContainer/myObject"))); + + try { + SwiftApi api = api(server.getUrl("/").toString(), "openstack-swift"); + assertEquals(api.getDynamicLargeObjectApi("DFW", "myContainer").putManifest("myObject", + ImmutableMap.of("MyFoo", "Bar"), ImmutableMap.of("MyFoo", "Bar")), "abcd"); + + assertEquals(server.getRequestCount(), 2); + assertAuthentication(server); + + RecordedRequest replaceRequest = server.takeRequest(); + assertRequest(replaceRequest, "PUT", + "/v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/myObject"); + assertEquals(replaceRequest.getHeader(OBJECT_METADATA_PREFIX + "myfoo"), "Bar"); + } finally { + server.shutdown(); + } + } + + @SuppressWarnings("deprecation") + public void testReplaceManifestUnicodeUTF8() throws Exception { + MockWebServer server = mockOpenStackServer(); + server.enqueue(addCommonHeaders(new MockResponse().setBody(stringFromResource("/access.json")))); + server.enqueue(addCommonHeaders(new MockResponse().addHeader(HttpHeaders.ETAG, "\"abcd\""))); + server.enqueue(addCommonHeaders(new MockResponse().addHeader("X-Object-Manifest", "myContainer/myObject"))); + + try { + SwiftApi api = api(server.getUrl("/").toString(), "openstack-swift"); + assertEquals(api.getDynamicLargeObjectApi("DFW", "myContainer").putManifest("unic₪de", + ImmutableMap.of("MyFoo", "Bar"), ImmutableMap.of("MyFoo", "Bar")), "abcd"); + + assertEquals(server.getRequestCount(), 2); + assertAuthentication(server); + + RecordedRequest replaceRequest = server.takeRequest(); + assertRequest(replaceRequest, "PUT", + "/v1/MossoCloudFS_5bcf396e-39dd-45ff-93a1-712b9aba90a9/myContainer/unic%E2%82%AAde"); + assertEquals(replaceRequest.getHeader(OBJECT_METADATA_PREFIX + "myfoo"), "Bar"); + } finally { + server.shutdown(); + } + } +}