From 914e3a6f856582049dcaaf035ef38aca40a5dd33 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Wed, 28 Jun 2017 16:04:46 +0100 Subject: [PATCH] JCLOUDS-1304: B2 native prefix and delimiter Previously B2 emulated prefix and delimiter via client-side filtering. Enabled by recent service additions. --- .../org/jclouds/b2/blobstore/B2BlobStore.java | 60 +++++-------------- .../java/org/jclouds/b2/domain/Action.java | 1 + .../org/jclouds/b2/domain/B2ObjectList.java | 4 +- .../org/jclouds/b2/features/ObjectApi.java | 20 +++++++ .../b2/features/ObjectApiLiveTest.java | 12 ++-- .../b2/features/ObjectApiMockTest.java | 4 +- 6 files changed, 47 insertions(+), 54 deletions(-) diff --git a/providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java b/providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java index b8d046c029..16b7860450 100644 --- a/providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java +++ b/providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java @@ -29,6 +29,7 @@ import javax.inject.Inject; import org.jclouds.b2.B2Api; import org.jclouds.b2.B2ResponseException; +import org.jclouds.b2.domain.Action; import org.jclouds.b2.domain.Authorization; import org.jclouds.b2.domain.B2Object; import org.jclouds.b2.domain.B2ObjectList; @@ -76,7 +77,6 @@ import org.jclouds.io.PayloadSlicer; import org.jclouds.io.payloads.BaseMutableContentMetadata; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -191,53 +191,25 @@ public final class B2BlobStore extends BaseBlobStore { Bucket bucket = getBucket(container); - int size = 0; ImmutableList.Builder builder = ImmutableList.builder(); - Set commonPrefixes = Sets.newHashSet(); - String marker = options.getMarker(); - while (true) { - B2ObjectList list = api.getObjectApi().listFileNames(bucket.bucketId(), marker, options.getMaxResults()); - for (B2ObjectList.Entry entry : list.files()) { - // B2 does not support server-side filtering via prefix and delimiter so we emulate it on the client. - if (options.getPrefix() != null && !entry.fileName().startsWith(options.getPrefix())) { - continue; + B2ObjectList list = api.getObjectApi().listFileNames(bucket.bucketId(), options.getMarker(), options.getMaxResults(), options.getPrefix(), delimiter); + for (B2ObjectList.Entry entry : list.files()) { + if (entry.action() == Action.FOLDER) { + builder.add(new StorageMetadataImpl(StorageType.RELATIVE_PATH, null, entry.fileName(), null, null, null, null, entry.uploadTimestamp(), ImmutableMap.of(), null)); + } else if (options.isDetailed()) { + BlobMetadata metadata = blobMetadata(container, entry.fileName()); + if (metadata != null) { + builder.add(metadata); } - - if (delimiter != null) { - String fileName = entry.fileName(); - int index = entry.fileName().indexOf(delimiter, Strings.nullToEmpty(options.getPrefix()).length()); - if (index != -1) { - String prefix = entry.fileName().substring(0, index + 1); - if (!commonPrefixes.contains(prefix)) { - commonPrefixes.add(prefix); - ++size; - builder.add(new StorageMetadataImpl(StorageType.RELATIVE_PATH, null, prefix, null, null, null, null, null, ImmutableMap.of(), null)); - } - continue; - } - } - - if (options.isDetailed()) { - BlobMetadata metadata = blobMetadata(container, entry.fileName()); - if (metadata != null) { - ++size; - builder.add(metadata); - } - } else { - Map userMetadata = ImmutableMap.of(); - ContentMetadata metadata = ContentMetadataBuilder.create() - .contentLength(entry.size()) - .build(); - ++size; - builder.add(new BlobMetadataImpl(null, entry.fileName(), null, null, null, null, entry.uploadTimestamp(), userMetadata, null, container, metadata, entry.size())); - } - } - marker = list.nextFileName(); - if (marker == null || options.getMaxResults() == null || size == options.getMaxResults()) { - break; + } else { + Map userMetadata = ImmutableMap.of(); + ContentMetadata metadata = ContentMetadataBuilder.create() + .contentLength(entry.size()) + .build(); + builder.add(new BlobMetadataImpl(null, entry.fileName(), null, null, null, null, entry.uploadTimestamp(), userMetadata, null, container, metadata, entry.size())); } } - return new PageSetImpl(builder.build(), marker); + return new PageSetImpl(builder.build(), list.nextFileName()); } @Override diff --git a/providers/b2/src/main/java/org/jclouds/b2/domain/Action.java b/providers/b2/src/main/java/org/jclouds/b2/domain/Action.java index bd6c852b1d..3dd6e44a6c 100644 --- a/providers/b2/src/main/java/org/jclouds/b2/domain/Action.java +++ b/providers/b2/src/main/java/org/jclouds/b2/domain/Action.java @@ -19,6 +19,7 @@ package org.jclouds.b2.domain; import com.google.common.base.CaseFormat; public enum Action { + FOLDER, UPLOAD, HIDE; diff --git a/providers/b2/src/main/java/org/jclouds/b2/domain/B2ObjectList.java b/providers/b2/src/main/java/org/jclouds/b2/domain/B2ObjectList.java index 780ab75f77..af256dd668 100644 --- a/providers/b2/src/main/java/org/jclouds/b2/domain/B2ObjectList.java +++ b/providers/b2/src/main/java/org/jclouds/b2/domain/B2ObjectList.java @@ -39,13 +39,13 @@ public abstract class B2ObjectList { @AutoValue public abstract static class Entry { public abstract Action action(); - public abstract String fileId(); + @Nullable public abstract String fileId(); public abstract String fileName(); public abstract long size(); public abstract Date uploadTimestamp(); @SerializedNames({"action", "fileId", "fileName", "size", "uploadTimestamp"}) - public static Entry create(Action action, String fileId, String fileName, long size, long uploadTimestamp) { + public static Entry create(Action action, @Nullable String fileId, String fileName, long size, long uploadTimestamp) { return new AutoValue_B2ObjectList_Entry(action, fileId, fileName, size, new Date(uploadTimestamp)); } } diff --git a/providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java b/providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java index c8379ebb6c..5da8f858c9 100644 --- a/providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java +++ b/providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java @@ -120,6 +120,7 @@ public interface ObjectApi { @Fallback(NullOnNotFoundOr404.class) B2Object downloadFileByName(@PathParam("bucketName") String bucketName, @PathParam("fileName") String fileName, GetOptions options); + @Deprecated @Named("b2_list_file_names") @GET @Path("/b2api/v1/b2_list_file_names") @@ -129,6 +130,16 @@ public interface ObjectApi { @Produces(APPLICATION_JSON) B2ObjectList listFileNames(@PayloadParam("bucketId") String bucketId, @PayloadParam("startFileName") @Nullable String startFileName, @PayloadParam("maxFileCount") @Nullable Integer maxFileCount); + @Named("b2_list_file_names") + @GET + @Path("/b2api/v1/b2_list_file_names") + @MapBinder(BindToJsonPayload.class) + @RequestFilters(RequestAuthorization.class) + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + B2ObjectList listFileNames(@PayloadParam("bucketId") String bucketId, @PayloadParam("startFileName") @Nullable String startFileName, @PayloadParam("maxFileCount") @Nullable Integer maxFileCount, @PayloadParam("prefix") @Nullable String prefix, @Nullable @PayloadParam("delimiter") String delimiter); + + @Deprecated @Named("b2_list_file_versions") @GET @Path("/b2api/v1/b2_list_file_versions") @@ -138,6 +149,15 @@ public interface ObjectApi { @Produces(APPLICATION_JSON) B2ObjectList listFileVersions(@PayloadParam("bucketId") String bucketId, @PayloadParam("startFileId") @Nullable String startFileId, @PayloadParam("startFileName") @Nullable String startFileName, @PayloadParam("maxFileCount") @Nullable Integer maxFileCount); + @Named("b2_list_file_versions") + @GET + @Path("/b2api/v1/b2_list_file_versions") + @MapBinder(BindToJsonPayload.class) + @RequestFilters(RequestAuthorization.class) + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + B2ObjectList listFileVersions(@PayloadParam("bucketId") String bucketId, @PayloadParam("startFileId") @Nullable String startFileId, @PayloadParam("startFileName") @Nullable String startFileName, @PayloadParam("maxFileCount") @Nullable Integer maxFileCount, @PayloadParam("prefix") @Nullable String prefix, @PayloadParam("delimiter") @Nullable String delimiter); + @Named("b2_hide_file") @POST @Path("/b2api/v1/b2_hide_file") diff --git a/providers/b2/src/test/java/org/jclouds/b2/features/ObjectApiLiveTest.java b/providers/b2/src/test/java/org/jclouds/b2/features/ObjectApiLiveTest.java index 3de3cdb0f4..9ee6538fa6 100644 --- a/providers/b2/src/test/java/org/jclouds/b2/features/ObjectApiLiveTest.java +++ b/providers/b2/src/test/java/org/jclouds/b2/features/ObjectApiLiveTest.java @@ -195,7 +195,7 @@ public final class ObjectApiLiveTest extends BaseB2ApiLiveTest { uploadFiles.add(createFile(objectApi, response.bucketId(), "file" + i)); } - B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null); + B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null, null, null); assertThat(list.files()).hasSize(numFiles); } finally { for (UploadFileResponse uploadFile : uploadFiles.build()) { @@ -218,10 +218,10 @@ public final class ObjectApiLiveTest extends BaseB2ApiLiveTest { uploadFiles.add(createFile(objectApi, response.bucketId(), "file")); } - B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null); + B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null, null, null); assertThat(list.files()).hasSize(1); - list = objectApi.listFileVersions(response.bucketId(), null, null, null); + list = objectApi.listFileVersions(response.bucketId(), null, null, null, null, null); assertThat(list.files()).hasSize(numFiles); } finally { for (UploadFileResponse uploadFile : uploadFiles.build()) { @@ -243,15 +243,15 @@ public final class ObjectApiLiveTest extends BaseB2ApiLiveTest { try { uploadFile = createFile(objectApi, response.bucketId(), fileName); - B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null); + B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null, null, null); assertThat(list.files()).hasSize(1); hideFile = objectApi.hideFile(response.bucketId(), fileName); - list = objectApi.listFileNames(response.bucketId(), null, null); + list = objectApi.listFileNames(response.bucketId(), null, null, null, null); assertThat(list.files()).isEmpty(); - list = objectApi.listFileVersions(response.bucketId(), null, null, null); + list = objectApi.listFileVersions(response.bucketId(), null, null, null, null, null); assertThat(list.files()).hasSize(2); } finally { if (hideFile != null) { diff --git a/providers/b2/src/test/java/org/jclouds/b2/features/ObjectApiMockTest.java b/providers/b2/src/test/java/org/jclouds/b2/features/ObjectApiMockTest.java index 8524210d37..43ac19b0db 100644 --- a/providers/b2/src/test/java/org/jclouds/b2/features/ObjectApiMockTest.java +++ b/providers/b2/src/test/java/org/jclouds/b2/features/ObjectApiMockTest.java @@ -340,7 +340,7 @@ public final class ObjectApiMockTest { ObjectApi api = api(server.getUrl("/").toString(), "b2").getObjectApi(); String accountId = "d522aa47a10f"; - B2ObjectList list = api.listFileNames(BUCKET_ID, null, null); + B2ObjectList list = api.listFileNames(BUCKET_ID, null, null, null, null); assertThat(list.nextFileName()).isNull(); assertThat(list.files()).hasSize(2); @@ -376,7 +376,7 @@ public final class ObjectApiMockTest { ObjectApi api = api(server.getUrl("/").toString(), "b2").getObjectApi(); String accountId = "d522aa47a10f"; - B2ObjectList list = api.listFileVersions(BUCKET_ID, null, null, null); + B2ObjectList list = api.listFileVersions(BUCKET_ID, null, null, null, null, null); assertThat(list.nextFileId()).isEqualTo("4_z27c88f1d182b150646ff0b16_f100920ddab886247_d20150809_m232316_c100_v0009990_t0003"); assertThat(list.nextFileName()).isEqualTo("files/world.txt");