Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-1304: B2 native prefix and delimiter 

Previously B2 emulated prefix and delimiter via client-side filtering.
Enabled by recent service additions.
This commit is contained in:
Andrew Gaul 2017-06-28 16:04:46 +01:00
parent 84653bac65
commit 914e3a6f85
6 changed files with 47 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
providers/b2/src/main/java/org/jclouds/b2/blobstore/B2BlobStore.java
View File

@ -29,6 +29,7 @@ import javax.inject.Inject;
import org.jclouds.b2.B2Api;
import org.jclouds.b2.B2ResponseException;
import org.jclouds.b2.domain.Action;
import org.jclouds.b2.domain.Authorization;
import org.jclouds.b2.domain.B2Object;
import org.jclouds.b2.domain.B2ObjectList;
@ -76,7 +77,6 @@ import org.jclouds.io.PayloadSlicer;
import org.jclouds.io.payloads.BaseMutableContentMetadata;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@ -191,53 +191,25 @@ public final class B2BlobStore extends BaseBlobStore {
Bucket bucket = getBucket(container);
int size = 0;
ImmutableList.Builder<StorageMetadata> builder = ImmutableList.builder();
Set<String> commonPrefixes = Sets.newHashSet();
String marker = options.getMarker();
while (true) {
B2ObjectList list = api.getObjectApi().listFileNames(bucket.bucketId(), marker, options.getMaxResults());
for (B2ObjectList.Entry entry : list.files()) {
// B2 does not support server-side filtering via prefix and delimiter so we emulate it on the client.
if (options.getPrefix() != null && !entry.fileName().startsWith(options.getPrefix())) {
continue;
B2ObjectList list = api.getObjectApi().listFileNames(bucket.bucketId(), options.getMarker(), options.getMaxResults(), options.getPrefix(), delimiter);
for (B2ObjectList.Entry entry : list.files()) {
if (entry.action() == Action.FOLDER) {
builder.add(new StorageMetadataImpl(StorageType.RELATIVE_PATH, null, entry.fileName(), null, null, null, null, entry.uploadTimestamp(), ImmutableMap.<String, String>of(), null));
} else if (options.isDetailed()) {
BlobMetadata metadata = blobMetadata(container, entry.fileName());
if (metadata != null) {
builder.add(metadata);
}
if (delimiter != null) {
String fileName = entry.fileName();
int index = entry.fileName().indexOf(delimiter, Strings.nullToEmpty(options.getPrefix()).length());
if (index != -1) {
String prefix = entry.fileName().substring(0, index + 1);
if (!commonPrefixes.contains(prefix)) {
commonPrefixes.add(prefix);
++size;
builder.add(new StorageMetadataImpl(StorageType.RELATIVE_PATH, null, prefix, null, null, null, null, null, ImmutableMap.<String, String>of(), null));
}
continue;
}
}
if (options.isDetailed()) {
BlobMetadata metadata = blobMetadata(container, entry.fileName());
if (metadata != null) {
++size;
builder.add(metadata);
}
} else {
Map<String, String> userMetadata = ImmutableMap.of();
ContentMetadata metadata = ContentMetadataBuilder.create()
.contentLength(entry.size())
.build();
++size;
builder.add(new BlobMetadataImpl(null, entry.fileName(), null, null, null, null, entry.uploadTimestamp(), userMetadata, null, container, metadata, entry.size()));
}
}
marker = list.nextFileName();
if (marker == null || options.getMaxResults() == null || size == options.getMaxResults()) {
break;
} else {
Map<String, String> userMetadata = ImmutableMap.of();
ContentMetadata metadata = ContentMetadataBuilder.create()
.contentLength(entry.size())
.build();
builder.add(new BlobMetadataImpl(null, entry.fileName(), null, null, null, null, entry.uploadTimestamp(), userMetadata, null, container, metadata, entry.size()));
}
}
return new PageSetImpl<StorageMetadata>(builder.build(), marker);
return new PageSetImpl<StorageMetadata>(builder.build(), list.nextFileName());
}
@Override

1
providers/b2/src/main/java/org/jclouds/b2/domain/Action.java
View File

@ -19,6 +19,7 @@ package org.jclouds.b2.domain;
import com.google.common.base.CaseFormat;
public enum Action {
FOLDER,
UPLOAD,
HIDE;

4
providers/b2/src/main/java/org/jclouds/b2/domain/B2ObjectList.java
View File

@ -39,13 +39,13 @@ public abstract class B2ObjectList {
@AutoValue
public abstract static class Entry {
public abstract Action action();
public abstract String fileId();
@Nullable public abstract String fileId();
public abstract String fileName();
public abstract long size();
public abstract Date uploadTimestamp();
@SerializedNames({"action", "fileId", "fileName", "size", "uploadTimestamp"})
public static Entry create(Action action, String fileId, String fileName, long size, long uploadTimestamp) {
public static Entry create(Action action, @Nullable String fileId, String fileName, long size, long uploadTimestamp) {
return new AutoValue_B2ObjectList_Entry(action, fileId, fileName, size, new Date(uploadTimestamp));
}
}

20
providers/b2/src/main/java/org/jclouds/b2/features/ObjectApi.java
View File

@ -120,6 +120,7 @@ public interface ObjectApi {
@Fallback(NullOnNotFoundOr404.class)
B2Object downloadFileByName(@PathParam("bucketName") String bucketName, @PathParam("fileName") String fileName, GetOptions options);
@Deprecated
@Named("b2_list_file_names")
@GET
@Path("/b2api/v1/b2_list_file_names")
@ -129,6 +130,16 @@ public interface ObjectApi {
@Produces(APPLICATION_JSON)
B2ObjectList listFileNames(@PayloadParam("bucketId") String bucketId, @PayloadParam("startFileName") @Nullable String startFileName, @PayloadParam("maxFileCount") @Nullable Integer maxFileCount);
@Named("b2_list_file_names")
@GET
@Path("/b2api/v1/b2_list_file_names")
@MapBinder(BindToJsonPayload.class)
@RequestFilters(RequestAuthorization.class)
@Consumes(APPLICATION_JSON)
@Produces(APPLICATION_JSON)
B2ObjectList listFileNames(@PayloadParam("bucketId") String bucketId, @PayloadParam("startFileName") @Nullable String startFileName, @PayloadParam("maxFileCount") @Nullable Integer maxFileCount, @PayloadParam("prefix") @Nullable String prefix, @Nullable @PayloadParam("delimiter") String delimiter);
@Deprecated
@Named("b2_list_file_versions")
@GET
@Path("/b2api/v1/b2_list_file_versions")
@ -138,6 +149,15 @@ public interface ObjectApi {
@Produces(APPLICATION_JSON)
B2ObjectList listFileVersions(@PayloadParam("bucketId") String bucketId, @PayloadParam("startFileId") @Nullable String startFileId, @PayloadParam("startFileName") @Nullable String startFileName, @PayloadParam("maxFileCount") @Nullable Integer maxFileCount);
@Named("b2_list_file_versions")
@GET
@Path("/b2api/v1/b2_list_file_versions")
@MapBinder(BindToJsonPayload.class)
@RequestFilters(RequestAuthorization.class)
@Consumes(APPLICATION_JSON)
@Produces(APPLICATION_JSON)
B2ObjectList listFileVersions(@PayloadParam("bucketId") String bucketId, @PayloadParam("startFileId") @Nullable String startFileId, @PayloadParam("startFileName") @Nullable String startFileName, @PayloadParam("maxFileCount") @Nullable Integer maxFileCount, @PayloadParam("prefix") @Nullable String prefix, @PayloadParam("delimiter") @Nullable String delimiter);
@Named("b2_hide_file")
@POST
@Path("/b2api/v1/b2_hide_file")

12
providers/b2/src/test/java/org/jclouds/b2/features/ObjectApiLiveTest.java
View File

@ -195,7 +195,7 @@ public final class ObjectApiLiveTest extends BaseB2ApiLiveTest {
uploadFiles.add(createFile(objectApi, response.bucketId(), "file" + i));
}
B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null);
B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null, null, null);
assertThat(list.files()).hasSize(numFiles);
} finally {
for (UploadFileResponse uploadFile : uploadFiles.build()) {
@ -218,10 +218,10 @@ public final class ObjectApiLiveTest extends BaseB2ApiLiveTest {
uploadFiles.add(createFile(objectApi, response.bucketId(), "file"));
}
B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null);
B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null, null, null);
assertThat(list.files()).hasSize(1);
list = objectApi.listFileVersions(response.bucketId(), null, null, null);
list = objectApi.listFileVersions(response.bucketId(), null, null, null, null, null);
assertThat(list.files()).hasSize(numFiles);
} finally {
for (UploadFileResponse uploadFile : uploadFiles.build()) {
@ -243,15 +243,15 @@ public final class ObjectApiLiveTest extends BaseB2ApiLiveTest {
try {
uploadFile = createFile(objectApi, response.bucketId(), fileName);
B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null);
B2ObjectList list = objectApi.listFileNames(response.bucketId(), null, null, null, null);
assertThat(list.files()).hasSize(1);
hideFile = objectApi.hideFile(response.bucketId(), fileName);
list = objectApi.listFileNames(response.bucketId(), null, null);
list = objectApi.listFileNames(response.bucketId(), null, null, null, null);
assertThat(list.files()).isEmpty();
list = objectApi.listFileVersions(response.bucketId(), null, null, null);
list = objectApi.listFileVersions(response.bucketId(), null, null, null, null, null);
assertThat(list.files()).hasSize(2);
} finally {
if (hideFile != null) {

4
providers/b2/src/test/java/org/jclouds/b2/features/ObjectApiMockTest.java
View File

@ -340,7 +340,7 @@ public final class ObjectApiMockTest {
ObjectApi api = api(server.getUrl("/").toString(), "b2").getObjectApi();
String accountId = "d522aa47a10f";
B2ObjectList list = api.listFileNames(BUCKET_ID, null, null);
B2ObjectList list = api.listFileNames(BUCKET_ID, null, null, null, null);
assertThat(list.nextFileName()).isNull();
assertThat(list.files()).hasSize(2);
@ -376,7 +376,7 @@ public final class ObjectApiMockTest {
ObjectApi api = api(server.getUrl("/").toString(), "b2").getObjectApi();
String accountId = "d522aa47a10f";
B2ObjectList list = api.listFileVersions(BUCKET_ID, null, null, null);
B2ObjectList list = api.listFileVersions(BUCKET_ID, null, null, null, null, null);
assertThat(list.nextFileId()).isEqualTo("4_z27c88f1d182b150646ff0b16_f100920ddab886247_d20150809_m232316_c100_v0009990_t0003");
assertThat(list.nextFileName()).isEqualTo("files/world.txt");