Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-654: Add object size to StorageMetadata 

This allows callers to read the content length during container
listing.  Tested against: atmosonline, aws-s3, azureblob, filesystem,
and transient.  Intentionally not implemented for legacy swift
provider.
This commit is contained in:
Andrew Gaul 2014-08-05 17:35:30 -07:00
parent e183d9e651
commit fae097e144
21 changed files with 125 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apis/atmos/src/main/java/org/jclouds/atmos/AtmosClient.java
View File

@ -88,6 +88,7 @@ public interface AtmosClient extends Closeable {
@ResponseParser(ParseDirectoryListFromContentAndHeaders.class)
@Fallback(ThrowContainerNotFoundOn404.class)
@Consumes(MediaType.TEXT_XML)
@Headers(keys = "x-emc-include-meta", values = "1")
BoundedSet<? extends DirectoryEntry> listDirectory(
@PathParam("directoryName") String directoryName, ListOptions... options);

9
apis/atmos/src/main/java/org/jclouds/atmos/blobstore/functions/DirectoryEntryListToResourceMetadataList.java
View File

@ -26,6 +26,7 @@ import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.domain.StorageType;
import org.jclouds.blobstore.domain.internal.BlobMetadataImpl;
import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl;
import org.jclouds.blobstore.domain.internal.PageSetImpl;
import org.jclouds.blobstore.domain.internal.StorageMetadataImpl;
import org.jclouds.domain.Location;
@ -56,10 +57,14 @@ public class DirectoryEntryListToResourceMetadataList implements
if (type == StorageType.FOLDER)
return new StorageMetadataImpl(type, from.getObjectID(), from.getObjectName(), defaultLocation
.get(), null, null, null, null, ImmutableMap.<String, String>of());
else
return new BlobMetadataImpl(from.getObjectID(), from.getObjectName(), defaultLocation.get(),
else {
BlobMetadataImpl metadata = new BlobMetadataImpl(from.getObjectID(), from.getObjectName(), defaultLocation.get(),
null, null, null, null, ImmutableMap.<String, String>of(), null,
null, new BaseMutableContentMetadata());
MutableBlobMetadataImpl mutable = new MutableBlobMetadataImpl(metadata);
mutable.setSize(from.getSize());
return mutable;
}
}
}), from.getToken());

39
apis/atmos/src/main/java/org/jclouds/atmos/domain/DirectoryEntry.java
View File

@ -16,6 +16,8 @@
*/
package org.jclouds.atmos.domain;
import com.google.common.base.Objects;
/**
* Metadata of a Atmos Online object
*/
@ -23,11 +25,13 @@ public class DirectoryEntry implements Comparable<DirectoryEntry> {
private final String objectid;
private final FileType type;
private final String objname;
private final long size;
public DirectoryEntry(String objectid, FileType type, String objname) {
public DirectoryEntry(String objectid, FileType type, String objname, long size) {
this.objectid = objectid;
this.objname = objname;
this.type = type;
this.size = size;
}
public String getObjectID() {
@ -42,6 +46,10 @@ public class DirectoryEntry implements Comparable<DirectoryEntry> {
return type;
}
public long getSize() {
return size;
}
public int compareTo(DirectoryEntry o) {
if (getObjectName() == null)
return -1;
@ -50,12 +58,7 @@ public class DirectoryEntry implements Comparable<DirectoryEntry> {
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((objectid == null) ? 0 : objectid.hashCode());
result = prime * result + ((objname == null) ? 0 : objname.hashCode());
result = prime * result + ((type == null) ? 0 : type.hashCode());
return result;
return Objects.hashCode(objectid, objname, type, size);
}
@Override
@ -67,27 +70,15 @@ public class DirectoryEntry implements Comparable<DirectoryEntry> {
if (getClass() != obj.getClass())
return false;
DirectoryEntry other = (DirectoryEntry) obj;
if (objectid == null) {
if (other.objectid != null)
return false;
} else if (!objectid.equals(other.objectid))
return false;
if (objname == null) {
if (other.objname != null)
return false;
} else if (!objname.equals(other.objname))
return false;
if (type == null) {
if (other.type != null)
return false;
} else if (!type.equals(other.type))
return false;
return true;
return Objects.equal(objectid, other.objectid) &&
Objects.equal(objname, other.objname) &&
Objects.equal(type, other.type) &&
Objects.equal(size, other.size);
}
@Override
public String toString() {
return "DirectoryEntry [type=" + type + ", objectid=" + objectid + ", objname=" + objname
+ "]";
+ ", size=" + size + "]";
}
}

2
apis/atmos/src/main/java/org/jclouds/atmos/domain/SystemMetadata.java
View File

@ -38,7 +38,7 @@ public class SystemMetadata extends DirectoryEntry {
public SystemMetadata(@Nullable byte [] contentmd5, Date atime, Date ctime, String gid, Date itime, Date mtime, int nlink,
String objectid, String objname, String policyname, long size, FileType type, String uid) {
super(objectid, type, objname);
super(objectid, type, objname, size);
this.contentmd5 = contentmd5;
this.atime = atime;
this.ctime = ctime;

7
apis/atmos/src/main/java/org/jclouds/atmos/options/ListOptions.java
View File

@ -19,13 +19,18 @@ package org.jclouds.atmos.options;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import java.util.Collection;
import org.jclouds.http.options.BaseHttpRequestOptions;
import com.google.common.collect.ImmutableList;
/**
* Options used to control paginated results (aka list commands).
*/
public class ListOptions extends BaseHttpRequestOptions {
public static final ListOptions NONE = new ListOptions();
private static final Collection<String> INCLUDE_META = ImmutableList.of(Integer.toString(1));
/**
* specifies the position to resume listing
@ -57,7 +62,7 @@ public class ListOptions extends BaseHttpRequestOptions {
* limit.
*/
public ListOptions includeMeta() {
headers.put("x-emc-include-meta", Integer.toString(1));
headers.replaceValues("x-emc-include-meta", INCLUDE_META);
return this;
}

16
apis/atmos/src/main/java/org/jclouds/atmos/xml/ListDirectoryResponseHandler.java
View File

@ -35,6 +35,10 @@ public class ListDirectoryResponseHandler extends ParseSax.HandlerWithResult<Set
private Set<DirectoryEntry> entries = Sets.newLinkedHashSet();
private String currentObjectId;
private FileType currentType;
private String currentFileName;
private long currentSize;
// metadata parsing
private String currentName;
private StringBuilder currentText = new StringBuilder();
@ -49,11 +53,17 @@ public class ListDirectoryResponseHandler extends ParseSax.HandlerWithResult<Set
} else if (qName.equals("FileType")) {
currentType = FileType.fromValue(currentText.toString().trim());
} else if (qName.equals("Filename")) {
currentFileName = currentText.toString().trim();
if (currentFileName.equals(""))
currentFileName = null;
} else if (qName.equals("Name")) {
currentName = currentText.toString().trim();
if (currentName.equals(""))
currentName = null;
} else if (qName.equals("Value")) {
if (currentName.equals("size")) {
currentSize = Long.parseLong(currentText.toString().trim());
}
} else if (qName.equals("DirectoryEntry")) {
entries.add(new DirectoryEntry(currentObjectId, currentType, currentName));
entries.add(new DirectoryEntry(currentObjectId, currentType, currentFileName, currentSize));
}
currentText.setLength(0);
}

4
apis/atmos/src/test/java/org/jclouds/atmos/AtmosClientTest.java
View File

@ -83,7 +83,7 @@ public class AtmosClientTest extends BaseRestAnnotationProcessingTest<AtmosClien
GeneratedHttpRequest request = processor.createRequest(method, ImmutableList.<Object> of("directory"));
assertRequestLineEquals(request, "GET https://accesspoint.atmosonline.com/rest/namespace/directory/ HTTP/1.1");
assertNonPayloadHeadersEqual(request, HttpHeaders.ACCEPT + ": text/xml\n");
assertNonPayloadHeadersEqual(request, HttpHeaders.ACCEPT + ": text/xml\nx-emc-include-meta: 1\n");
assertPayloadEquals(request, null, null, false);
assertResponseParserClassEquals(method, request, ParseDirectoryListFromContentAndHeaders.class);
@ -113,7 +113,7 @@ public class AtmosClientTest extends BaseRestAnnotationProcessingTest<AtmosClien
GeneratedHttpRequest request = processor.createRequest(method, ImmutableList.<Object> of("directory", new ListOptions().limit(1).token("asda")));
assertRequestLineEquals(request, "GET https://accesspoint.atmosonline.com/rest/namespace/directory/ HTTP/1.1");
assertNonPayloadHeadersEqual(request, HttpHeaders.ACCEPT + ": text/xml\nx-emc-limit: 1\nx-emc-token: asda\n");
assertNonPayloadHeadersEqual(request, HttpHeaders.ACCEPT + ": text/xml\nx-emc-include-meta: 1\nx-emc-limit: 1\nx-emc-token: asda\n");
assertPayloadEquals(request, null, null, false);
assertResponseParserClassEquals(method, request, ParseDirectoryListFromContentAndHeaders.class);

2
apis/atmos/src/test/java/org/jclouds/atmos/blobstore/functions/ResourceMetadataListToDirectoryEntryList.java
View File

@ -41,7 +41,7 @@ public class ResourceMetadataListToDirectoryEntryList
public DirectoryEntry apply(StorageMetadata from) {
FileType type = (from.getType() == StorageType.FOLDER || from.getType() == StorageType.RELATIVE_PATH) ? FileType.DIRECTORY
: FileType.REGULAR;
return new DirectoryEntry(from.getProviderId(), type, from.getName());
return new DirectoryEntry(from.getProviderId(), type, from.getName(), from.getSize());
}
}), from.getNextMarker());

4
apis/atmos/src/test/java/org/jclouds/atmos/functions/ParseDirectoryListFromContentAndHeadersTest.java
View File

@ -72,9 +72,9 @@ public class ParseDirectoryListFromContentAndHeadersTest extends BaseHandlerTest
protected Set<DirectoryEntry> values() {
Builder<DirectoryEntry> expected = ImmutableSet.builder();
expected.add(new DirectoryEntry("4980cdb2a411106a04a4538c92a1b204ad92077de6e3", FileType.DIRECTORY,
"adriancole-blobstore-2096685753"));
"adriancole-blobstore-2096685753", 0));
expected.add(new DirectoryEntry("4980cdb2a410105404980d99e53a0504ad93939e7dc3", FileType.DIRECTORY,
"adriancole-blobstore247496608"));
"adriancole-blobstore247496608", 0));
return expected.build();
}
}

4
apis/atmos/src/test/java/org/jclouds/atmos/xml/ListDirectoryResponseHandlerTest.java
View File

@ -47,9 +47,9 @@ public class ListDirectoryResponseHandlerTest extends BaseHandlerTest {
ParseSax<Set<DirectoryEntry>> parser = createParser();
Set<DirectoryEntry> expected = Sets.newTreeSet();
expected.add(new DirectoryEntry("4980cdb2a411106a04a4538c92a1b204ad92077de6e3",
FileType.DIRECTORY, "adriancole-blobstore-2096685753"));
FileType.DIRECTORY, "adriancole-blobstore-2096685753", 0));
expected.add(new DirectoryEntry("4980cdb2a410105404980d99e53a0504ad93939e7dc3",
FileType.DIRECTORY, "adriancole-blobstore247496608"));
FileType.DIRECTORY, "adriancole-blobstore247496608", 0));
Set<DirectoryEntry> result = parser.parse(is);
assertEquals(result, expected);
}

1
apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
View File

@ -307,6 +307,7 @@ public class FilesystemStorageStrategyImpl implements LocalStorageStrategy {
Blob blob = builder.build();
blob.getMetadata().setContainer(container);
blob.getMetadata().setLastModified(new Date(file.lastModified()));
blob.getMetadata().setSize(file.length());
if (blob.getPayload().getContentMetadata().getContentMD5() != null)
blob.getMetadata().setETag(base16().lowerCase().encode(blob.getPayload().getContentMetadata().getContentMD5()));
return blob;

1
apis/s3/src/main/java/org/jclouds/s3/blobstore/functions/ObjectToBlobMetadata.java
View File

@ -74,6 +74,7 @@ public class ObjectToBlobMetadata implements Function<ObjectMetadata, MutableBlo
} else {
to.setType(StorageType.BLOB);
}
to.setSize(from.getContentMetadata().getContentLength());
return to;
}
}

7
apis/swift/src/test/java/org/jclouds/openstack/swift/blobstore/integration/SwiftContainerIntegrationLiveTest.java
View File

@ -26,6 +26,7 @@ import org.jclouds.openstack.keystone.v2_0.config.KeystoneProperties;
import org.jclouds.openstack.swift.CommonSwiftClient;
import org.jclouds.openstack.swift.domain.ContainerMetadata;
import org.jclouds.openstack.swift.options.CreateContainerOptions;
import org.testng.SkipException;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList;
@ -94,4 +95,10 @@ public class SwiftContainerIntegrationLiveTest extends BaseContainerIntegrationT
returnContainer(containerName);
}
}
@Test(groups = { "integration", "live" })
public void testListContainerGetBlobSize() throws Exception {
// use new Swift provider instead
throw new SkipException("Intentionally not implemented for the legacy Swift provider");
}
}

1
blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java
View File

@ -192,6 +192,7 @@ public class TransientStorageStrategy implements LocalStorageStrategy {
blob.getMetadata().setUri(
uriBuilder(new StringBuilder("mem://").append(containerName)).path(in.getMetadata().getName()).build());
blob.getMetadata().setLastModified(new Date());
blob.getMetadata().setSize((long) input.length);
String eTag = base16().lowerCase().encode(contentMd5.asBytes());
blob.getMetadata().setETag(eTag);
// Set HTTP headers to match metadata

5
blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java
View File

@ -48,6 +48,7 @@ import org.jclouds.blobstore.LocalStorageStrategy;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobBuilder;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.MutableBlobMetadata;
import org.jclouds.blobstore.domain.MutableStorageMetadata;
import org.jclouds.blobstore.domain.PageSet;
import org.jclouds.blobstore.domain.StorageMetadata;
@ -225,7 +226,9 @@ public final class LocalBlobStore implements BlobStore {
checkState(oldBlob != null, "blob " + key + " is not present although it was in the list of "
+ containerName);
checkState(oldBlob.getMetadata() != null, "blob " + containerName + "/" + key + " has no metadata");
return BlobStoreUtils.copy(oldBlob.getMetadata());
MutableBlobMetadata md = BlobStoreUtils.copy(oldBlob.getMetadata());
md.setSize(oldBlob.getMetadata().getSize());
return md;
}
}));

3
blobstore/src/main/java/org/jclouds/blobstore/domain/MutableStorageMetadata.java
View File

@ -46,4 +46,7 @@ public interface MutableStorageMetadata extends MutableResourceMetadata<StorageT
*/
void setLastModified(@Nullable Date lastModified);
/** @see #getSize */
void setSize(@Nullable Long size);
}

2
blobstore/src/main/java/org/jclouds/blobstore/domain/StorageMetadata.java
View File

@ -92,4 +92,6 @@ public interface StorageMetadata extends ResourceMetadata<StorageType> {
*/
Date getLastModified();
/** Size of the resource, possibly null. */
Long getSize();
}

11
blobstore/src/main/java/org/jclouds/blobstore/domain/internal/MutableStorageMetadataImpl.java
View File

@ -32,6 +32,7 @@ public class MutableStorageMetadataImpl extends MutableResourceMetadataImpl<Stor
private String eTag;
private Date creationDate;
private Date lastModified;
private Long size;
public MutableStorageMetadataImpl() {
super();
@ -41,6 +42,7 @@ public class MutableStorageMetadataImpl extends MutableResourceMetadataImpl<Stor
super(from);
this.eTag = from.getETag();
this.lastModified = from.getLastModified();
this.size = from.getSize();
}
/**
@ -85,4 +87,13 @@ public class MutableStorageMetadataImpl extends MutableResourceMetadataImpl<Stor
this.eTag = eTag;
}
@Override
public Long getSize() {
return size;
}
@Override
public void setSize(Long size) {
this.size = size;
}
}

19
blobstore/src/main/java/org/jclouds/blobstore/domain/internal/StorageMetadataImpl.java
View File

@ -42,16 +42,28 @@ public class StorageMetadataImpl extends ResourceMetadataImpl<StorageType> imple
@Nullable
private final Date lastModified;
private final StorageType type;
@Nullable
private final Long size;
public StorageMetadataImpl(StorageType type, @Nullable String id, @Nullable String name,
@Nullable Location location, @Nullable URI uri, @Nullable String eTag,
@Nullable Date creationDate, @Nullable Date lastModified,
Map<String, String> userMetadata) {
Map<String, String> userMetadata, @Nullable Long size) {
super(id, name, location, uri, userMetadata);
this.eTag = eTag;
this.creationDate = creationDate;
this.lastModified = lastModified;
this.type = checkNotNull(type, "type");
this.size = size;
}
/** @deprecated call StorageMetadataImpl(StorageType.class, String.class, String.class, Location.class, URI.class, String.class, Date.class, Date.class, Map.class, Long.class) */
@Deprecated
public StorageMetadataImpl(StorageType type, @Nullable String id, @Nullable String name,
@Nullable Location location, @Nullable URI uri, @Nullable String eTag,
@Nullable Date creationDate, @Nullable Date lastModified,
Map<String, String> userMetadata) {
this(type, id, name, location, uri, eTag, creationDate, lastModified, userMetadata, null);
}
/**
@ -105,4 +117,9 @@ public class StorageMetadataImpl extends ResourceMetadataImpl<StorageType> imple
return lastModified;
}
@Override
public Long getSize() {
return size;
}
}

26
blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseContainerIntegrationTest.java
View File

@ -35,6 +35,8 @@ import java.util.concurrent.TimeoutException;
import javax.ws.rs.core.MediaType;
import com.google.common.io.ByteSource;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.PageSet;
@ -375,6 +377,30 @@ public class BaseContainerIntegrationTest extends BaseBlobStoreIntegrationTest {
}
@Test(groups = { "integration", "live" })
public void testListContainerGetBlobSize() throws Exception {
String containerName = getContainerName();
try {
ByteSource byteSource = ByteSource.wrap(new byte[42]);
for (int i = 0; i < 2; i++) {
view.getBlobStore().putBlob(containerName, view.getBlobStore()
.blobBuilder(i + "")
.payload(byteSource)
.contentLength(byteSource.size())
.build());
}
PageSet<? extends StorageMetadata> container = view.getBlobStore().list(containerName);
for (StorageMetadata metadata : container) {
assertEquals(metadata.getSize(), Long.valueOf(byteSource.size()));
}
} finally {
returnContainer(containerName);
}
}
protected void addAlphabetUnderRoot(String containerName) throws InterruptedException {
for (char letter = 'a'; letter <= 'z'; letter++) {
view.getBlobStore().putBlob(containerName,

1
providers/azureblob/src/main/java/org/jclouds/azureblob/blobstore/functions/BlobPropertiesToBlobMetadata.java
View File

@ -71,6 +71,7 @@ public class BlobPropertiesToBlobMetadata implements Function<BlobProperties, Mu
} else {
to.setType(StorageType.BLOB);
}
to.setSize(from.getContentMetadata().getContentLength());
return to;
}
}