Further refinements to blobstore2 in response to feedback on Github and IRC.

* Removed redundant functions, made others delegate to others instead
  of calling right into Java.
* Fixed typos, removed unused dependency references.
* Better naming for key functions: list-container -> blobs, list-blobs ->
  container-seq.
* Made blob fn no longer require a blobstore argument.
* Better comments, updated tests.
This commit is contained in:
David Santiago 2011-04-05 02:42:45 -05:00
parent 3011ffad23
commit ed90e29d68
2 changed files with 80 additions and 88 deletions

View File

@ -47,18 +47,31 @@ See http://code.google.com/p/jclouds for details."
[org.jclouds.blobstore
AsyncBlobStore domain.BlobBuilder BlobStore BlobStoreContext
BlobStoreContextFactory domain.BlobMetadata domain.StorageMetadata
domain.Blob options.ListContainerOptions]
domain.Blob domain.internal.BlobBuilderImpl
options.ListContainerOptions]
org.jclouds.io.Payloads
org.jclouds.io.payloads.PhantomPayload
java.util.Arrays
[java.security DigestOutputStream MessageDigest]
com.google.common.collect.ImmutableSet))
com.google.common.collect.ImmutableSet
org.jclouds.encryption.internal.JCECrypto))
(try
(require '[clojure.contrib.io :as io])
(catch Exception e
(require '[clojure.contrib.duck-streams :as io])))
(def ^{:private true}
crypto-impl
;; BouncyCastle might not be present. Try to load it, but fall back to
;; JCECrypto if we can't.
(try
(import 'org.jclouds.encryption.bouncycastle.BouncyCastleCrypto)
(.newInstance
(Class/forName
"org.jclouds.encryption.bouncycastle.BouncyCastleCrypto"))
(catch Exception e
(JCECrypto.))))
(defn blobstore
"Create a logged in context.
Options for communication style
@ -110,14 +123,15 @@ Options can also be specified for extension modules
:with-details #(when %2 (.withDetails %1))
:recursive #(when %2 (.recursive %1))})
(defn list-container
"Low-level container listing. Use list-blobs where possible since
it's higher-level and returns a lazy seq. Options are:
:after-marker string
:in-direcory path
:max-results n
:with-details true
:recursive true"
(defn blobs
"Returns a set of blobs in the given container, as directed by the
query options below.
Options are:
:after-marker string
:in-directory path
:max-results n
:with-details true
:recursive true"
[blobstore container-name & args]
(let [options (apply hash-map args)
list-options (reduce
@ -128,19 +142,20 @@ Options can also be specified for extension modules
options)]
(.list blobstore container-name list-options)))
(defn- list-blobs-chunk [blobstore container prefix & [marker]]
(apply list-container blobstore container
(defn- container-seq-chunk
[blobstore container prefix marker]
(apply blobs blobstore container
(concat (when prefix
[:in-directory prefix])
(when (string? marker)
[:after-marker marker]))))
(defn- list-blobs-chunks [blobstore container prefix marker]
(when marker
(let [chunk (list-blobs-chunk blobstore container prefix marker)]
(defn- container-seq-chunks [blobstore container prefix marker]
(when marker ;; When getNextMarker returns null, there's no more.
(let [chunk (container-seq-chunk blobstore container prefix marker)]
(lazy-seq (cons chunk
(list-blobs-chunks blobstore container prefix
(.getNextMarker chunk)))))))
(container-seq-chunks blobstore container prefix
(.getNextMarker chunk)))))))
(defn- concat-elements
"Make a lazy concatenation of the lazy sequences contained in coll.
@ -150,12 +165,15 @@ Options can also be specified for extension modules
(if-let [s (seq coll)]
(lazy-seq (concat (first s) (concat-elements (next s))))))
(defn list-blobs
(defn container-seq
"Returns a lazy seq of all blobs in the given container."
([blobstore container]
(list-blobs blobstore container nil))
(container-seq blobstore container nil))
([blobstore container prefix]
(concat-elements (list-blobs-chunks blobstore container prefix :start))))
;; :start has no special meaning, it is just a non-null (null indicates
;; end), non-string (markers are strings).
(concat-elements (container-seq-chunks blobstore container prefix
:start))))
(defn locations
"Retrieve the available container locations for the blobstore context."
@ -243,7 +261,7 @@ Options can also be specified for extension modules
(defn get-blob-stream
"Get an inputstream from the blob at a given path"
[^BlobStore blobstore container-name path]
(.getInput (.getPayload (.getBlob blobstore container-name path))))
(.getInput (.getPayload (get-blob blobstore container-name path))))
(defn remove-blob
"Remove blob from given path"
@ -255,34 +273,17 @@ Options can also be specified for extension modules
[^BlobStore blobstore container-name]
(.countBlobs blobstore container-name))
(defn blobs
"List the blobs in a container:
blobstore container -> blobs
List the blobs in a container under a path:
blobstore container dir -> blobs
example:
(pprint
(blobs
(blobstore-context flightcaster-creds)
\"somecontainer\" \"some-dir\"))"
([^BlobStore blobstore container-name]
(.list blobstore container-name))
([^BlobStore blobstore container-name dir]
(.list blobstore container-name
(.inDirectory (new ListContainerOptions) dir))))
(defn blob
"Create a new blob with the specified payload and options."
([^BlobStore blobstore ^String name
([^String name &
{:keys [payload content-type content-length content-md5 calculate-md5
content-disposition content-encoding content-language metadata]}]
{:pre [(not (and content-md5 calculate-md5))
(not (and (nil? payload) calculate-md5))]}
(let [blob-builder (if payload
(.payload (.blobBuilder blobstore name) payload)
(.forSigning (.blobBuilder blobstore name)))
(let [blob-builder (.name (BlobBuilderImpl. crypto-impl) name)
blob-builder (if payload
(.payload blob-builder payload)
(.forSigning blob-builder))
blob-builder (if content-length ;; Special case, arg is prim.
(.contentLength blob-builder content-length)
blob-builder)
@ -299,21 +300,6 @@ Options can also be specified for extension modules
(.userMetadata metadata))
(.build blob-builder))))
(defn md5-blob
"add a content md5 to a blob, or make a new blob that has an md5.
note that this implies rebuffering, if the blob's payload isn't repeatable"
([^Blob blob]
(Payloads/calculateMD5 blob))
([^BlobStore blobstore ^String name payload]
(md5-blob (blob blobstore name {:payload payload}))))
(defn upload-blob
"Create anrepresenting text data:
container, name, string -> etag"
([^BlobStore blobstore container-name name data]
(put-blob blobstore container-name
(md5-blob blobstore name data))))
(define-accessors StorageMetadata "blob" type id name
location-id uri last-modfied)
(define-accessors BlobMetadata "blob" content-type)

View File

@ -62,42 +62,47 @@
(is (create-container *blobstore* "fred"))
(is (= 1 (count (containers *blobstore*)))))
(deftest list-container-test
(deftest blobs-test
(is (create-container *blobstore* "container"))
(is (empty? (list-container *blobstore* "container")))
(is (upload-blob *blobstore* "container" "blob1" "blob1"))
(is (upload-blob *blobstore* "container" "blob2" "blob2"))
(is (= 2 (count (list-container *blobstore* "container"))))
(is (= 1 (count (list-container *blobstore* "container" :max-results 1))))
(is (empty? (blobs *blobstore* "container")))
(is (put-blob *blobstore* "container"
(blob "blob1" :payload "blob1" :calculate-md5 true)))
(is (put-blob *blobstore* "container"
(blob "blob2" :payload "blob2" :calculate-md5 true)))
(is (= 2 (count (blobs *blobstore* "container"))))
(is (= 1 (count (blobs *blobstore* "container" :max-results 1))))
(create-directory *blobstore* "container" "dir")
(is (upload-blob *blobstore* "container" "dir/blob2" "blob2"))
(is (put-blob *blobstore* "container"
(blob "dir/blob2" :payload "blob2" :calculate-md5 true)))
(is (= 3 (count-blobs *blobstore* "container")))
(is (= 3 (count (list-container *blobstore* "container"))))
(is (= 4 (count (list-container *blobstore* "container" :recursive true))))
(is (= 3 (count (list-container *blobstore* "container" :with-details true))))
(is (= 1 (count (list-container *blobstore* "container"
:in-directory "dir")))))
(is (= 3 (count (blobs *blobstore* "container"))))
(is (= 4 (count (blobs *blobstore* "container" :recursive true))))
(is (= 3 (count (blobs *blobstore* "container" :with-details true))))
(is (= 1 (count (blobs *blobstore* "container" :in-directory "dir")))))
(deftest large-container-list-test
(let [container-name "test"
total-blobs 5000]
;; create a container full of blobs
(create-container *blobstore* container-name)
(dotimes [i total-blobs] (upload-blob *blobstore*
container-name (str i) (str i)))
(dotimes [i total-blobs] (put-blob *blobstore* container-name
(blob (str i)
:payload (str i)
:calculate-md5 true)))
;; verify
(is (= total-blobs (count-blobs *blobstore* container-name)))))
(deftest list-blobs-test
(deftest container-seq-test
(is (create-container *blobstore* "container"))
(is (empty? (list-blobs *blobstore* "container"))))
(is (empty? (container-seq *blobstore* "container")))
(is (empty? (container-seq *blobstore* "container" "/a"))))
(deftest get-blob-test
(is (create-container *blobstore* "blob"))
(is (upload-blob *blobstore* "blob" "blob1" "blob1"))
(is (upload-blob *blobstore* "blob" "blob2" "blob2"))
(is (put-blob *blobstore* "blob"
(blob "blob1" :payload "blob1" :calculate-md5 true)))
(is (put-blob *blobstore* "blob"
(blob "blob2" :payload "blob2" :calculate-md5 true)))
(is (= "blob2" (Strings2/toStringAndClose (get-blob-stream *blobstore*
"blob" "blob2")))))
@ -108,7 +113,7 @@
(deftest sign-put-test
(let [request (sign-put *blobstore* "container"
(blob *blobstore* "path" {:content-length 10}))]
(blob "path" :content-length 10))]
(is (= "http://localhost/container/path" (str (.getEndpoint request))))
(is (= "PUT" (.getMethod request)))
(is (= "10" (first (.get (.getHeaders request) "Content-Length"))))
@ -118,11 +123,12 @@
(deftest sign-put-with-headers-test
(let [request (sign-put *blobstore*
"container"
(blob *blobstore* "path" {:content-length 10
:content-type "x"
:content-language "en"
:content-disposition "f"
:content-encoding "g"}))]
(blob "path"
:content-length 10
:content-type "x"
:content-language "en"
:content-disposition "f"
:content-encoding "g"))]
(is (= "PUT" (.getMethod request)))
(is (= "10" (first (.get (.getHeaders request) "Content-Length"))))
(is (= "x" (first (.get (.getHeaders request) "Content-Type"))))
@ -136,9 +142,9 @@
(is (= "DELETE" (.getMethod request)))))
(deftest blob-test
(let [a-blob (blob *blobstore* "test-name"
{:payload (.getBytes "test-payload")
:calculate-md5 true})]
(let [a-blob (blob "test-name"
:payload (.getBytes "test-payload")
:calculate-md5 true)]
(is (= (seq (.. a-blob (getPayload) (getContentMetadata) (getContentMD5)))
(seq (CryptoStreams/md5 (.getBytes "test-payload")))))))