From 2c74c70b1f7522216db3c44a9d2fa752715f8adc Mon Sep 17 00:00:00 2001 From: David Santiago Date: Fri, 1 Apr 2011 23:32:07 -0500 Subject: [PATCH 1/2] Updates to improve blobstore.clj. * New function blob2 exposes many more options for creating blobs. * Updated functions that use blob to use blob2. * New functions sign-get, sign-put, and sign-delete replace the functionality of sign-blob-request. * Added tests. * Deprecation metadata added to blob and sign-blob-request. --- .../main/clojure/org/jclouds/blobstore.clj | 71 ++++++++++++++++--- .../clojure/org/jclouds/blobstore_test.clj | 34 +++++++++ 2 files changed, 96 insertions(+), 9 deletions(-) diff --git a/blobstore/src/main/clojure/org/jclouds/blobstore.clj b/blobstore/src/main/clojure/org/jclouds/blobstore.clj index d759fd91a2..6cf7d3b1af 100644 --- a/blobstore/src/main/clojure/org/jclouds/blobstore.clj +++ b/blobstore/src/main/clojure/org/jclouds/blobstore.clj @@ -44,9 +44,9 @@ See http://code.google.com/p/jclouds for details." (:import [java.io File FileOutputStream OutputStream] java.util.Properties [org.jclouds.blobstore - AsyncBlobStore BlobStore BlobStoreContext BlobStoreContextFactory - domain.BlobMetadata domain.StorageMetadata domain.Blob - options.ListContainerOptions] + AsyncBlobStore domain.BlobBuilder BlobStore BlobStoreContext + BlobStoreContextFactory domain.BlobMetadata domain.StorageMetadata + domain.Blob options.ListContainerOptions] org.jclouds.io.Payloads org.jclouds.io.payloads.PhantomPayload java.util.Arrays @@ -267,6 +267,7 @@ Note: (apply concat coll) or (lazy-cat coll) are not lazy wrt coll itself." :put. For :put requests, :content-length must be specified. Optionally, :content-type, :content-disposition, :content-language, :content-encoding and :content-md5 may be given." + {:deprecated "1.0-beta-10"} ([container-name path] (sign-blob-request container-name path {:method :get} *blobstore*)) ([container-name path @@ -299,6 +300,33 @@ Note: (apply concat coll) or (lazy-cat coll) are not lazy wrt coll itself." (.setContentLanguage content-language)) payload))))))) +(defn sign-get + "Get a signed http GET request for manipulating a blob in another + application, Ex. curl." + ([container-name name] + (sign-get container-name name *blobstore*)) + ([container-name name ^BlobStore blobstore] + (.signGetBlob (.. blobstore getContext getSigner) container-name name))) + +(defn sign-put + "Get a signed http PUT request for manipulating a blob in another + application, Ex. curl. A Blob with at least the name and content-length + must be given." + ([container-name blob] + (sign-put container-name blob *blobstore*)) + ([container-name ^Blob blob ^BlobStore blobstore] + (.signPutBlob (.. blobstore getContext getSigner) + container-name + blob))) + +(defn sign-delete + "Get a signed http DELETE request for manipulating a blob in another + applicaiton, Ex. curl." + ([container-name name] + (sign-delete container-name name *blobstore*)) + ([container-name name ^BlobStore blobstore] + (.signRemoveBlob (.. blobstore getContext getSigner) container-name name))) + (defn get-blob-stream "Get an inputstream from the blob at a given path" ([container-name path] @@ -340,21 +368,46 @@ example: (.inDirectory (new ListContainerOptions) dir)))) (defn blob "create a new blob with the specified payload" + {:deprecated "1.0-beta-10"} ([#^String name payload] (blob name payload *blobstore*)) ([#^String name payload #^BlobStore blobstore] (doto (.newBlob blobstore name) (.setPayload payload)))) +(defn blob2 + "Create a new blob with the specified payload and options." + ([^String name option-map] + (blob2 name option-map *blobstore*)) + ([^String name + {:keys [payload content-type content-length content-md5 + content-disposition content-encoding content-language metadata] + :or {for-signing false}} + ^BlobStore blobstore] + (let [blob-builder (if payload + (.payload (.blobBuilder blobstore name) payload) + (.forSigning (.blobBuilder blobstore name))) + blob-builder (if content-length ;; Special case, arg is prim. + (.contentLength blob-builder content-length) + blob-builder)] + (doto blob-builder + (.contentType content-type) + (.contentMD5 content-md5) + (.contentDisposition content-disposition) + (.contentEncoding content-encoding) + (.contentLanguage content-language) + (.userMetadata metadata)) + (.build blob-builder)))) + (defn md5-blob "add a content md5 to a blob, or make a new blob that has an md5. note that this implies rebuffering, if the blob's payload isn't repeatable" - ([#^Blob blob] - (Payloads/calculateMD5 blob)) - ([#^String name payload] - (blob name payload *blobstore*)) - ([#^String name payload #^BlobStore blobstore] - (md5-blob (blob name payload blobstore)))) + ([#^Blob blob] + (Payloads/calculateMD5 blob)) + ([#^String name payload] + (blob2 name {:payload payload} *blobstore*)) + ([#^String name payload #^BlobStore blobstore] + (md5-blob (blob2 name {:payload payload} blobstore)))) (defn upload-blob "Create anrepresenting text data: diff --git a/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj b/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj index 8ab2988eb1..9c571f88b8 100644 --- a/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj +++ b/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj @@ -146,6 +146,40 @@ (is (= "f" (first (.get (.getHeaders request) "Content-Disposition")))) (is (= "g" (first (.get (.getHeaders request) "Content-Encoding"))))))) +(deftest sign-get-test + (let [request (sign-get "container" "path")] + (is (= "http://localhost/container/path" (str (.getEndpoint request)))) + (is (= "GET" (.getMethod request))))) + +(deftest sign-put-test + (let [request (sign-put "container" + (blob2 "path" {:content-length 10}))] + (is (= "http://localhost/container/path" (str (.getEndpoint request)))) + (is (= "PUT" (.getMethod request))) + (is (= "10" (first (.get (.getHeaders request) "Content-Length")))) + (is (nil? + (first (.get (.getHeaders request) "Content-Type")))))) + +(deftest sign-put-with-headers-test + (let [request (sign-put + "container" + (blob2 "path" {:content-length 10 + :content-type "x" + :content-language "en" + :content-disposition "f" + :content-encoding "g"}))] + (is (= "PUT" (.getMethod request))) + (is (= "10" (first (.get (.getHeaders request) "Content-Length")))) + (is (= "x" (first (.get (.getHeaders request) "Content-Type")))) + (is (= "en" (first (.get (.getHeaders request) "Content-Language")))) + (is (= "f" (first (.get (.getHeaders request) "Content-Disposition")))) + (is (= "g" (first (.get (.getHeaders request) "Content-Encoding")))))) + +(deftest sign-delete-test + (let [request (sign-delete "container" "path")] + (is (= "http://localhost/container/path" (str (.getEndpoint request)))) + (is (= "DELETE" (.getMethod request))))) + ;; TODO: more tests involving blob-specific functions (deftest corruption-hunt From 3459df3653326213eac0226b6f1f5f92bfeb180d Mon Sep 17 00:00:00 2001 From: David Santiago Date: Sat, 2 Apr 2011 02:26:45 -0500 Subject: [PATCH 2/2] Additional refinements to blobstore.clj updates. * Added an option to conditionally call calculateMD5() in blob2. * Fixed a bug in md5-blob (which predated these changes). * Added a direct test for blob2 and calculateMD5. --- .../src/main/clojure/org/jclouds/blobstore.clj | 17 +++++++++++------ .../test/clojure/org/jclouds/blobstore_test.clj | 7 +++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/blobstore/src/main/clojure/org/jclouds/blobstore.clj b/blobstore/src/main/clojure/org/jclouds/blobstore.clj index 6cf7d3b1af..91bc83f417 100644 --- a/blobstore/src/main/clojure/org/jclouds/blobstore.clj +++ b/blobstore/src/main/clojure/org/jclouds/blobstore.clj @@ -380,19 +380,24 @@ example: ([^String name option-map] (blob2 name option-map *blobstore*)) ([^String name - {:keys [payload content-type content-length content-md5 - content-disposition content-encoding content-language metadata] - :or {for-signing false}} + {:keys [payload content-type content-length content-md5 calculate-md5 + content-disposition content-encoding content-language metadata]} ^BlobStore blobstore] + {:pre [(not (and content-md5 calculate-md5)) + (not (and (nil? payload) calculate-md5))]} (let [blob-builder (if payload (.payload (.blobBuilder blobstore name) payload) (.forSigning (.blobBuilder blobstore name))) blob-builder (if content-length ;; Special case, arg is prim. (.contentLength blob-builder content-length) - blob-builder)] + blob-builder) + blob-builder (if calculate-md5 ;; Only do calculateMD5 OR contentMD5. + (.calculateMD5 blob-builder) + (if content-md5 + (.contentMD5 blob-builder content-md5) + blob-builder))] (doto blob-builder (.contentType content-type) - (.contentMD5 content-md5) (.contentDisposition content-disposition) (.contentEncoding content-encoding) (.contentLanguage content-language) @@ -405,7 +410,7 @@ note that this implies rebuffering, if the blob's payload isn't repeatable" ([#^Blob blob] (Payloads/calculateMD5 blob)) ([#^String name payload] - (blob2 name {:payload payload} *blobstore*)) + (md5-blob name payload *blobstore*)) ([#^String name payload #^BlobStore blobstore] (md5-blob (blob2 name {:payload payload} blobstore)))) diff --git a/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj b/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj index 9c571f88b8..1a347a7d1c 100644 --- a/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj +++ b/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj @@ -21,6 +21,7 @@ (:use [org.jclouds.blobstore] :reload-all) (:use [clojure.test]) (:import [org.jclouds.blobstore BlobStoreContextFactory] + [org.jclouds.crypto CryptoStreams] [java.io ByteArrayOutputStream] [org.jclouds.util Strings2])) @@ -180,6 +181,12 @@ (is (= "http://localhost/container/path" (str (.getEndpoint request)))) (is (= "DELETE" (.getMethod request))))) +(deftest blob2-test + (let [a-blob (blob2 "test-name" {:payload (.getBytes "test-payload") + :calculate-md5 true})] + (is (= (seq (.. a-blob (getPayload) (getContentMetadata) (getContentMD5))) + (seq (CryptoStreams/md5 (.getBytes "test-payload"))))))) + ;; TODO: more tests involving blob-specific functions (deftest corruption-hunt