From 94b6efe8027ad89d031838778d25d612d3b44b8d Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 26 Mar 2010 00:21:05 +0000 Subject: [PATCH 1/6] Reorder arguments so optional args are always last. --- .../main/clojure/org/jclouds/blobstore.clj | 84 +++++++++---------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/blobstore/src/main/clojure/org/jclouds/blobstore.clj b/blobstore/src/main/clojure/org/jclouds/blobstore.clj index a8883a393f..ac6149bc12 100644 --- a/blobstore/src/main/clojure/org/jclouds/blobstore.clj +++ b/blobstore/src/main/clojure/org/jclouds/blobstore.clj @@ -127,96 +127,94 @@ Options can also be specified for extension modules (defn create-container "Create a container." ([container-name] - (create-container *blobstore* "default" container-name)) - ([blobstore container-name] - (if (blobstore? blobstore) - (create-container blobstore "default" container-name) - (create-container *blobstore* container-name blobstore))) - ([blobstore location-name container-name] + (create-container container-name "default" *blobstore*)) + ([container-name location-name] + (create-container container-name location-name *blobstore*)) + ([container-name location-name blobstore] (.createContainerInLocation blobstore location-name container-name))) (defn clear-container "Clear a container." ([container-name] - (clear-container *blobstore* container-name)) - ([blobstore container-name] + (clear-container container-name *blobstore*)) + ([container-name blobstore] (.clearContainer blobstore container-name))) (defn delete-container "Delete a container." ([container-name] - (delete-container *blobstore* container-name)) - ([blobstore container-name] + (delete-container container-name *blobstore*)) + ([container-name blobstore] (.deleteContainer blobstore container-name))) (defn container-exists? "Predicate to check presence of a container" ([container-name] - (container-exists? *blobstore* container-name)) - ([blobstore container-name] + (container-exists? container-name *blobstore*)) + ([container-name blobstore] (.containerExists blobstore container-name))) (defn directory-exists? "Predicate to check presence of a directory" ([container-name path] - (directory-exists? *blobstore* container-name path)) - ([blobstore container-name path] + (directory-exists? container-name path *blobstore*)) + ([container-name path blobstore] (.directoryExists blobstore container-name path))) (defn create-directory "Create a directory path." ([container-name path] - (create-directory *blobstore* container-name path)) - ([blobstore container-name path] + (create-directory container-name path *blobstore*)) + ([container-name path blobstore] (.createDirectory blobstore container-name path))) (defn delete-directory "Delete a directory path." ([container-name path] - (delete-directory *blobstore* container-name path)) - ([blobstore container-name path] + (delete-directory container-name path *blobstore*)) + ([container-name path blobstore] (.deleteDirectory blobstore container-name path))) (defn blob-exists? "Predicate to check presence of a blob" ([container-name path] - (blob-exists? *blobstore* container-name path)) - ([blobstore container-name path] + (blob-exists? container-name path *blobstore*)) + ([container-name path blobstore] (.blobExists blobstore container-name path))) (defn put-blob "Put a blob. Metadata in the blob determines location." ([container-name blob] - (put-blob *blobstore* container-name blob)) - ([blobstore container-name blob] + (put-blob container-name blob *blobstore*)) + ([container-name blob blobstore] (.putBlob blobstore container-name blob))) (defn blob-metadata "Get blob metadata from given path" ([container-name path] - (blob-metadata *blobstore* container-name path)) - ([blobstore container-name path] + (blob-metadata container-name path *blobstore*)) + ([container-name path blobstore] (.blobMetadata blobstore container-name path))) (defn get-blob "Get blob from given path" ([container-name path] - (get-blob *blobstore* container-name path)) - ([blobstore container-name path] + (get-blob container-name path *blobstore*)) + ([container-name path blobstore] (.getBlob blobstore container-name path))) (defn remove-blob "Remove blob from given path" ([container-name path] - (remove-blob *blobstore* container-name path)) - ([blobstore container-name path] + (remove-blob container-name path *blobstore*)) + ([container-name path blobstore] (.removeBlob blobstore container-name path))) (defn count-blobs "Count blobs" ([container-name] - (count-blobs *blobstore* container-name)) - ([blobstore container-name] + (count-blobs container-name *blobstore*)) + ([container-name blobstore] (.countBlob blobstore container-name))) (defn blobs @@ -241,18 +239,18 @@ example: (defn create-blob "Create an blob representing text data: -container, name, string -> etag -" +container, name, string -> etag" ([container-name name data] ;; TODO: allow payload to be a stream - (create-blob *blobstore* container-name name data)) - ([blobstore container-name name data] - (put-blob blobstore container-name + (create-blob container-name name data *blobstore*)) + ([container-name name data blobstore] + (put-blob container-name (doto (.newBlob blobstore name) (.setPayload data) - (.generateMD5))))) + (.generateMD5)) + blobstore))) (defmulti #^{:arglists '[[container-name name target] - [blobstore container-name name target]]} + [container-name name target blobstore]]} download-blob (fn [& args] (if (= (count args) 3) ::short-form @@ -260,11 +258,11 @@ container, name, string -> etag (defmethod download-blob ::short-form [container-name name target] - (download-blob *blobstore* container-name name target)) + (download-blob container-name name target *blobstore*)) -(defmethod download-blob OutputStream [blobstore container-name name target +(defmethod download-blob OutputStream [container-name name target blobstore & [retries]] - (let [blob (get-blob blobstore container-name name) + (let [blob (get-blob container-name name blobstore) digest-stream (.md5OutputStream ;; TODO: not all clouds use MD5 *encryption-service* target)] (copy (.getContent blob) digest-stream) @@ -272,12 +270,12 @@ container, name, string -> etag metadata-digest (.getContentMD5 (.getMetadata blob))] (when-not (Arrays/equals digest metadata-digest) (if (<= (or retries 0) *max-retries*) - (recur blobstore container-name name target [(inc (or retries 1))]) + (recur container-name name target blobstore [(inc (or retries 1))]) (throw (Exception. (format "Download failed for %s/%s" container-name name)))))))) -(defmethod download-blob File [blobstore container-name name target] - (download-blob blobstore container-name name (FileOutputStream. target))) +(defmethod download-blob File [container-name name target blobstore] + (download-blob container-name name (FileOutputStream. target) blobstore)) (define-accessors StorageMetadata "blob" type id name location-id uri last-modfied) From 91f09690b67eaa5c9c4a0a3d3a204251b3e1a5bb Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 26 Mar 2010 00:33:12 +0000 Subject: [PATCH 2/6] Rename create-blob to upload-blob. --- blobstore/src/main/clojure/org/jclouds/blobstore.clj | 4 ++-- .../src/test/clojure/org/jclouds/blobstore_test.clj | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/blobstore/src/main/clojure/org/jclouds/blobstore.clj b/blobstore/src/main/clojure/org/jclouds/blobstore.clj index ac6149bc12..433b4b716c 100644 --- a/blobstore/src/main/clojure/org/jclouds/blobstore.clj +++ b/blobstore/src/main/clojure/org/jclouds/blobstore.clj @@ -237,11 +237,11 @@ example: (.list (as-blobstore blobstore) container-name (.inDirectory (new ListContainerOptions) dir)))) -(defn create-blob +(defn upload-blob "Create an blob representing text data: container, name, string -> etag" ([container-name name data] ;; TODO: allow payload to be a stream - (create-blob container-name name data *blobstore*)) + (upload-blob container-name name data *blobstore*)) ([container-name name data blobstore] (put-blob container-name (doto (.newBlob blobstore name) diff --git a/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj b/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj index 913db4d843..72be8f158d 100644 --- a/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj +++ b/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj @@ -41,12 +41,12 @@ (deftest list-container-test (is (create-container "container")) (is (empty? (list-container "container"))) - (is (create-blob "container" "blob1" "blob1")) - (is (create-blob "container" "blob2" "blob2")) + (is (upload-blob "container" "blob1" "blob1")) + (is (upload-blob "container" "blob2" "blob2")) (is (= 2 (count (list-container "container")))) (is (= 1 (count (list-container "container" :max-results 1)))) (create-directory "container" "dir") - (is (create-blob "container" "dir/blob2" "blob2")) + (is (upload-blob "container" "dir/blob2" "blob2")) (is (= 3 (count (list-container "container")))) (is (= 4 (count (list-container "container" :recursive)))) (is (= 1 (count (list-container "container" :in-directory "dir"))))) @@ -57,7 +57,7 @@ data "test content" data-file (java.io.File/createTempFile "jclouds" "data")] (try (create-container container-name) - (create-blob container-name name data) + (upload-blob container-name name data) (download-blob container-name name data-file) (is (= data (slurp (.getAbsolutePath data-file)))) (finally (.delete data-file))))) @@ -74,7 +74,7 @@ data "test content" data-file (java.io.File/createTempFile "jclouds" "data")] (try (create-container container-name) - (create-blob container-name name data) + (upload-blob container-name name data) (is (thrown? Exception (download-blob container-name name data-file))) (finally (.delete data-file)))))) @@ -92,7 +92,7 @@ (when-not (blob-exists? container-name name) (let [data-stream (java.io.ByteArrayOutputStream.)] (dotimes [i 5000000] (.write data-stream i)) - (create-blob container-name name (.toByteArray data-stream)))) + (upload-blob container-name name (.toByteArray data-stream)))) ;; download (let [total (atom total-downloads)] From a87560113e2c9d31057f19acf23db2bb894b735c Mon Sep 17 00:00:00 2001 From: phil Date: Fri, 26 Mar 2010 01:09:21 +0000 Subject: [PATCH 3/6] Remove magic ListOptionsContainer macros in favour of explicit fn mapping. --- .../main/clojure/org/jclouds/blobstore.clj | 46 ++++++------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/blobstore/src/main/clojure/org/jclouds/blobstore.clj b/blobstore/src/main/clojure/org/jclouds/blobstore.clj index 433b4b716c..93729981f0 100644 --- a/blobstore/src/main/clojure/org/jclouds/blobstore.clj +++ b/blobstore/src/main/clojure/org/jclouds/blobstore.clj @@ -34,8 +34,7 @@ See http://code.google.com/p/jclouds for details." Options for communication style :sync and :async. Options can also be specified for extension modules - :log4j :enterprise :httpnio :apachehc :bouncycastle :joda :gae -" + :log4j :enterprise :httpnio :apachehc :bouncycastle :joda :gae" [#^String service #^String account #^String key & options] (let [context (.createContext @@ -77,48 +76,30 @@ Options can also be specified for extension modules `(binding [*blobstore* (as-blobstore ~@blobstore-or-args)] ~@body)) -(defn- parse-args - "Parses arguments, recognises keywords in the set single as boolean switches." - [args single default] - (loop [[arg :as args] args - opts default] - (if-not args - opts - (if (single arg) - (recur (next args) (assoc opts arg true)) - (recur (nnext args) (assoc opts arg (second args))))))) - -(def list-options - (apply array-map - (concat (make-option-map option-fn-0arg [:recursive]) - (make-option-map option-fn-1arg [:after-marker :in-directory - :max-results])))) - -(defn- list-options-apply - [single target key value] - (if (single key) - ((list-options key) target) - ((list-options key) target value)) - target) - (defn containers "List all containers in a blobstore." ([] (containers *blobstore*)) ([blobstore] (.list blobstore))) +(def #^{:private true} list-option-map + {:after-marker #(.afterMarker %1 %2) + :in-directory #(.inDirectory %1 %2) + :max-results #(.maxResults %1 %2) + :recursive #(when %2 (.recursive %1))}) + (defn list-container "List a container. Options are: :after-marker string :in-direcory path :max-results n - :recursive" + :recursive true" [blobstore & args] (if (blobstore? blobstore) - (let [single-keywords #{:recursive} - options (parse-args (next args) single-keywords {}) + (let [options (apply hash-map args) list-options (reduce - #(list-options-apply single-keywords %1 - (first %2) (second %2)) + (fn [lco [k v]] + ((list-option-map k) lco v) + lco) (ListContainerOptions.) options)] (.list blobstore (first args) list-options)) @@ -228,8 +209,7 @@ example: (pprint (blobs (blobstore-context flightcaster-creds) - \"somecontainer\" \"some-dir\")) -" + \"somecontainer\" \"some-dir\"))" ([blobstore container-name] (.list (as-blobstore blobstore) container-name)) From f17352c70fc19aa139fb2277ed3a3d9ed2878281 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Fri, 26 Mar 2010 10:30:55 -0700 Subject: [PATCH 4/6] Handle container-name arg correctly in list-container. --- blobstore/src/main/clojure/org/jclouds/blobstore.clj | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/blobstore/src/main/clojure/org/jclouds/blobstore.clj b/blobstore/src/main/clojure/org/jclouds/blobstore.clj index 93729981f0..c8a353c673 100644 --- a/blobstore/src/main/clojure/org/jclouds/blobstore.clj +++ b/blobstore/src/main/clojure/org/jclouds/blobstore.clj @@ -95,14 +95,15 @@ Options can also be specified for extension modules :recursive true" [blobstore & args] (if (blobstore? blobstore) - (let [options (apply hash-map args) + (let [[container-name & args] args + options (apply hash-map args) list-options (reduce (fn [lco [k v]] ((list-option-map k) lco v) lco) (ListContainerOptions.) options)] - (.list blobstore (first args) list-options)) + (.list blobstore container-name list-options)) (apply list-container *blobstore* blobstore args))) (defn create-container @@ -234,7 +235,7 @@ container, name, string -> etag" download-blob (fn [& args] (if (= (count args) 3) ::short-form - (class (last args))))) + (class (last (butlast args)))))) (defmethod download-blob ::short-form [container-name name target] From a5454fe0402308a763737e7e3a9c4ecf7487b3c0 Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Fri, 26 Mar 2010 10:31:14 -0700 Subject: [PATCH 5/6] Update clojure blobstore tests to reflect new argument order. --- .../clojure/org/jclouds/blobstore_test.clj | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj b/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj index 72be8f158d..424fe024c1 100644 --- a/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj +++ b/blobstore/src/test/clojure/org/jclouds/blobstore_test.clj @@ -24,19 +24,18 @@ (is (blobstore? (as-blobstore (blobstore-context *blobstore*))))) (deftest create-existing-container-test - (is (not (container-exists? *blobstore* ""))) (is (not (container-exists? ""))) - (is (create-container *blobstore* "fred")) - (is (container-exists? *blobstore* "fred"))) + (is (create-container "fred")) + (is (container-exists? "fred"))) (deftest create-container-test - (is (create-container *blobstore* "fred")) - (is (container-exists? *blobstore* "fred"))) + (is (create-container "fred")) + (is (container-exists? "fred"))) (deftest containers-test - (is (empty? (containers *blobstore*))) - (is (create-container *blobstore* "fred")) - (is (= 1 (count (containers *blobstore*))))) + (is (empty? (containers))) + (is (create-container "fred")) + (is (= 1 (count (containers))))) (deftest list-container-test (is (create-container "container")) @@ -48,7 +47,7 @@ (create-directory "container" "dir") (is (upload-blob "container" "dir/blob2" "blob2")) (is (= 3 (count (list-container "container")))) - (is (= 4 (count (list-container "container" :recursive)))) + (is (= 4 (count (list-container "container" :recursive true)))) (is (= 1 (count (list-container "container" :in-directory "dir"))))) (deftest download-blob-test @@ -103,7 +102,7 @@ (when-not (<= @total 0) (with-open [baos (java.io.ByteArrayOutputStream.)] (try - (download-blob blob-s container-name file baos) + (download-blob container-name file baos blob-s) (catch Exception e (with-open [of (java.io.FileOutputStream. (java.io.File/createTempFile "jclouds" ".dl"))] From 9b510a9efe5a8a38ff7cb83a3fa05978a73c8459 Mon Sep 17 00:00:00 2001 From: Alex Yarmula Date: Fri, 26 Mar 2010 12:15:17 -0700 Subject: [PATCH 6/6] corrected BaseJettyTest: by HTML spec, HEAD requests shouldnt get any body in response --- .../test/java/org/jclouds/http/BaseJettyTest.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/org/jclouds/http/BaseJettyTest.java b/core/src/test/java/org/jclouds/http/BaseJettyTest.java index 6eae4c75d9..15c5eca3f4 100644 --- a/core/src/test/java/org/jclouds/http/BaseJettyTest.java +++ b/core/src/test/java/org/jclouds/http/BaseJettyTest.java @@ -170,7 +170,7 @@ public abstract class BaseJettyTest { if (failIfNoContentLength(request, response)) { return; } else if (target.indexOf("redirect") > 0) { - response.sendRedirect("http://localhost:" + (testPort + 1)); + response.sendRedirect("http://localhost:" + (testPort + 1) + "/"); } else if (target.indexOf("101constitutions") > 0) { response.setContentType("text/plain"); response.setHeader("Content-MD5", md5); @@ -213,6 +213,11 @@ public abstract class BaseJettyTest { response.setContentType("text/plain"); response.setStatus(HttpServletResponse.SC_OK); response.getWriter().println("test"); + } else if (request.getMethod().equals("HEAD")) { + /* NOTE: by HTML specification, HEAD response MUST NOT + include a body */ + response.setContentType("text/xml"); + response.setStatus(HttpServletResponse.SC_OK); } else { if (failEveryTenRequests(request, response)) return; @@ -257,6 +262,11 @@ public abstract class BaseJettyTest { } else { handleAction(request, response); } + } else if (request.getMethod().equals("HEAD")) { + /* NOTE: by HTML specification, HEAD response MUST NOT + include a body */ + response.setContentType("text/xml"); + response.setStatus(HttpServletResponse.SC_OK); } else { response.setContentType("text/xml"); response.setStatus(HttpServletResponse.SC_OK);