Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote branch 'origin'

This commit is contained in:
Andrew Phillips 2010-03-29 16:34:09 +02:00
parent 378e5ba91f 1fbf71a9d4
commit 1f6784b40f
3 changed files with 84 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
blobstore/src/main/clojure/org/jclouds/blobstore.clj
View File

@ -34,8 +34,7 @@ See http://code.google.com/p/jclouds for details."
Options for communication style Options for communication style
:sync and :async. :sync and :async.
Options can also be specified for extension modules Options can also be specified for extension modules
:log4j :enterprise :httpnio :apachehc :bouncycastle :joda :gae :log4j :enterprise :httpnio :apachehc :bouncycastle :joda :gae"
"
[#^String service #^String account #^String key & options] [#^String service #^String account #^String key & options]
(let [context (let [context
(.createContext (.createContext
@ -77,146 +76,127 @@ Options can also be specified for extension modules
`(binding [*blobstore* (as-blobstore ~@blobstore-or-args)] `(binding [*blobstore* (as-blobstore ~@blobstore-or-args)]
~@body)) ~@body))
(defn- parse-args
"Parses arguments, recognises keywords in the set single as boolean switches."
[args single default]
(loop [[arg :as args] args
opts default]
(if-not args
opts
(if (single arg)
(recur (next args) (assoc opts arg true))
(recur (nnext args) (assoc opts arg (second args)))))))
(def list-options
(apply array-map
(concat (make-option-map option-fn-0arg [:recursive])
(make-option-map option-fn-1arg [:after-marker :in-directory
:max-results]))))
(defn- list-options-apply
[single target key value]
(if (single key)
((list-options key) target)
((list-options key) target value))
target)
(defn containers (defn containers
"List all containers in a blobstore." "List all containers in a blobstore."
([] (containers *blobstore*)) ([] (containers *blobstore*))
([blobstore] (.list blobstore))) ([blobstore] (.list blobstore)))
(def #^{:private true} list-option-map
{:after-marker #(.afterMarker %1 %2)
:in-directory #(.inDirectory %1 %2)
:max-results #(.maxResults %1 %2)
:recursive #(when %2 (.recursive %1))})
(defn list-container (defn list-container
"List a container. Options are: "List a container. Options are:
:after-marker string :after-marker string
:in-direcory path :in-direcory path
:max-results n :max-results n
:recursive" :recursive true"
[blobstore & args] [blobstore & args]
(if (blobstore? blobstore) (if (blobstore? blobstore)
(let [single-keywords #{:recursive} (let [[container-name & args] args
options (parse-args (next args) single-keywords {}) options (apply hash-map args)
list-options (reduce list-options (reduce
#(list-options-apply single-keywords %1 (fn [lco [k v]]
(first %2) (second %2)) ((list-option-map k) lco v)
lco)
(ListContainerOptions.) (ListContainerOptions.)
options)] options)]
(.list blobstore (first args) list-options)) (.list blobstore container-name list-options))
(apply list-container *blobstore* blobstore args))) (apply list-container *blobstore* blobstore args)))
(defn create-container (defn create-container
"Create a container." "Create a container."
([container-name] ([container-name]
(create-container *blobstore* "default" container-name)) (create-container container-name "default" *blobstore*))
([blobstore container-name] ([container-name location-name]
(if (blobstore? blobstore) (create-container container-name location-name *blobstore*))
(create-container blobstore "default" container-name) ([container-name location-name blobstore]
(create-container *blobstore* container-name blobstore)))
([blobstore location-name container-name]
(.createContainerInLocation blobstore location-name container-name))) (.createContainerInLocation blobstore location-name container-name)))
(defn clear-container (defn clear-container
"Clear a container." "Clear a container."
([container-name] ([container-name]
(clear-container *blobstore* container-name)) (clear-container container-name *blobstore*))
([blobstore container-name] ([container-name blobstore]
(.clearContainer blobstore container-name))) (.clearContainer blobstore container-name)))
(defn delete-container (defn delete-container
"Delete a container." "Delete a container."
([container-name] ([container-name]
(delete-container *blobstore* container-name)) (delete-container container-name *blobstore*))
([blobstore container-name] ([container-name blobstore]
(.deleteContainer blobstore container-name))) (.deleteContainer blobstore container-name)))
(defn container-exists? (defn container-exists?
"Predicate to check presence of a container" "Predicate to check presence of a container"
([container-name] ([container-name]
(container-exists? *blobstore* container-name)) (container-exists? container-name *blobstore*))
([blobstore container-name] ([container-name blobstore]
(.containerExists blobstore container-name))) (.containerExists blobstore container-name)))
(defn directory-exists? (defn directory-exists?
"Predicate to check presence of a directory" "Predicate to check presence of a directory"
([container-name path] ([container-name path]
(directory-exists? *blobstore* container-name path)) (directory-exists? container-name path *blobstore*))
([blobstore container-name path] ([container-name path blobstore]
(.directoryExists blobstore container-name path))) (.directoryExists blobstore container-name path)))
(defn create-directory (defn create-directory
"Create a directory path." "Create a directory path."
([container-name path] ([container-name path]
(create-directory *blobstore* container-name path)) (create-directory container-name path *blobstore*))
([blobstore container-name path] ([container-name path blobstore]
(.createDirectory blobstore container-name path))) (.createDirectory blobstore container-name path)))
(defn delete-directory (defn delete-directory
"Delete a directory path." "Delete a directory path."
([container-name path] ([container-name path]
(delete-directory *blobstore* container-name path)) (delete-directory container-name path *blobstore*))
([blobstore container-name path] ([container-name path blobstore]
(.deleteDirectory blobstore container-name path))) (.deleteDirectory blobstore container-name path)))
(defn blob-exists? (defn blob-exists?
"Predicate to check presence of a blob" "Predicate to check presence of a blob"
([container-name path] ([container-name path]
(blob-exists? *blobstore* container-name path)) (blob-exists? container-name path *blobstore*))
([blobstore container-name path] ([container-name path blobstore]
(.blobExists blobstore container-name path))) (.blobExists blobstore container-name path)))
(defn put-blob (defn put-blob
"Put a blob. Metadata in the blob determines location." "Put a blob. Metadata in the blob determines location."
([container-name blob] ([container-name blob]
(put-blob *blobstore* container-name blob)) (put-blob container-name blob *blobstore*))
([blobstore container-name blob] ([container-name blob blobstore]
(.putBlob blobstore container-name blob))) (.putBlob blobstore container-name blob)))
(defn blob-metadata (defn blob-metadata
"Get blob metadata from given path" "Get blob metadata from given path"
([container-name path] ([container-name path]
(blob-metadata *blobstore* container-name path)) (blob-metadata container-name path *blobstore*))
([blobstore container-name path] ([container-name path blobstore]
(.blobMetadata blobstore container-name path))) (.blobMetadata blobstore container-name path)))
(defn get-blob (defn get-blob
"Get blob from given path" "Get blob from given path"
([container-name path] ([container-name path]
(get-blob *blobstore* container-name path)) (get-blob container-name path *blobstore*))
([blobstore container-name path] ([container-name path blobstore]
(.getBlob blobstore container-name path))) (.getBlob blobstore container-name path)))
(defn remove-blob (defn remove-blob
"Remove blob from given path" "Remove blob from given path"
([container-name path] ([container-name path]
(remove-blob *blobstore* container-name path)) (remove-blob container-name path *blobstore*))
([blobstore container-name path] ([container-name path blobstore]
(.removeBlob blobstore container-name path))) (.removeBlob blobstore container-name path)))
(defn count-blobs (defn count-blobs
"Count blobs" "Count blobs"
([container-name] ([container-name]
(count-blobs *blobstore* container-name)) (count-blobs container-name *blobstore*))
([blobstore container-name] ([container-name blobstore]
(.countBlob blobstore container-name))) (.countBlob blobstore container-name)))
(defn blobs (defn blobs
@ -230,8 +210,7 @@ example:
(pprint (pprint
(blobs (blobs
(blobstore-context flightcaster-creds) (blobstore-context flightcaster-creds)
\"somecontainer\" \"some-dir\")) \"somecontainer\" \"some-dir\"))"
"
([blobstore container-name] ([blobstore container-name]
(.list (as-blobstore blobstore) container-name)) (.list (as-blobstore blobstore) container-name))
@ -239,32 +218,32 @@ example:
(.list (as-blobstore blobstore) container-name (.list (as-blobstore blobstore) container-name
(.inDirectory (new ListContainerOptions) dir)))) (.inDirectory (new ListContainerOptions) dir))))
(defn create-blob (defn upload-blob
"Create an blob representing text data: "Create an blob representing text data:
container, name, string -> etag container, name, string -> etag"
"
([container-name name data] ;; TODO: allow payload to be a stream ([container-name name data] ;; TODO: allow payload to be a stream
(create-blob *blobstore* container-name name data)) (upload-blob container-name name data *blobstore*))
([blobstore container-name name data] ([container-name name data blobstore]
(put-blob blobstore container-name (put-blob container-name
(doto (.newBlob blobstore name) (doto (.newBlob blobstore name)
(.setPayload data) (.setPayload data)
(.generateMD5))))) (.generateMD5))
blobstore)))
(defmulti #^{:arglists '[[container-name name target] (defmulti #^{:arglists '[[container-name name target]
[blobstore container-name name target]]} [container-name name target blobstore]]}
download-blob (fn [& args] download-blob (fn [& args]
(if (= (count args) 3) (if (= (count args) 3)
::short-form ::short-form
(class (last args))))) (class (last (butlast args))))))
(defmethod download-blob ::short-form (defmethod download-blob ::short-form
[container-name name target] [container-name name target]
(download-blob *blobstore* container-name name target)) (download-blob container-name name target *blobstore*))
(defmethod download-blob OutputStream [blobstore container-name name target (defmethod download-blob OutputStream [container-name name target blobstore
& [retries]] & [retries]]
(let [blob (get-blob blobstore container-name name) (let [blob (get-blob container-name name blobstore)
digest-stream (.md5OutputStream ;; TODO: not all clouds use MD5 digest-stream (.md5OutputStream ;; TODO: not all clouds use MD5
*encryption-service* target)] *encryption-service* target)]
(copy (.getContent blob) digest-stream) (copy (.getContent blob) digest-stream)
@ -272,12 +251,12 @@ container, name, string -> etag
metadata-digest (.getContentMD5 (.getMetadata blob))] metadata-digest (.getContentMD5 (.getMetadata blob))]
(when-not (Arrays/equals digest metadata-digest) (when-not (Arrays/equals digest metadata-digest)
(if (<= (or retries 0) *max-retries*) (if (<= (or retries 0) *max-retries*)
(recur blobstore container-name name target [(inc (or retries 1))]) (recur container-name name target blobstore [(inc (or retries 1))])
(throw (Exception. (format "Download failed for %s/%s" (throw (Exception. (format "Download failed for %s/%s"
container-name name)))))))) container-name name))))))))
(defmethod download-blob File [blobstore container-name name target] (defmethod download-blob File [container-name name target blobstore]
(download-blob blobstore container-name name (FileOutputStream. target))) (download-blob container-name name (FileOutputStream. target) blobstore))
(define-accessors StorageMetadata "blob" type id name (define-accessors StorageMetadata "blob" type id name
location-id uri last-modfied) location-id uri last-modfied)

31
blobstore/src/test/clojure/org/jclouds/blobstore_test.clj
View File

@ -24,31 +24,30 @@
(is (blobstore? (as-blobstore (blobstore-context *blobstore*))))) (is (blobstore? (as-blobstore (blobstore-context *blobstore*)))))
(deftest create-existing-container-test (deftest create-existing-container-test
(is (not (container-exists? *blobstore* "")))
(is (not (container-exists? ""))) (is (not (container-exists? "")))
(is (create-container *blobstore* "fred")) (is (create-container "fred"))
(is (container-exists? *blobstore* "fred"))) (is (container-exists? "fred")))
(deftest create-container-test (deftest create-container-test
(is (create-container *blobstore* "fred")) (is (create-container "fred"))
(is (container-exists? *blobstore* "fred"))) (is (container-exists? "fred")))
(deftest containers-test (deftest containers-test
(is (empty? (containers *blobstore*))) (is (empty? (containers)))
(is (create-container *blobstore* "fred")) (is (create-container "fred"))
(is (= 1 (count (containers *blobstore*))))) (is (= 1 (count (containers)))))
(deftest list-container-test (deftest list-container-test
(is (create-container "container")) (is (create-container "container"))
(is (empty? (list-container "container"))) (is (empty? (list-container "container")))
(is (create-blob "container" "blob1" "blob1")) (is (upload-blob "container" "blob1" "blob1"))
(is (create-blob "container" "blob2" "blob2")) (is (upload-blob "container" "blob2" "blob2"))
(is (= 2 (count (list-container "container")))) (is (= 2 (count (list-container "container"))))
(is (= 1 (count (list-container "container" :max-results 1)))) (is (= 1 (count (list-container "container" :max-results 1))))
(create-directory "container" "dir") (create-directory "container" "dir")
(is (create-blob "container" "dir/blob2" "blob2")) (is (upload-blob "container" "dir/blob2" "blob2"))
(is (= 3 (count (list-container "container")))) (is (= 3 (count (list-container "container"))))
(is (= 4 (count (list-container "container" :recursive)))) (is (= 4 (count (list-container "container" :recursive true))))
(is (= 1 (count (list-container "container" :in-directory "dir"))))) (is (= 1 (count (list-container "container" :in-directory "dir")))))
(deftest download-blob-test (deftest download-blob-test
@ -57,7 +56,7 @@
data "test content" data "test content"
data-file (java.io.File/createTempFile "jclouds" "data")] data-file (java.io.File/createTempFile "jclouds" "data")]
(try (create-container container-name) (try (create-container container-name)
(create-blob container-name name data) (upload-blob container-name name data)
(download-blob container-name name data-file) (download-blob container-name name data-file)
(is (= data (slurp (.getAbsolutePath data-file)))) (is (= data (slurp (.getAbsolutePath data-file))))
(finally (.delete data-file))))) (finally (.delete data-file)))))
@ -74,7 +73,7 @@
data "test content" data "test content"
data-file (java.io.File/createTempFile "jclouds" "data")] data-file (java.io.File/createTempFile "jclouds" "data")]
(try (create-container container-name) (try (create-container container-name)
(create-blob container-name name data) (upload-blob container-name name data)
(is (thrown? Exception (is (thrown? Exception
(download-blob container-name name data-file))) (download-blob container-name name data-file)))
(finally (.delete data-file)))))) (finally (.delete data-file))))))
@ -92,7 +91,7 @@
(when-not (blob-exists? container-name name) (when-not (blob-exists? container-name name)
(let [data-stream (java.io.ByteArrayOutputStream.)] (let [data-stream (java.io.ByteArrayOutputStream.)]
(dotimes [i 5000000] (.write data-stream i)) (dotimes [i 5000000] (.write data-stream i))
(create-blob container-name name (.toByteArray data-stream)))) (upload-blob container-name name (.toByteArray data-stream))))
;; download ;; download
(let [total (atom total-downloads)] (let [total (atom total-downloads)]
@ -103,7 +102,7 @@
(when-not (<= @total 0) (when-not (<= @total 0)
(with-open [baos (java.io.ByteArrayOutputStream.)] (with-open [baos (java.io.ByteArrayOutputStream.)]
(try (try
(download-blob blob-s container-name file baos) (download-blob container-name file baos blob-s)
(catch Exception e (catch Exception e
(with-open [of (java.io.FileOutputStream. (with-open [of (java.io.FileOutputStream.
(java.io.File/createTempFile "jclouds" ".dl"))] (java.io.File/createTempFile "jclouds" ".dl"))]

12
core/src/test/java/org/jclouds/http/BaseJettyTest.java
View File

@ -170,7 +170,7 @@ public abstract class BaseJettyTest {
if (failIfNoContentLength(request, response)) { if (failIfNoContentLength(request, response)) {
return; return;
} else if (target.indexOf("redirect") > 0) { } else if (target.indexOf("redirect") > 0) {
response.sendRedirect("http://localhost:" + (testPort + 1)); response.sendRedirect("http://localhost:" + (testPort + 1) + "/");
} else if (target.indexOf("101constitutions") > 0) { } else if (target.indexOf("101constitutions") > 0) {
response.setContentType("text/plain"); response.setContentType("text/plain");
response.setHeader("Content-MD5", md5); response.setHeader("Content-MD5", md5);
@ -213,6 +213,11 @@ public abstract class BaseJettyTest {
response.setContentType("text/plain"); response.setContentType("text/plain");
response.setStatus(HttpServletResponse.SC_OK); response.setStatus(HttpServletResponse.SC_OK);
response.getWriter().println("test"); response.getWriter().println("test");
} else if (request.getMethod().equals("HEAD")) {
/* NOTE: by HTML specification, HEAD response MUST NOT
include a body */
response.setContentType("text/xml");
response.setStatus(HttpServletResponse.SC_OK);
} else { } else {
if (failEveryTenRequests(request, response)) if (failEveryTenRequests(request, response))
return; return;
@ -257,6 +262,11 @@ public abstract class BaseJettyTest {
} else { } else {
handleAction(request, response); handleAction(request, response);
} }
} else if (request.getMethod().equals("HEAD")) {
/* NOTE: by HTML specification, HEAD response MUST NOT
include a body */
response.setContentType("text/xml");
response.setStatus(HttpServletResponse.SC_OK);
} else { } else {
response.setContentType("text/xml"); response.setContentType("text/xml");
response.setStatus(HttpServletResponse.SC_OK); response.setStatus(HttpServletResponse.SC_OK);