mirror of https://github.com/apache/jclouds.git
Merge branch 'master' of github.com:jclouds/jclouds
This commit is contained in:
Alex Yarmula
commit
1fbf71a9d4
|
|
@ -34,8 +34,7 @@ See http://code.google.com/p/jclouds for details."
|
|||
Options for communication style
|
||||
:sync and :async.
|
||||
Options can also be specified for extension modules
|
||||
:log4j :enterprise :httpnio :apachehc :bouncycastle :joda :gae
|
||||
"
|
||||
:log4j :enterprise :httpnio :apachehc :bouncycastle :joda :gae"
|
||||
[#^String service #^String account #^String key & options]
|
||||
(let [context
|
||||
(.createContext
|
||||
|
|
@ -77,146 +76,127 @@ Options can also be specified for extension modules
|
|||
`(binding [*blobstore* (as-blobstore ~@blobstore-or-args)]
|
||||
~@body))
|
||||
|
||||
(defn- parse-args
|
||||
"Parses arguments, recognises keywords in the set single as boolean switches."
|
||||
[args single default]
|
||||
(loop [[arg :as args] args
|
||||
opts default]
|
||||
(if-not args
|
||||
opts
|
||||
(if (single arg)
|
||||
(recur (next args) (assoc opts arg true))
|
||||
(recur (nnext args) (assoc opts arg (second args)))))))
|
||||
|
||||
(def list-options
|
||||
(apply array-map
|
||||
(concat (make-option-map option-fn-0arg [:recursive])
|
||||
(make-option-map option-fn-1arg [:after-marker :in-directory
|
||||
:max-results]))))
|
||||
|
||||
(defn- list-options-apply
|
||||
[single target key value]
|
||||
(if (single key)
|
||||
((list-options key) target)
|
||||
((list-options key) target value))
|
||||
target)
|
||||
|
||||
(defn containers
|
||||
"List all containers in a blobstore."
|
||||
([] (containers *blobstore*))
|
||||
([blobstore] (.list blobstore)))
|
||||
|
||||
(def #^{:private true} list-option-map
|
||||
{:after-marker #(.afterMarker %1 %2)
|
||||
:in-directory #(.inDirectory %1 %2)
|
||||
:max-results #(.maxResults %1 %2)
|
||||
:recursive #(when %2 (.recursive %1))})
|
||||
|
||||
(defn list-container
|
||||
"List a container. Options are:
|
||||
:after-marker string
|
||||
:in-direcory path
|
||||
:max-results n
|
||||
:recursive"
|
||||
:recursive true"
|
||||
[blobstore & args]
|
||||
(if (blobstore? blobstore)
|
||||
(let [single-keywords #{:recursive}
|
||||
options (parse-args (next args) single-keywords {})
|
||||
(let [[container-name & args] args
|
||||
options (apply hash-map args)
|
||||
list-options (reduce
|
||||
#(list-options-apply single-keywords %1
|
||||
(first %2) (second %2))
|
||||
(fn [lco [k v]]
|
||||
((list-option-map k) lco v)
|
||||
lco)
|
||||
(ListContainerOptions.)
|
||||
options)]
|
||||
(.list blobstore (first args) list-options))
|
||||
(.list blobstore container-name list-options))
|
||||
(apply list-container *blobstore* blobstore args)))
|
||||
|
||||
(defn create-container
|
||||
"Create a container."
|
||||
([container-name]
|
||||
(create-container *blobstore* "default" container-name))
|
||||
([blobstore container-name]
|
||||
(if (blobstore? blobstore)
|
||||
(create-container blobstore "default" container-name)
|
||||
(create-container *blobstore* container-name blobstore)))
|
||||
([blobstore location-name container-name]
|
||||
(create-container container-name "default" *blobstore*))
|
||||
([container-name location-name]
|
||||
(create-container container-name location-name *blobstore*))
|
||||
([container-name location-name blobstore]
|
||||
(.createContainerInLocation blobstore location-name container-name)))
|
||||
|
||||
(defn clear-container
|
||||
"Clear a container."
|
||||
([container-name]
|
||||
(clear-container *blobstore* container-name))
|
||||
([blobstore container-name]
|
||||
(clear-container container-name *blobstore*))
|
||||
([container-name blobstore]
|
||||
(.clearContainer blobstore container-name)))
|
||||
|
||||
(defn delete-container
|
||||
"Delete a container."
|
||||
([container-name]
|
||||
(delete-container *blobstore* container-name))
|
||||
([blobstore container-name]
|
||||
(delete-container container-name *blobstore*))
|
||||
([container-name blobstore]
|
||||
(.deleteContainer blobstore container-name)))
|
||||
|
||||
(defn container-exists?
|
||||
"Predicate to check presence of a container"
|
||||
([container-name]
|
||||
(container-exists? *blobstore* container-name))
|
||||
([blobstore container-name]
|
||||
(container-exists? container-name *blobstore*))
|
||||
([container-name blobstore]
|
||||
(.containerExists blobstore container-name)))
|
||||
|
||||
(defn directory-exists?
|
||||
"Predicate to check presence of a directory"
|
||||
([container-name path]
|
||||
(directory-exists? *blobstore* container-name path))
|
||||
([blobstore container-name path]
|
||||
(directory-exists? container-name path *blobstore*))
|
||||
([container-name path blobstore]
|
||||
(.directoryExists blobstore container-name path)))
|
||||
|
||||
(defn create-directory
|
||||
"Create a directory path."
|
||||
([container-name path]
|
||||
(create-directory *blobstore* container-name path))
|
||||
([blobstore container-name path]
|
||||
(create-directory container-name path *blobstore*))
|
||||
([container-name path blobstore]
|
||||
(.createDirectory blobstore container-name path)))
|
||||
|
||||
(defn delete-directory
|
||||
"Delete a directory path."
|
||||
([container-name path]
|
||||
(delete-directory *blobstore* container-name path))
|
||||
([blobstore container-name path]
|
||||
(delete-directory container-name path *blobstore*))
|
||||
([container-name path blobstore]
|
||||
(.deleteDirectory blobstore container-name path)))
|
||||
|
||||
(defn blob-exists?
|
||||
"Predicate to check presence of a blob"
|
||||
([container-name path]
|
||||
(blob-exists? *blobstore* container-name path))
|
||||
([blobstore container-name path]
|
||||
(blob-exists? container-name path *blobstore*))
|
||||
([container-name path blobstore]
|
||||
(.blobExists blobstore container-name path)))
|
||||
|
||||
(defn put-blob
|
||||
"Put a blob. Metadata in the blob determines location."
|
||||
([container-name blob]
|
||||
(put-blob *blobstore* container-name blob))
|
||||
([blobstore container-name blob]
|
||||
(put-blob container-name blob *blobstore*))
|
||||
([container-name blob blobstore]
|
||||
(.putBlob blobstore container-name blob)))
|
||||
|
||||
(defn blob-metadata
|
||||
"Get blob metadata from given path"
|
||||
([container-name path]
|
||||
(blob-metadata *blobstore* container-name path))
|
||||
([blobstore container-name path]
|
||||
(blob-metadata container-name path *blobstore*))
|
||||
([container-name path blobstore]
|
||||
(.blobMetadata blobstore container-name path)))
|
||||
|
||||
(defn get-blob
|
||||
"Get blob from given path"
|
||||
([container-name path]
|
||||
(get-blob *blobstore* container-name path))
|
||||
([blobstore container-name path]
|
||||
(get-blob container-name path *blobstore*))
|
||||
([container-name path blobstore]
|
||||
(.getBlob blobstore container-name path)))
|
||||
|
||||
(defn remove-blob
|
||||
"Remove blob from given path"
|
||||
([container-name path]
|
||||
(remove-blob *blobstore* container-name path))
|
||||
([blobstore container-name path]
|
||||
(remove-blob container-name path *blobstore*))
|
||||
([container-name path blobstore]
|
||||
(.removeBlob blobstore container-name path)))
|
||||
|
||||
(defn count-blobs
|
||||
"Count blobs"
|
||||
([container-name]
|
||||
(count-blobs *blobstore* container-name))
|
||||
([blobstore container-name]
|
||||
(count-blobs container-name *blobstore*))
|
||||
([container-name blobstore]
|
||||
(.countBlob blobstore container-name)))
|
||||
|
||||
(defn blobs
|
||||
|
|
@ -230,8 +210,7 @@ example:
|
|||
(pprint
|
||||
(blobs
|
||||
(blobstore-context flightcaster-creds)
|
||||
\"somecontainer\" \"some-dir\"))
|
||||
"
|
||||
\"somecontainer\" \"some-dir\"))"
|
||||
([blobstore container-name]
|
||||
(.list (as-blobstore blobstore) container-name))
|
||||
|
||||
|
|
@ -239,32 +218,32 @@ example:
|
|||
(.list (as-blobstore blobstore) container-name
|
||||
(.inDirectory (new ListContainerOptions) dir))))
|
||||
|
||||
(defn create-blob
|
||||
(defn upload-blob
|
||||
"Create an blob representing text data:
|
||||
container, name, string -> etag
|
||||
"
|
||||
container, name, string -> etag"
|
||||
([container-name name data] ;; TODO: allow payload to be a stream
|
||||
(create-blob *blobstore* container-name name data))
|
||||
([blobstore container-name name data]
|
||||
(put-blob blobstore container-name
|
||||
(upload-blob container-name name data *blobstore*))
|
||||
([container-name name data blobstore]
|
||||
(put-blob container-name
|
||||
(doto (.newBlob blobstore name)
|
||||
(.setPayload data)
|
||||
(.generateMD5)))))
|
||||
(.generateMD5))
|
||||
blobstore)))
|
||||
|
||||
(defmulti #^{:arglists '[[container-name name target]
|
||||
[blobstore container-name name target]]}
|
||||
[container-name name target blobstore]]}
|
||||
download-blob (fn [& args]
|
||||
(if (= (count args) 3)
|
||||
::short-form
|
||||
(class (last args)))))
|
||||
(class (last (butlast args))))))
|
||||
|
||||
(defmethod download-blob ::short-form
|
||||
[container-name name target]
|
||||
(download-blob *blobstore* container-name name target))
|
||||
(download-blob container-name name target *blobstore*))
|
||||
|
||||
(defmethod download-blob OutputStream [blobstore container-name name target
|
||||
(defmethod download-blob OutputStream [container-name name target blobstore
|
||||
& [retries]]
|
||||
(let [blob (get-blob blobstore container-name name)
|
||||
(let [blob (get-blob container-name name blobstore)
|
||||
digest-stream (.md5OutputStream ;; TODO: not all clouds use MD5
|
||||
*encryption-service* target)]
|
||||
(copy (.getContent blob) digest-stream)
|
||||
|
|
@ -272,12 +251,12 @@ container, name, string -> etag
|
|||
metadata-digest (.getContentMD5 (.getMetadata blob))]
|
||||
(when-not (Arrays/equals digest metadata-digest)
|
||||
(if (<= (or retries 0) *max-retries*)
|
||||
(recur blobstore container-name name target [(inc (or retries 1))])
|
||||
(recur container-name name target blobstore [(inc (or retries 1))])
|
||||
(throw (Exception. (format "Download failed for %s/%s"
|
||||
container-name name))))))))
|
||||
|
||||
(defmethod download-blob File [blobstore container-name name target]
|
||||
(download-blob blobstore container-name name (FileOutputStream. target)))
|
||||
(defmethod download-blob File [container-name name target blobstore]
|
||||
(download-blob container-name name (FileOutputStream. target) blobstore))
|
||||
|
||||
(define-accessors StorageMetadata "blob" type id name
|
||||
location-id uri last-modfied)
|
||||
|
|
|
|||
|
|
@ -24,31 +24,30 @@
|
|||
(is (blobstore? (as-blobstore (blobstore-context *blobstore*)))))
|
||||
|
||||
(deftest create-existing-container-test
|
||||
(is (not (container-exists? *blobstore* "")))
|
||||
(is (not (container-exists? "")))
|
||||
(is (create-container *blobstore* "fred"))
|
||||
(is (container-exists? *blobstore* "fred")))
|
||||
(is (create-container "fred"))
|
||||
(is (container-exists? "fred")))
|
||||
|
||||
(deftest create-container-test
|
||||
(is (create-container *blobstore* "fred"))
|
||||
(is (container-exists? *blobstore* "fred")))
|
||||
(is (create-container "fred"))
|
||||
(is (container-exists? "fred")))
|
||||
|
||||
(deftest containers-test
|
||||
(is (empty? (containers *blobstore*)))
|
||||
(is (create-container *blobstore* "fred"))
|
||||
(is (= 1 (count (containers *blobstore*)))))
|
||||
(is (empty? (containers)))
|
||||
(is (create-container "fred"))
|
||||
(is (= 1 (count (containers)))))
|
||||
|
||||
(deftest list-container-test
|
||||
(is (create-container "container"))
|
||||
(is (empty? (list-container "container")))
|
||||
(is (create-blob "container" "blob1" "blob1"))
|
||||
(is (create-blob "container" "blob2" "blob2"))
|
||||
(is (upload-blob "container" "blob1" "blob1"))
|
||||
(is (upload-blob "container" "blob2" "blob2"))
|
||||
(is (= 2 (count (list-container "container"))))
|
||||
(is (= 1 (count (list-container "container" :max-results 1))))
|
||||
(create-directory "container" "dir")
|
||||
(is (create-blob "container" "dir/blob2" "blob2"))
|
||||
(is (upload-blob "container" "dir/blob2" "blob2"))
|
||||
(is (= 3 (count (list-container "container"))))
|
||||
(is (= 4 (count (list-container "container" :recursive))))
|
||||
(is (= 4 (count (list-container "container" :recursive true))))
|
||||
(is (= 1 (count (list-container "container" :in-directory "dir")))))
|
||||
|
||||
(deftest download-blob-test
|
||||
|
|
@ -57,7 +56,7 @@
|
|||
data "test content"
|
||||
data-file (java.io.File/createTempFile "jclouds" "data")]
|
||||
(try (create-container container-name)
|
||||
(create-blob container-name name data)
|
||||
(upload-blob container-name name data)
|
||||
(download-blob container-name name data-file)
|
||||
(is (= data (slurp (.getAbsolutePath data-file))))
|
||||
(finally (.delete data-file)))))
|
||||
|
|
@ -74,7 +73,7 @@
|
|||
data "test content"
|
||||
data-file (java.io.File/createTempFile "jclouds" "data")]
|
||||
(try (create-container container-name)
|
||||
(create-blob container-name name data)
|
||||
(upload-blob container-name name data)
|
||||
(is (thrown? Exception
|
||||
(download-blob container-name name data-file)))
|
||||
(finally (.delete data-file))))))
|
||||
|
|
@ -92,7 +91,7 @@
|
|||
(when-not (blob-exists? container-name name)
|
||||
(let [data-stream (java.io.ByteArrayOutputStream.)]
|
||||
(dotimes [i 5000000] (.write data-stream i))
|
||||
(create-blob container-name name (.toByteArray data-stream))))
|
||||
(upload-blob container-name name (.toByteArray data-stream))))
|
||||
|
||||
;; download
|
||||
(let [total (atom total-downloads)]
|
||||
|
|
@ -103,7 +102,7 @@
|
|||
(when-not (<= @total 0)
|
||||
(with-open [baos (java.io.ByteArrayOutputStream.)]
|
||||
(try
|
||||
(download-blob blob-s container-name file baos)
|
||||
(download-blob container-name file baos blob-s)
|
||||
(catch Exception e
|
||||
(with-open [of (java.io.FileOutputStream.
|
||||
(java.io.File/createTempFile "jclouds" ".dl"))]
|
||||
|
|
|
|||
Loading…
Reference in New Issue