Added retry to download-blob if MD5 checksums don't match.

This commit is contained in:
Jim 2010-03-17 16:02:09 -05:00 committed by Phil Hagelberg
parent d670ecfa6d
commit c0d8e90e72
2 changed files with 33 additions and 3 deletions

View File

@ -25,6 +25,7 @@ See http://code.google.com/p/jclouds for details."
AsyncBlobStore BlobStore BlobStoreContext BlobStoreContextFactory
domain.BlobMetadata domain.StorageMetadata domain.Blob
options.ListContainerOptions]
[java.security DigestOutputStream MessageDigest]
[com.google.common.collect ImmutableSet]))
(defn blobstore
@ -67,6 +68,8 @@ Options can also be specified for extension modules
(def *blobstore*)
(def *max-retries* 3)
(defmacro with-blobstore [[& blobstore-or-args] & body]
`(binding [*blobstore* (as-blobstore ~@blobstore-or-args)]
~@body))
@ -256,9 +259,19 @@ container, name, string -> etag
[container-name name target]
(download-blob *blobstore* container-name name target))
(defmethod download-blob OutputStream [blobstore container-name name target]
(let [blob (get-blob blobstore container-name name)]
(copy (.getContent blob) target)))
(defmethod download-blob OutputStream [blobstore container-name name target
& [retries]]
(let [blob (get-blob blobstore container-name name)
digest-stream (DigestOutputStream. ;; TODO: not all clouds use MD5
target (MessageDigest/getInstance "MD5"))]
(copy (.getContent blob) digest-stream)
(let [digest (.digest (.getMessageDigest digest-stream))
metadata-digest (.getContentMD5 (.getMetadata blob))]
(when-not (MessageDigest/isEqual digest metadata-digest)
(if (<= (or retries 0) *max-retries*)
(recur blobstore container-name name target [(inc (or retries 1))])
(throw (Exception. (format "Download failed for %s/%s"
container-name name))))))))
(defmethod download-blob File [blobstore container-name name target]
(download-blob blobstore container-name name (FileOutputStream. target)))

View File

@ -72,5 +72,22 @@
(is (= data (slurp (.getAbsolutePath data-file))))
(finally (.delete data-file)))))
(deftest download-checksum-test
(binding [get-blob (fn [blobstore c-name name]
(let [blob (.newBlob blobstore name)
md (.getMetadata blob)]
(.setPayload blob "bogus payload")
(.setContentMD5 md (.getBytes "bogus MD5"))
blob))]
(let [name "test"
container-name "test-container"
data "test content"
data-file (java.io.File/createTempFile "jclouds" "data")]
(try (create-container container-name)
(create-blob container-name name data)
(is (thrown? Exception
(download-blob container-name name data-file)))
(finally (.delete data-file))))))
;; TODO: more tests involving blob-specific functions