let's try 3 times to download images locally

This commit is contained in:
Régis Hanol 2017-10-11 23:11:44 +02:00
parent c9b5ef9f58
commit 4e78abb537
1 changed files with 24 additions and 18 deletions

View File

@ -9,10 +9,31 @@ module Jobs
sidekiq_options queue: 'low'
def initialize
# maximum size of the file in bytes
@max_size = SiteSetting.max_image_size_kb.kilobytes
end
def download(src)
downloaded = nil
begin
retries ||= 3
downloaded = FileHelper.download(
src,
max_file_size: @max_size,
tmp_file_name: "discourse-hotlinked",
follow_redirect: true
)
rescue
if (retries -= 1) > 0
sleep 1
retry
end
end
downloaded
end
def execute(args)
return unless SiteSetting.download_remote_images_to_local?
@ -32,23 +53,10 @@ module Jobs
src = "http:#{src}" if src.start_with?("//")
if is_valid_image_url(src)
hotlinked = nil
begin
# have we already downloaded that file?
unless downloaded_urls.include?(src)
begin
hotlinked = FileHelper.download(
src,
max_file_size: @max_size,
tmp_file_name: "discourse-hotlinked",
follow_redirect: true
)
rescue Discourse::InvalidParameters
log(:error, "InvalidParameters while downloading hotlinked image (#{src}) for post: #{post_id}")
rescue => e
log(:error, "Failed to download image #{e}")
end
if hotlinked
if hotlinked = download(src)
if File.size(hotlinked.path) <= @max_size
filename = File.basename(URI.parse(src).path)
filename << File.extname(hotlinked.path) unless filename["."]
@ -56,14 +64,12 @@ module Jobs
if upload.persisted?
downloaded_urls[src] = upload.url
else
log(:error, "Failed to pull hotlinked image for post: #{post_id}: #{src} - #{upload.errors.join("\n")}")
log(:info, "Failed to pull hotlinked image for post: #{post_id}: #{src} - #{upload.errors.join("\n")}")
end
else
log(:error, "Failed to pull hotlinked image for post: #{post_id}: #{src} - Image is bigger than #{@max_size}")
large_images << original_src
end
else
log(:error, "There was an error while downloading '#{src}' locally for post: #{post_id}")
broken_images << original_src
end
end