FIX: Open file handles 'just in time' during s3 migration (#28806)
Previously we were opening the file handles, then putting them in a queue for upload. If that queue grows too large, we can hit a maximum open files limit. This commit opens the file handle 'just in time', so the maximum number of open handles is equal to the upload concurrency (20).
This commit is contained in:
parent
78e3436056
commit
9a36286920
|
@ -211,7 +211,8 @@ module FileStore
|
|||
UPLOAD_CONCURRENCY.times.map do
|
||||
Thread.new do
|
||||
while obj = queue.pop
|
||||
if s3.put_object(obj[:options])
|
||||
opts_with_file = obj[:options].merge(body: File.open(obj[:path], "rb"))
|
||||
if s3.put_object(opts_with_file)
|
||||
putc "."
|
||||
lock.synchronize { synced += 1 }
|
||||
else
|
||||
|
@ -237,7 +238,6 @@ module FileStore
|
|||
|
||||
options = {
|
||||
acl: SiteSetting.s3_use_acls ? "public-read" : nil,
|
||||
body: File.open(path, "rb"),
|
||||
bucket: bucket,
|
||||
content_type: MiniMime.lookup_by_filename(name)&.content_type,
|
||||
content_md5: content_md5,
|
||||
|
|
Loading…
Reference in New Issue