From 9a362869201be5ad258ead26a34c57460b3958ee Mon Sep 17 00:00:00 2001 From: David Taylor Date: Mon, 9 Sep 2024 18:39:26 +0100 Subject: [PATCH] FIX: Open file handles 'just in time' during s3 migration (#28806) Previously we were opening the file handles, then putting them in a queue for upload. If that queue grows too large, we can hit a maximum open files limit. This commit opens the file handle 'just in time', so the maximum number of open handles is equal to the upload concurrency (20). --- lib/file_store/to_s3_migration.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/file_store/to_s3_migration.rb b/lib/file_store/to_s3_migration.rb index ba8b79ee518..75b5be278ac 100644 --- a/lib/file_store/to_s3_migration.rb +++ b/lib/file_store/to_s3_migration.rb @@ -211,7 +211,8 @@ module FileStore UPLOAD_CONCURRENCY.times.map do Thread.new do while obj = queue.pop - if s3.put_object(obj[:options]) + opts_with_file = obj[:options].merge(body: File.open(obj[:path], "rb")) + if s3.put_object(opts_with_file) putc "." lock.synchronize { synced += 1 } else @@ -237,7 +238,6 @@ module FileStore options = { acl: SiteSetting.s3_use_acls ? "public-read" : nil, - body: File.open(path, "rb"), bucket: bucket, content_type: MiniMime.lookup_by_filename(name)&.content_type, content_md5: content_md5,