DEV: Upload assets to S3 in parallel (#30210)

In my local setup (with Minio), this uploads the assets to S3 ~40% faster.
This commit is contained in:
Bianca Nenciu 2024-12-11 04:51:05 +02:00 committed by GitHub
parent 864b7b6bc8
commit b9f8a77d9b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 9 additions and 1 deletions

View File

@ -194,7 +194,15 @@ task "s3:ensure_cors_rules" => :environment do
end
task "s3:upload_assets" => [:environment, "s3:ensure_cors_rules"] do
assets.each { |asset| upload(*asset) }
pool =
Concurrent::FixedThreadPool.new(
ENV["DISCOURSE_S3_UPLOAD_ASSETS_RAKE_THREAD_POOL_SIZE"] || Concurrent.processor_count,
)
assets.each { |asset| pool.post { upload(*asset) } }
pool.shutdown
pool.wait_for_termination
end
task "s3:expire_missing_assets" => :environment do