diff --git a/lib/s3_helper.rb b/lib/s3_helper.rb index 859ca462cac..d1a08cef371 100644 --- a/lib/s3_helper.rb +++ b/lib/s3_helper.rb @@ -105,6 +105,15 @@ class S3Helper rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NotFound end + def delete_objects(keys) + s3_bucket.delete_objects({ + delete: { + objects: keys.map { |k| { key: k } }, + quiet: true, + }, + }) + end + def copy(source, destination, options: {}) if options[:apply_metadata_to_destination] options = options.except(:apply_metadata_to_destination).merge(metadata_directive: "REPLACE") diff --git a/lib/tasks/s3.rake b/lib/tasks/s3.rake index 87553048360..d2a21fed37d 100644 --- a/lib/tasks/s3.rake +++ b/lib/tasks/s3.rake @@ -204,6 +204,8 @@ end task 's3:expire_missing_assets' => :environment do ensure_s3_configured! + puts "Checking for stale S3 assets..." + assets_to_delete = existing_assets.dup # Check that all current assets are uploaded, and remove them from the to_delete list @@ -217,13 +219,20 @@ task 's3:expire_missing_assets' => :environment do if assets_to_delete.size > 0 puts "Found #{assets_to_delete.size} assets to delete..." + assets_to_delete.each do |to_delete| if !to_delete.start_with?(prefix_s3_path("assets/")) # Sanity check, this should never happen raise "Attempted to delete a non-/asset S3 path (#{to_delete}). Aborting" end - puts "Deleting #{to_delete}..." - helper.delete_object(to_delete) + end + + assets_to_delete.each_slice(500) do |slice| + message = "Deleting #{slice.size} assets...\n" + message += slice.join("\n").indent(2) + puts message + helper.delete_objects(slice) + puts "... done" end else puts "No stale assets found" diff --git a/spec/lib/s3_helper_spec.rb b/spec/lib/s3_helper_spec.rb index 623580dbc0f..394cc6324b9 100644 --- a/spec/lib/s3_helper_spec.rb +++ b/spec/lib/s3_helper_spec.rb @@ -212,4 +212,14 @@ RSpec.describe "S3Helper" do expect(s3_helper.ensure_cors!([S3CorsRulesets::BACKUP_DIRECT_UPLOAD])).to eq(false) end end + + describe "#delete_objects" do + let(:s3_helper) { S3Helper.new("test-bucket", "", client: client) } + + it "works" do + # The S3::Client with `stub_responses: true` includes validation of requests. + # If the request were invalid, this spec would raise an error + s3_helper.delete_objects(["object/one.txt", "object/two.txt"]) + end + end end