diff --git a/lib/s3_helper.rb b/lib/s3_helper.rb index 5903c650624..d1e5a0aa78f 100644 --- a/lib/s3_helper.rb +++ b/lib/s3_helper.rb @@ -44,31 +44,71 @@ class S3Helper rescue Aws::S3::Errors::NoSuchKey end - def update_lifecycle(id, days, prefix: nil) + # make sure we have a cors config for assets + # otherwise we will have no fonts + def ensure_cors! + rule = nil + + begin + rule = s3_resource.client.get_bucket_cors( + bucket: @s3_bucket_name + ).cors_rules&.first + rescue Aws::S3::Errors::NoSuchCORSConfiguration + # no rule + end + + unless rule + puts "installing CORS rule" + + s3_resource.client.put_bucket_cors( + bucket: @s3_bucket_name, + cors_configuration: { + cors_rules: [{ + allowed_headers: ["Authorization"], + allowed_methods: ["GET", "HEAD"], + allowed_origins: ["*"], + max_age_seconds: 3000 + }] + } + ) + end + end + + def update_lifecycle(id, days, prefix: nil, tag: nil) + + filter = {} + + if prefix + filter[:prefix] = prefix + elsif tag + filter[:tag] = tag + end # cf. http://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html rule = { id: id, status: "Enabled", - expiration: { days: days } + expiration: { days: days }, + filter: filter } - if prefix - rule[:prefix] = prefix - end + rules = [] - rules = s3_resource.client.get_bucket_lifecycle_configuration(bucket: @s3_bucket_name).rules + begin + rules = s3_resource.client.get_bucket_lifecycle_configuration(bucket: @s3_bucket_name).rules + rescue Aws::S3::Errors::NoSuchLifecycleConfiguration + # skip trying to merge + end rules.delete_if do |r| r.id == id end - rules.map! { |r| r.to_h } - rules << rule - s3_resource.client.put_bucket_lifecycle(bucket: @s3_bucket_name, - lifecycle_configuration: { + s3_resource.client.put_bucket_lifecycle_configuration( + bucket: @s3_bucket_name, + lifecycle_configuration: { rules: rules }) end diff --git a/lib/tasks/s3.rake b/lib/tasks/s3.rake index bc35e78b40b..7b2889ae938 100644 --- a/lib/tasks/s3.rake +++ b/lib/tasks/s3.rake @@ -71,28 +71,8 @@ def assets results end -def in_manifest - found = [] - assets.each do |_, path| - fullpath = (Rails.root + "public/assets/#{path}").to_s - - asset_path = "assets/#{path}" - found << asset_path - - if File.exist?(fullpath + '.br') - found << brotli_s3_path(asset_path) - end - - if File.exist?(fullpath + '.gz') - found << gzip_s3_path(asset_path) - end - - if File.exist?(fullpath + '.map') - found << asset_path + '.map' - end - - end - Set.new(found) +def asset_paths + Set.new(assets.map { |_, asset_path| asset_path }) end def ensure_s3_configured! @@ -104,6 +84,7 @@ end task 's3:upload_assets' => :environment do ensure_s3_configured! + helper.ensure_cors! assets.each do |asset| upload(*asset) @@ -112,25 +93,27 @@ end task 's3:expire_missing_assets' => :environment do ensure_s3_configured! - keep = in_manifest count = 0 + keep = 0 + + in_manifest = asset_paths + puts "Ensuring AWS assets are tagged correctly for removal" - helper.list.each do |f| - if keep.include?(f.key) + helper.list('assets/').each do |f| + if !in_manifest.include?(f.key) helper.tag_file(f.key, old: true) count += 1 else # ensure we do not delete this by mistake helper.tag_file(f.key, {}) + keep += 1 end end - puts "#{count} assets were flagged for removal in 10 days" + puts "#{count} assets were flagged for removal in 10 days (#{keep} assets will be retained)" puts "Ensuring AWS rule exists for purging old assets" - #helper.update_lifecycle("delete_old_assets", 10, prefix: 'old=true') - - puts "Waiting on https://github.com/aws/aws-sdk-ruby/issues/1623" + helper.update_lifecycle("delete_old_assets", 10, tag: { key: 'old', value: 'true' }) end