discourse/spec/jobs/clean_up_uploads_spec.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

391 lines
12 KiB
Ruby
Raw Normal View History

# frozen_string_literal: true
describe Jobs::CleanUpUploads do
def fabricate_upload(attributes = {})
Fabricate(:upload, { created_at: 2.hours.ago }.merge(attributes))
end
fab! :expired_upload do
fabricate_upload
end
2016-06-20 16:05:41 -04:00
before do
SiteSetting.clean_up_uploads = true
2016-06-20 16:05:41 -04:00
SiteSetting.clean_orphan_uploads_grace_period_hours = 1
# pre-fabrication resets created_at, so re-expire the upload
expired_upload
freeze_time 2.hours.from_now
Jobs::CleanUpUploads.new.reset_last_cleanup!
end
it "only runs upload cleanup every grace period / 2 time" do
SiteSetting.clean_orphan_uploads_grace_period_hours = 48
expired = fabricate_upload(created_at: 49.hours.ago)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired.id)).to eq(false)
upload = fabricate_upload(created_at: 72.hours.ago)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: upload.id)).to eq(true)
freeze_time 25.hours.from_now
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: upload.id)).to eq(false)
2016-06-20 16:05:41 -04:00
end
it "deletes orphan uploads" do
expect do
Jobs::CleanUpUploads.new.execute(nil)
end.to change { Upload.count }.by(-1)
2016-06-20 16:05:41 -04:00
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
end
describe 'unused callbacks' do
before do
Upload.add_unused_callback do |uploads|
uploads.where.not(id: expired_upload.id)
end
end
after do
Upload.reset_unused_callbacks
end
it 'does not delete uploads skipped by an unused callback' do
expect do
Jobs::CleanUpUploads.new.execute(nil)
end.to change { Upload.count }.by(0)
expect(Upload.exists?(id: expired_upload.id)).to eq(true)
end
it 'deletes other uploads not skipped by an unused callback' do
expired_upload2 = fabricate_upload
upload = fabricate_upload
UploadReference.create(target: Fabricate(:post), upload: upload)
expect do
Jobs::CleanUpUploads.new.execute(nil)
end.to change { Upload.count }.by(-1)
expect(Upload.exists?(id: expired_upload.id)).to eq(true)
expect(Upload.exists?(id: expired_upload2.id)).to eq(false)
expect(Upload.exists?(id: upload.id)).to eq(true)
end
end
describe 'in use callbacks' do
before do
Upload.add_in_use_callback do |upload|
expired_upload.id == upload.id
end
end
after do
Upload.reset_in_use_callbacks
end
it 'does not delete uploads that are in use by callback' do
expect do
Jobs::CleanUpUploads.new.execute(nil)
end.to change { Upload.count }.by(0)
expect(Upload.exists?(id: expired_upload.id)).to eq(true)
end
it 'deletes other uploads that are not in use by callback' do
expired_upload2 = fabricate_upload
upload = fabricate_upload
UploadReference.create(target: Fabricate(:post), upload: upload)
expect do
Jobs::CleanUpUploads.new.execute(nil)
end.to change { Upload.count }.by(-1)
expect(Upload.exists?(id: expired_upload.id)).to eq(true)
expect(Upload.exists?(id: expired_upload2.id)).to eq(false)
expect(Upload.exists?(id: upload.id)).to eq(true)
end
end
describe 'when clean_up_uploads is disabled' do
before do
SiteSetting.clean_up_uploads = false
end
it 'should still delete invalid upload records' do
upload2 = fabricate_upload(
url: "",
retain_hours: nil
)
expect do
Jobs::CleanUpUploads.new.execute(nil)
end.to change { Upload.count }.by(-1)
2016-06-20 16:05:41 -04:00
expect(Upload.exists?(id: expired_upload.id)).to eq(true)
expect(Upload.exists?(id: upload2.id)).to eq(false)
end
end
2016-06-20 16:05:41 -04:00
2018-11-14 02:03:02 -05:00
it 'does not clean up upload site settings' do
begin
original_provider = SiteSetting.provider
SiteSetting.provider = SiteSettings::DbProvider.new(SiteSetting)
SiteSetting.clean_orphan_uploads_grace_period_hours = 1
2018-11-14 02:03:02 -05:00
system_upload = fabricate_upload(id: -999)
2018-11-14 02:03:02 -05:00
logo_upload = fabricate_upload
logo_small_upload = fabricate_upload
digest_logo_upload = fabricate_upload
mobile_logo_upload = fabricate_upload
large_icon_upload = fabricate_upload
opengraph_image_upload = fabricate_upload
twitter_summary_large_image_upload = fabricate_upload
favicon_upload = fabricate_upload
apple_touch_icon_upload = fabricate_upload
SiteSetting.logo = logo_upload
SiteSetting.logo_small = logo_small_upload
SiteSetting.digest_logo = digest_logo_upload
SiteSetting.mobile_logo = mobile_logo_upload
SiteSetting.large_icon = large_icon_upload
SiteSetting.opengraph_image = opengraph_image_upload
SiteSetting.twitter_summary_large_image =
twitter_summary_large_image_upload
SiteSetting.favicon = favicon_upload
SiteSetting.apple_touch_icon = apple_touch_icon_upload
Jobs::CleanUpUploads.new.execute(nil)
[
logo_upload,
logo_small_upload,
digest_logo_upload,
mobile_logo_upload,
large_icon_upload,
opengraph_image_upload,
twitter_summary_large_image_upload,
favicon_upload,
apple_touch_icon_upload,
system_upload
2018-11-14 02:03:02 -05:00
].each { |record| expect(Upload.exists?(id: record.id)).to eq(true) }
fabricate_upload
SiteSetting.opengraph_image = ''
Jobs::CleanUpUploads.new.execute(nil)
2018-11-14 02:03:02 -05:00
ensure
SiteSetting.delete_all
SiteSetting.provider = original_provider
end
end
it "does not clean up selectable avatars" do
original_provider = SiteSetting.provider
SiteSetting.provider = SiteSettings::DbProvider.new(SiteSetting)
SiteSetting.clean_orphan_uploads_grace_period_hours = 1
2018-07-18 06:57:43 -04:00
avatar1_upload = fabricate_upload
avatar2_upload = fabricate_upload
SiteSetting.selectable_avatars = [avatar1_upload, avatar2_upload]
2016-09-08 16:58:07 -04:00
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
2018-07-18 06:57:43 -04:00
expect(Upload.exists?(id: avatar1_upload.id)).to eq(true)
expect(Upload.exists?(id: avatar2_upload.id)).to eq(true)
ensure
SiteSetting.delete_all
SiteSetting.provider = original_provider
2016-09-08 16:58:07 -04:00
end
it "does not delete profile background uploads" do
profile_background_upload = fabricate_upload
UserProfile.last.upload_profile_background(profile_background_upload)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: profile_background_upload.id)).to eq(true)
end
it "does not delete card background uploads" do
card_background_upload = fabricate_upload
UserProfile.last.upload_card_background(card_background_upload)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: card_background_upload.id)).to eq(true)
end
it "does not delete category logo uploads" do
category_logo_upload = fabricate_upload
Fabricate(:category, uploaded_logo: category_logo_upload)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: category_logo_upload.id)).to eq(true)
end
it "does not delete category background url uploads" do
category_logo_upload = fabricate_upload
Fabricate(:category, uploaded_background: category_logo_upload)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: category_logo_upload.id)).to eq(true)
end
it "does not delete post uploads" do
upload = fabricate_upload
2016-09-08 16:58:07 -04:00
Fabricate(:post, uploads: [upload])
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: upload.id)).to eq(true)
end
it "does not delete user uploaded avatar" do
upload = fabricate_upload
2016-09-08 16:58:07 -04:00
Fabricate(:user, uploaded_avatar: upload)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: upload.id)).to eq(true)
end
it "does not delete user gravatar" do
upload = fabricate_upload
2016-09-08 16:58:07 -04:00
Fabricate(:user, user_avatar: Fabricate(:user_avatar, gravatar_upload: upload))
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: upload.id)).to eq(true)
end
it "does not delete user custom upload" do
upload = fabricate_upload
Fabricate(:user, user_avatar: Fabricate(:user_avatar, custom_upload: upload))
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: upload.id)).to eq(true)
end
it "does not delete uploads in a queued post" do
upload = fabricate_upload
upload2 = fabricate_upload
upload3 = fabricate_upload
Fabricate(:reviewable_queued_post_topic, payload: {
raw: "#{upload.short_url}\n#{upload2.short_url}"
})
Fabricate(:reviewable_queued_post_topic,
payload: {
raw: "#{upload3.short_url}"
},
status: Reviewable.statuses[:rejected]
)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: upload.id)).to eq(true)
expect(Upload.exists?(id: upload2.id)).to eq(true)
expect(Upload.exists?(id: upload3.id)).to eq(false)
end
it "does not delete uploads in a draft" do
upload = fabricate_upload
upload2 = fabricate_upload
Draft.set(Fabricate(:user), "test", 0, "upload://#{upload.sha1}\n#{upload2.short_url}")
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: upload.id)).to eq(true)
expect(Upload.exists?(id: upload2.id)).to eq(true)
end
FEATURE: Secure media allowing duplicated uploads with category-level privacy and post-based access rules (#8664) ### General Changes and Duplication * We now consider a post `with_secure_media?` if it is in a read-restricted category. * When uploading we now set an upload's secure status straight away. * When uploading if `SiteSetting.secure_media` is enabled, we do not check to see if the upload already exists using the `sha1` digest of the upload. The `sha1` column of the upload is filled with a `SecureRandom.hex(20)` value which is the same length as `Upload::SHA1_LENGTH`. The `original_sha1` column is filled with the _real_ sha1 digest of the file. * Whether an upload `should_be_secure?` is now determined by whether the `access_control_post` is `with_secure_media?` (if there is no access control post then we leave the secure status as is). * When serializing the upload, we now cook the URL if the upload is secure. This is so it shows up correctly in the composer preview, because we set secure status on upload. ### Viewing Secure Media * The secure-media-upload URL will take the post that the upload is attached to into account via `Guardian.can_see?` for access permissions * If there is no `access_control_post` then we just deliver the media. This should be a rare occurrance and shouldn't cause issues as the `access_control_post` is set when `link_post_uploads` is called via `CookedPostProcessor` ### Removed We no longer do any of these because we do not reuse uploads by sha1 if secure media is enabled. * We no longer have a way to prevent cross-posting of a secure upload from a private context to a public context. * We no longer have to set `secure: false` for uploads when uploading for a theme component.
2020-01-15 22:50:27 -05:00
it "does not delete uploads with an access control post ID (secure uploads)" do
upload = fabricate_upload(access_control_post_id: Fabricate(:post).id, secure: true)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: upload.id)).to eq(true)
end
it "does not delete custom emojis" do
upload = fabricate_upload
CustomEmoji.create!(name: 'test', upload: upload)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: upload.id)).to eq(true)
end
it "does not delete user exported csv uploads" do
csv_file = fabricate_upload
UserExport.create(file_name: "export.csv", user_id: Fabricate(:user).id, upload_id: csv_file.id)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: csv_file.id)).to eq(true)
end
it "does not delete theme setting uploads" do
theme = Fabricate(:theme)
theme_upload = fabricate_upload
ThemeSetting.create!(theme: theme, data_type: ThemeSetting.types[:upload], value: theme_upload.id.to_s, name: "my_setting_name")
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: theme_upload.id)).to eq(true)
end
it "does not delete badges uploads" do
badge_image = fabricate_upload
badge = Fabricate(:badge, image_upload_id: badge_image.id)
Jobs::CleanUpUploads.new.execute(nil)
expect(Upload.exists?(id: expired_upload.id)).to eq(false)
expect(Upload.exists?(id: badge_image.id)).to eq(true)
end
FEATURE: Initial implementation of direct S3 uploads with uppy and stubs (#13787) This adds a few different things to allow for direct S3 uploads using uppy. **These changes are still not the default.** There are hidden `enable_experimental_image_uploader` and `enable_direct_s3_uploads` settings that must be turned on for any of this code to be used, and even if they are turned on only the User Card Background for the user profile actually uses uppy-image-uploader. A new `ExternalUploadStub` model and database table is introduced in this pull request. This is used to keep track of uploads that are uploaded to a temporary location in S3 with the direct to S3 code, and they are eventually deleted a) when the direct upload is completed and b) after a certain time period of not being used. ### Starting a direct S3 upload When an S3 direct upload is initiated with uppy, we first request a presigned PUT URL from the new `generate-presigned-put` endpoint in `UploadsController`. This generates an S3 key in the `temp` folder inside the correct bucket path, along with any metadata from the clientside (e.g. the SHA1 checksum described below). This will also create an `ExternalUploadStub` and store the details of the temp object key and the file being uploaded. Once the clientside has this URL, uppy will upload the file direct to S3 using the presigned URL. Once the upload is complete we go to the next stage. ### Completing a direct S3 upload Once the upload to S3 is done we call the new `complete-external-upload` route with the unique identifier of the `ExternalUploadStub` created earlier. Only the user who made the stub can complete the external upload. One of two paths is followed via the `ExternalUploadManager`. 1. If the object in S3 is too large (currently 100mb defined by `ExternalUploadManager::DOWNLOAD_LIMIT`) we do not download and generate the SHA1 for that file. Instead we create the `Upload` record via `UploadCreator` and simply copy it to its final destination on S3 then delete the initial temp file. Several modifications to `UploadCreator` have been made to accommodate this. 2. If the object in S3 is small enough, we download it. When the temporary S3 file is downloaded, we compare the SHA1 checksum generated by the browser with the actual SHA1 checksum of the file generated by ruby. The browser SHA1 checksum is stored on the object in S3 with metadata, and is generated via the `UppyChecksum` plugin. Keep in mind that some browsers will not generate this due to compatibility or other issues. We then follow the normal `UploadCreator` path with one exception. To cut down on having to re-upload the file again, if there are no changes (such as resizing etc) to the file in `UploadCreator` we follow the same copy + delete temp path that we do for files that are too large. 3. Finally we return the serialized upload record back to the client There are several errors that could happen that are handled by `UploadsController` as well. Also in this PR is some refactoring of `displayErrorForUpload` to handle both uppy and jquery file uploader errors.
2021-07-27 18:42:25 -04:00
it "deletes external upload stubs that have expired" do
external_stub1 = Fabricate(:external_upload_stub, status: ExternalUploadStub.statuses[:created], created_at: 10.minutes.ago)
external_stub2 = Fabricate(:external_upload_stub, status: ExternalUploadStub.statuses[:created], created_at: (ExternalUploadStub::CREATED_EXPIRY_HOURS.hours + 10.minutes).ago)
external_stub3 = Fabricate(:external_upload_stub, status: ExternalUploadStub.statuses[:uploaded], created_at: 10.minutes.ago)
external_stub4 = Fabricate(:external_upload_stub, status: ExternalUploadStub.statuses[:uploaded], created_at: (ExternalUploadStub::UPLOADED_EXPIRY_HOURS.hours + 10.minutes).ago)
Jobs::CleanUpUploads.new.execute(nil)
expect(ExternalUploadStub.pluck(:id)).to contain_exactly(external_stub1.id, external_stub3.id)
end
end