# frozen_string_literal: true require 'rails_helper' RSpec.describe ExternalUploadManager do fab!(:user) { Fabricate(:user) } let(:type) { "card_background" } let!(:logo_file) { file_from_fixtures("logo.png") } let!(:pdf_file) { file_from_fixtures("large.pdf", "pdf") } let(:object_size) { 1.megabyte } let(:etag) { "e696d20564859cbdf77b0f51cbae999a" } let(:client_sha1) { Upload.generate_digest(object_file) } let(:sha1) { Upload.generate_digest(object_file) } let(:object_file) { logo_file } let(:metadata_headers) { {} } let!(:external_upload_stub) { Fabricate(:image_external_upload_stub, created_by: user) } let(:upload_base_url) { "https://#{SiteSetting.s3_upload_bucket}.s3.#{SiteSetting.s3_region}.amazonaws.com" } subject do ExternalUploadManager.new(external_upload_stub) end before do SiteSetting.authorized_extensions += "|pdf" SiteSetting.max_attachment_size_kb = 210.megabytes / 1000 setup_s3 SiteSetting.s3_backup_bucket = "s3-backup-bucket" SiteSetting.backup_location = BackupLocationSiteSetting::S3 stub_head_object stub_download_object_filehelper stub_copy_object stub_delete_object end describe "#ban_user_from_external_uploads!" do after { Discourse.redis.flushdb } it "bans the user from external uploads using a redis key" do ExternalUploadManager.ban_user_from_external_uploads!(user: user) expect(ExternalUploadManager.user_banned?(user)).to eq(true) end end describe "#can_promote?" do it "returns false if the external stub status is not created" do external_upload_stub.update!(status: ExternalUploadStub.statuses[:uploaded]) expect(subject.can_promote?).to eq(false) end end describe "#transform!" do context "when stubbed upload is < DOWNLOAD_LIMIT (small enough to download + generate sha)" do let!(:external_upload_stub) { Fabricate(:image_external_upload_stub, created_by: user, filesize: object_size) } let(:object_size) { 1.megabyte } let(:object_file) { logo_file } context "when the download of the s3 file fails" do before do FileHelper.stubs(:download).returns(nil) end it "raises an error" do expect { subject.transform! }.to raise_error(ExternalUploadManager::DownloadFailedError) end end context "when the upload is not in the created status" do before do external_upload_stub.update!(status: ExternalUploadStub.statuses[:uploaded]) end it "raises an error" do expect { subject.transform! }.to raise_error(ExternalUploadManager::CannotPromoteError) end end context "when the upload does not get changed in UploadCreator (resized etc.)" do it "copies the stubbed upload on S3 to its new destination and deletes it" do upload = subject.transform! expect(WebMock).to have_requested( :put, "#{upload_base_url}/#{Discourse.store.get_path_for_upload(upload)}", ).with(headers: { 'X-Amz-Copy-Source' => "#{SiteSetting.s3_upload_bucket}/#{external_upload_stub.key}" }) expect(WebMock).to have_requested( :delete, "#{upload_base_url}/#{external_upload_stub.key}" ) end it "errors if the image upload is too big" do SiteSetting.max_image_size_kb = 1 upload = subject.transform! expect(upload.errors.full_messages).to include( "Filesize " + I18n.t("upload.images.too_large_humanized", max_size: ActiveSupport::NumberHelper.number_to_human_size(SiteSetting.max_image_size_kb.kilobytes)) ) end it "errors if the extension is not supported" do SiteSetting.authorized_extensions = "" upload = subject.transform! expect(upload.errors.full_messages).to include( "Original filename " + I18n.t("upload.unauthorized", authorized_extensions: "") ) end end context "when the upload does get changed by the UploadCreator" do let(:file) { file_from_fixtures("should_be_jpeg.heic", "images") } it "creates a new upload in s3 (not copy) and deletes the original stubbed upload" do upload = subject.transform! expect(WebMock).to have_requested( :put, "#{upload_base_url}/#{Discourse.store.get_path_for_upload(upload)}", ) expect(WebMock).to have_requested( :delete, "#{upload_base_url}/#{external_upload_stub.key}" ) end end context "when the sha has been set on the s3 object metadata by the clientside JS" do let(:metadata_headers) { { "x-amz-meta-sha1-checksum" => client_sha1 } } context "when the downloaded file sha1 does not match the client sha1" do let(:client_sha1) { "blahblah" } it "raises an error, deletes the stub" do expect { subject.transform! }.to raise_error(ExternalUploadManager::ChecksumMismatchError) expect(ExternalUploadStub.exists?(id: external_upload_stub.id)).to eq(false) end it "does not delete the stub if enable_upload_debug_mode" do SiteSetting.enable_upload_debug_mode = true expect { subject.transform! }.to raise_error(ExternalUploadManager::ChecksumMismatchError) external_stub = ExternalUploadStub.find(external_upload_stub.id) expect(external_stub.status).to eq(ExternalUploadStub.statuses[:failed]) end end end context "when the downloaded file size does not match the expected file size for the upload stub" do before do external_upload_stub.update!(filesize: 10) end after { Discourse.redis.flushdb } it "raises an error, deletes the file immediately, and prevents the user from uploading external files for a few minutes" do expect { subject.transform! }.to raise_error(ExternalUploadManager::SizeMismatchError) expect(ExternalUploadStub.exists?(id: external_upload_stub.id)).to eq(false) expect(Discourse.redis.get("#{ExternalUploadManager::BAN_USER_REDIS_PREFIX}#{external_upload_stub.created_by_id}")).to eq("1") expect(WebMock).to have_requested( :delete, "#{upload_base_url}/#{external_upload_stub.key}" ) end it "does not delete the stub if enable_upload_debug_mode" do SiteSetting.enable_upload_debug_mode = true expect { subject.transform! }.to raise_error(ExternalUploadManager::SizeMismatchError) external_stub = ExternalUploadStub.find(external_upload_stub.id) expect(external_stub.status).to eq(ExternalUploadStub.statuses[:failed]) end end end context "when stubbed upload is > DOWNLOAD_LIMIT (too big to download, generate a fake sha)" do let(:object_size) { 200.megabytes } let(:object_file) { pdf_file } let!(:external_upload_stub) { Fabricate(:attachment_external_upload_stub, created_by: user, filesize: object_size) } before do UploadCreator.any_instance.stubs(:generate_fake_sha1_hash).returns("testbc60eb18e8f974cbfae8bb0f069c3a311024") end it "does not try and download the file" do FileHelper.expects(:download).never subject.transform! end it "generates a fake sha for the upload record" do upload = subject.transform! expect(upload.sha1).not_to eq(sha1) expect(upload.original_sha1).to eq(nil) expect(upload.filesize).to eq(object_size) end it "marks the stub as uploaded" do subject.transform! expect(external_upload_stub.reload.status).to eq(ExternalUploadStub.statuses[:uploaded]) end it "copies the stubbed upload on S3 to its new destination and deletes it" do upload = subject.transform! expect(WebMock).to have_requested( :put, "#{upload_base_url}/#{Discourse.store.get_path_for_upload(upload)}", ).with(headers: { 'X-Amz-Copy-Source' => "#{SiteSetting.s3_upload_bucket}/#{external_upload_stub.key}" }) expect(WebMock).to have_requested( :delete, "#{upload_base_url}/#{external_upload_stub.key}" ) end end context "when the upload type is backup" do let(:upload_base_url) { "https://#{SiteSetting.s3_backup_bucket}.s3.#{SiteSetting.s3_region}.amazonaws.com" } let(:object_size) { 200.megabytes } let(:object_file) { file_from_fixtures("backup_since_v1.6.tar.gz", "backups") } let!(:external_upload_stub) do Fabricate( :attachment_external_upload_stub, created_by: user, filesize: object_size, upload_type: "backup", original_filename: "backup_since_v1.6.tar.gz", folder_prefix: RailsMultisite::ConnectionManagement.current_db ) end before do stub_request(:head, "https://#{SiteSetting.s3_backup_bucket}.s3.#{SiteSetting.s3_region}.amazonaws.com/") # stub copy and delete object for backup, which copies the original filename to the root, # and also uses current_db in the bucket name always stub_request( :put, "#{upload_base_url}/#{RailsMultisite::ConnectionManagement.current_db}/backup_since_v1.6.tar.gz" ).to_return( status: 200, headers: { "ETag" => etag }, body: copy_object_result ) end it "does not try and download the file" do FileHelper.expects(:download).never subject.transform! end it "raises an error when backups are disabled" do SiteSetting.enable_backups = false expect { subject.transform! }.to raise_error(Discourse::InvalidAccess) end it "raises an error when backups are local, not s3" do SiteSetting.backup_location = BackupLocationSiteSetting::LOCAL expect { subject.transform! }.to raise_error(Discourse::InvalidAccess) end it "does not create an upload record" do expect { subject.transform! }.not_to change { Upload.count } end it "copies the stubbed upload on S3 to its new destination and deletes it" do upload = subject.transform! expect(WebMock).to have_requested( :put, "#{upload_base_url}/#{RailsMultisite::ConnectionManagement.current_db}/backup_since_v1.6.tar.gz", ).with(headers: { 'X-Amz-Copy-Source' => "#{SiteSetting.s3_backup_bucket}/#{external_upload_stub.key}" }) expect(WebMock).to have_requested( :delete, "#{upload_base_url}/#{external_upload_stub.key}" ) end end end def stub_head_object stub_request( :head, "#{upload_base_url}/#{external_upload_stub.key}" ).to_return( status: 200, headers: { ETag: etag, "Content-Length" => object_size, "Content-Type" => "image/png", }.merge(metadata_headers) ) end def stub_download_object_filehelper signed_url = Discourse.store.signed_url_for_path(external_upload_stub.key) uri = URI.parse(signed_url) signed_url = uri.to_s.gsub(uri.query, "") stub_request(:get, signed_url).with(query: hash_including({})).to_return( status: 200, body: object_file.read ) end def copy_object_result <<~BODY \n 2021-07-19T04:10:41.000Z "#{etag}" BODY end def stub_copy_object upload_pdf = Fabricate(:upload, sha1: "testbc60eb18e8f974cbfae8bb0f069c3a311024", original_filename: "test.pdf", extension: "pdf") upload_path = Discourse.store.get_path_for_upload(upload_pdf) upload_pdf.destroy! stub_request( :put, "#{upload_base_url}/#{upload_path}" ).to_return( status: 200, headers: { "ETag" => etag }, body: copy_object_result ) upload_png = Fabricate(:upload, sha1: "bc975735dfc6409c1c2aa5ebf2239949bcbdbd65", original_filename: "test.png", extension: "png") upload_path = Discourse.store.get_path_for_upload(upload_png) upload_png.destroy! stub_request( :put, "#{upload_base_url}/#{upload_path}" ).to_return( status: 200, headers: { "ETag" => etag }, body: copy_object_result ) end def stub_delete_object stub_request( :delete, "#{upload_base_url}/#{external_upload_stub.key}" ).to_return( status: 200 ) end end