discourse/spec/lib/file_store/s3_store_spec.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

489 lines
20 KiB
Ruby
Raw Normal View History

# frozen_string_literal: true
require 'file_store/s3_store'
require 'file_store/local_store'
RSpec.describe FileStore::S3Store do
let(:store) { FileStore::S3Store.new }
let(:s3_helper) { store.s3_helper }
let(:client) { Aws::S3::Client.new(stub_responses: true) }
let(:resource) { Aws::S3::Resource.new(client: client) }
let(:s3_bucket) { resource.bucket("s3-upload-bucket") }
let(:s3_object) { stub }
let(:upload_path) { Discourse.store.upload_path }
fab!(:optimized_image) { Fabricate(:optimized_image) }
let(:optimized_image_file) { file_from_fixtures("logo.png") }
let(:uploaded_file) { file_from_fixtures("logo.png") }
fab!(:upload) do
Fabricate(:upload, sha1: Digest::SHA1.hexdigest('secret image string'))
end
before do
setup_s3
SiteSetting.s3_region = 'us-west-1'
end
describe 'uploading to s3' do
let(:etag) { "etag" }
describe "#store_upload" do
it "returns an absolute schemaless url" do
2018-08-07 23:26:05 -04:00
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with(regexp_matches(%r{original/\d+X.*/#{upload.sha1}\.png})).returns(s3_object)
s3_object.expects(:put).with(
acl: "public-read",
cache_control: "max-age=31556952, public, immutable",
content_type: "image/png",
body: uploaded_file).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
expect(store.store_upload(uploaded_file, upload)).to match(
%r{//s3-upload-bucket\.s3\.dualstack\.us-west-1\.amazonaws\.com/original/\d+X.*/#{upload.sha1}\.png}
)
expect(upload.etag).to eq(etag)
end
describe "when s3_upload_bucket includes folders path" do
before do
s3_object.stubs(:put).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
SiteSetting.s3_upload_bucket = "s3-upload-bucket/discourse-uploads"
end
it "returns an absolute schemaless url" do
s3_helper.expects(:s3_bucket).returns(s3_bucket)
s3_bucket.expects(:object).with(regexp_matches(%r{discourse-uploads/original/\d+X.*/#{upload.sha1}\.png})).returns(s3_object)
expect(store.store_upload(uploaded_file, upload)).to match(
%r{//s3-upload-bucket\.s3\.dualstack\.us-west-1\.amazonaws\.com/discourse-uploads/original/\d+X.*/#{upload.sha1}\.png}
)
expect(upload.etag).to eq(etag)
end
end
describe "when secure uploads are enabled" do
it "saves secure attachment using private ACL" do
SiteSetting.prevent_anons_from_downloading_files = true
SiteSetting.authorized_extensions = "pdf|png|jpg|gif"
upload = Fabricate(:upload, original_filename: "small.pdf", extension: "pdf", secure: true)
s3_helper.expects(:s3_bucket).returns(s3_bucket)
s3_bucket.expects(:object).with(regexp_matches(%r{original/\d+X.*/#{upload.sha1}\.pdf})).returns(s3_object)
s3_object.expects(:put).with(
acl: "private",
cache_control: "max-age=31556952, public, immutable",
content_type: "application/pdf",
content_disposition: "attachment; filename=\"#{upload.original_filename}\"; filename*=UTF-8''#{upload.original_filename}",
body: uploaded_file).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
expect(store.store_upload(uploaded_file, upload)).to match(
%r{//s3-upload-bucket\.s3\.dualstack\.us-west-1\.amazonaws\.com/original/\d+X.*/#{upload.sha1}\.pdf}
)
end
it "saves image upload using public ACL" do
SiteSetting.prevent_anons_from_downloading_files = true
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with(regexp_matches(%r{original/\d+X.*/#{upload.sha1}\.png})).returns(s3_object).at_least_once
s3_object.expects(:put).with(
acl: "public-read",
cache_control: "max-age=31556952, public, immutable",
content_type: "image/png",
body: uploaded_file).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
expect(store.store_upload(uploaded_file, upload)).to match(
%r{//s3-upload-bucket\.s3\.dualstack\.us-west-1\.amazonaws\.com/original/\d+X.*/#{upload.sha1}\.png}
)
expect(store.url_for(upload)).to eq(upload.url)
end
end
end
describe "#store_optimized_image" do
before do
s3_object.stubs(:put).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
end
it "returns an absolute schemaless url" do
s3_helper.expects(:s3_bucket).returns(s3_bucket)
path = %r{optimized/\d+X.*/#{optimized_image.upload.sha1}_#{OptimizedImage::VERSION}_100x200\.png}
s3_bucket.expects(:object).with(regexp_matches(path)).returns(s3_object)
expect(store.store_optimized_image(optimized_image_file, optimized_image)).to match(
%r{//s3-upload-bucket\.s3\.dualstack\.us-west-1\.amazonaws\.com/#{path}}
)
expect(optimized_image.etag).to eq(etag)
end
describe "when s3_upload_bucket includes folders path" do
before do
SiteSetting.s3_upload_bucket = "s3-upload-bucket/discourse-uploads"
end
it "returns an absolute schemaless url" do
s3_helper.expects(:s3_bucket).returns(s3_bucket)
path = %r{discourse-uploads/optimized/\d+X.*/#{optimized_image.upload.sha1}_#{OptimizedImage::VERSION}_100x200\.png}
s3_bucket.expects(:object).with(regexp_matches(path)).returns(s3_object)
expect(store.store_optimized_image(optimized_image_file, optimized_image)).to match(
%r{//s3-upload-bucket\.s3\.dualstack\.us-west-1\.amazonaws\.com/#{path}}
)
expect(optimized_image.etag).to eq(etag)
end
end
end
describe "#move_existing_stored_upload" do
let(:uploaded_file) { file_from_fixtures(original_filename) }
let(:upload_sha1) { Digest::SHA1.hexdigest(File.read(uploaded_file)) }
let(:original_filename) { "smallest.png" }
let(:s3_client) { Aws::S3::Client.new(stub_responses: true) }
let(:s3_helper) { S3Helper.new(SiteSetting.s3_upload_bucket, '', client: s3_client) }
let(:store) { FileStore::S3Store.new(s3_helper) }
let(:upload_opts) do
{
acl: "public-read",
cache_control: "max-age=31556952, public, immutable",
content_type: "image/png",
apply_metadata_to_destination: true
}
end
let(:external_upload_stub) { Fabricate(:image_external_upload_stub) }
let(:existing_external_upload_key) { external_upload_stub.key }
before do
SiteSetting.authorized_extensions = "pdf|png"
end
it "does not provide a content_disposition for images" do
s3_helper.expects(:copy).with(external_upload_stub.key, kind_of(String), options: upload_opts).returns(["path", "etag"])
s3_helper.expects(:delete_object).with(external_upload_stub.key)
upload = Fabricate(:upload, extension: "png", sha1: upload_sha1, original_filename: original_filename)
FEATURE: Direct S3 multipart uploads for backups (#14736) This PR introduces a new `enable_experimental_backup_uploads` site setting (default false and hidden), which when enabled alongside `enable_direct_s3_uploads` will allow for direct S3 multipart uploads of backup .tar.gz files. To make multipart external uploads work with both the S3BackupStore and the S3Store, I've had to move several methods out of S3Store and into S3Helper, including: * presigned_url * create_multipart * abort_multipart * complete_multipart * presign_multipart_part * list_multipart_parts Then, S3Store and S3BackupStore either delegate directly to S3Helper or have their own special methods to call S3Helper for these methods. FileStore.temporary_upload_path has also removed its dependence on upload_path, and can now be used interchangeably between the stores. A similar change was made in the frontend as well, moving the multipart related JS code out of ComposerUppyUpload and into a mixin of its own, so it can also be used by UppyUploadMixin. Some changes to ExternalUploadManager had to be made here as well. The backup direct uploads do not need an Upload record made for them in the database, so they can be moved to their final S3 resting place when completing the multipart upload. This changeset is not perfect; it introduces some special cases in UploadController to handle backups that was previously in BackupController, because UploadController is where the multipart routes are located. A subsequent pull request will pull these routes into a module or some other sharing pattern, along with hooks, so the backup controller and the upload controller (and any future controllers that may need them) can include these routes in a nicer way.
2021-11-10 17:25:31 -05:00
store.move_existing_stored_upload(
existing_external_upload_key: external_upload_stub.key,
upload: upload,
content_type: "image/png"
)
end
context "when the file is a PDF" do
let(:external_upload_stub) { Fabricate(:attachment_external_upload_stub, original_filename: original_filename) }
let(:original_filename) { "small.pdf" }
let(:uploaded_file) { file_from_fixtures("small.pdf", "pdf") }
it "adds an attachment content-disposition with the original filename" do
disp_opts = { content_disposition: "attachment; filename=\"#{original_filename}\"; filename*=UTF-8''#{original_filename}", content_type: "application/pdf" }
s3_helper.expects(:copy).with(external_upload_stub.key, kind_of(String), options: upload_opts.merge(disp_opts)).returns(["path", "etag"])
upload = Fabricate(:upload, extension: "png", sha1: upload_sha1, original_filename: original_filename)
FEATURE: Direct S3 multipart uploads for backups (#14736) This PR introduces a new `enable_experimental_backup_uploads` site setting (default false and hidden), which when enabled alongside `enable_direct_s3_uploads` will allow for direct S3 multipart uploads of backup .tar.gz files. To make multipart external uploads work with both the S3BackupStore and the S3Store, I've had to move several methods out of S3Store and into S3Helper, including: * presigned_url * create_multipart * abort_multipart * complete_multipart * presign_multipart_part * list_multipart_parts Then, S3Store and S3BackupStore either delegate directly to S3Helper or have their own special methods to call S3Helper for these methods. FileStore.temporary_upload_path has also removed its dependence on upload_path, and can now be used interchangeably between the stores. A similar change was made in the frontend as well, moving the multipart related JS code out of ComposerUppyUpload and into a mixin of its own, so it can also be used by UppyUploadMixin. Some changes to ExternalUploadManager had to be made here as well. The backup direct uploads do not need an Upload record made for them in the database, so they can be moved to their final S3 resting place when completing the multipart upload. This changeset is not perfect; it introduces some special cases in UploadController to handle backups that was previously in BackupController, because UploadController is where the multipart routes are located. A subsequent pull request will pull these routes into a module or some other sharing pattern, along with hooks, so the backup controller and the upload controller (and any future controllers that may need them) can include these routes in a nicer way.
2021-11-10 17:25:31 -05:00
store.move_existing_stored_upload(
existing_external_upload_key: external_upload_stub.key,
upload: upload,
content_type: "application/pdf"
)
end
end
end
end
describe 'copying files in S3' do
2018-08-07 23:26:05 -04:00
describe '#copy_file' do
it "copies the from in S3 with the right paths" do
upload.update!(
url: "//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/original/1X/#{upload.sha1}.png"
2018-08-07 23:26:05 -04:00
)
source = "#{upload_path}/#{Discourse.store.get_path_for_upload(upload)}"
destination = source.sub('.png', '.jpg')
bucket = prepare_fake_s3(source, upload)
2018-08-07 23:26:05 -04:00
expect(bucket.find_object(source)).to be_present
expect(bucket.find_object(destination)).to be_nil
2018-08-07 23:26:05 -04:00
store.copy_file(upload.url, source, destination)
expect(bucket.find_object(source)).to be_present
expect(bucket.find_object(destination)).to be_present
2018-08-07 23:26:05 -04:00
end
end
end
describe 'removal from s3' do
describe "#remove_upload" do
it "removes the file from s3 with the right paths" do
upload_key = Discourse.store.get_path_for_upload(upload)
tombstone_key = "tombstone/#{upload_key}"
bucket = prepare_fake_s3(upload_key, upload)
upload.update!(url: "//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/#{upload_key}")
expect(bucket.find_object(upload_key)).to be_present
expect(bucket.find_object(tombstone_key)).to be_nil
store.remove_upload(upload)
expect(bucket.find_object(upload_key)).to be_nil
expect(bucket.find_object(tombstone_key)).to be_present
end
describe "when s3_upload_bucket includes folders path" do
before do
SiteSetting.s3_upload_bucket = "s3-upload-bucket/discourse-uploads"
end
it "removes the file from s3 with the right paths" do
upload_key = "discourse-uploads/#{Discourse.store.get_path_for_upload(upload)}"
tombstone_key = "discourse-uploads/tombstone/#{Discourse.store.get_path_for_upload(upload)}"
bucket = prepare_fake_s3(upload_key, upload)
upload.update!(url: "//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/#{upload_key}")
expect(bucket.find_object(upload_key)).to be_present
expect(bucket.find_object(tombstone_key)).to be_nil
store.remove_upload(upload)
expect(bucket.find_object(upload_key)).to be_nil
expect(bucket.find_object(tombstone_key)).to be_present
end
end
2013-11-05 13:04:47 -05:00
end
2013-08-13 16:08:29 -04:00
describe "#remove_optimized_image" do
let(:optimized_key) { FileStore::BaseStore.new.get_path_for_optimized_image(optimized_image) }
let(:tombstone_key) { "tombstone/#{optimized_key}" }
let(:upload) { optimized_image.upload }
let(:upload_key) { Discourse.store.get_path_for_upload(upload) }
before do
optimized_image.update!(url: "//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/#{optimized_key}")
upload.update!(url: "//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/#{upload_key}")
end
it "removes the optimized image from s3 with the right paths" do
bucket = prepare_fake_s3(upload_key, upload)
store_fake_s3_object(optimized_key, optimized_image)
expect(bucket.find_object(upload_key)).to be_present
expect(bucket.find_object(optimized_key)).to be_present
expect(bucket.find_object(tombstone_key)).to be_nil
store.remove_optimized_image(optimized_image)
expect(bucket.find_object(upload_key)).to be_present
expect(bucket.find_object(optimized_key)).to be_nil
expect(bucket.find_object(tombstone_key)).to be_present
end
describe "when s3_upload_bucket includes folders path" do
before do
SiteSetting.s3_upload_bucket = "s3-upload-bucket/discourse-uploads"
end
let(:image_path) { FileStore::BaseStore.new.get_path_for_optimized_image(optimized_image) }
let(:optimized_key) { "discourse-uploads/#{image_path}" }
let(:tombstone_key) { "discourse-uploads/tombstone/#{image_path}" }
let(:upload_key) { "discourse-uploads/#{Discourse.store.get_path_for_upload(upload)}" }
it "removes the file from s3 with the right paths" do
bucket = prepare_fake_s3(upload_key, upload)
store_fake_s3_object(optimized_key, optimized_image)
expect(bucket.find_object(upload_key)).to be_present
expect(bucket.find_object(optimized_key)).to be_present
expect(bucket.find_object(tombstone_key)).to be_nil
store.remove_optimized_image(optimized_image)
expect(bucket.find_object(upload_key)).to be_present
expect(bucket.find_object(optimized_key)).to be_nil
expect(bucket.find_object(tombstone_key)).to be_present
end
end
end
2013-08-13 16:08:29 -04:00
end
2013-11-05 13:04:47 -05:00
describe ".has_been_uploaded?" do
it "doesn't crash for invalid URLs" do
expect(store.has_been_uploaded?("https://site.discourse.com/#bad#6")).to eq(false)
end
it "doesn't crash if URL contains non-ascii characters" do
expect(store.has_been_uploaded?("//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/漢1337.png")).to eq(true)
expect(store.has_been_uploaded?("//s3-upload-bucket.s3.amazonaws.com/漢1337.png")).to eq(false)
end
it "identifies S3 uploads" do
expect(store.has_been_uploaded?("//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/1337.png")).to eq(true)
end
it "does not match other s3 urls" do
expect(store.has_been_uploaded?("//s3-upload-bucket.s3.amazonaws.com/1337.png")).to eq(false)
expect(store.has_been_uploaded?("//s3-upload-bucket.s3-us-west-1.amazonaws.com/1337.png")).to eq(false)
expect(store.has_been_uploaded?("//s3.amazonaws.com/s3-upload-bucket/1337.png")).to eq(false)
expect(store.has_been_uploaded?("//s4_upload_bucket.s3.amazonaws.com/1337.png")).to eq(false)
2013-11-05 13:04:47 -05:00
end
end
describe ".absolute_base_url" do
it "returns a lowercase schemaless absolute url" do
expect(store.absolute_base_url).to eq("//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com")
end
it "uses the proper endpoint" do
SiteSetting.s3_region = "us-east-1"
expect(FileStore::S3Store.new(s3_helper).absolute_base_url).to eq("//s3-upload-bucket.s3.dualstack.us-east-1.amazonaws.com")
SiteSetting.s3_region = "us-west-2"
expect(FileStore::S3Store.new(s3_helper).absolute_base_url).to eq("//s3-upload-bucket.s3.dualstack.us-west-2.amazonaws.com")
2016-07-14 10:56:09 -04:00
SiteSetting.s3_region = "cn-north-1"
expect(FileStore::S3Store.new(s3_helper).absolute_base_url).to eq("//s3-upload-bucket.s3.cn-north-1.amazonaws.com.cn")
2016-07-14 10:56:09 -04:00
SiteSetting.s3_region = "cn-northwest-1"
expect(FileStore::S3Store.new(s3_helper).absolute_base_url).to eq("//s3-upload-bucket.s3.cn-northwest-1.amazonaws.com.cn")
2013-11-05 13:04:47 -05:00
end
end
it "is external" do
2015-01-09 11:34:37 -05:00
expect(store.external?).to eq(true)
expect(store.internal?).to eq(false)
2013-11-05 13:04:47 -05:00
end
describe ".purge_tombstone" do
it "updates tombstone lifecycle" do
s3_helper.expects(:update_tombstone_lifecycle)
store.purge_tombstone(1.day)
end
end
describe ".path_for" do
def assert_path(path, expected)
upload = Upload.new(url: path)
path = store.path_for(upload)
expected = FileStore::LocalStore.new.path_for(upload) if expected
expect(path).to eq(expected)
end
it "correctly falls back to local" do
assert_path("/hello", "/hello")
assert_path("//hello", nil)
assert_path("http://hello", nil)
assert_path("https://hello", nil)
end
end
describe 'update ACL' do
before do
SiteSetting.authorized_extensions = "pdf|png"
end
describe ".update_upload_ACL" do
let(:upload) { Fabricate(:upload, original_filename: "small.pdf", extension: "pdf") }
it "sets acl to public by default" do
s3_helper.expects(:s3_bucket).returns(s3_bucket)
s3_bucket.expects(:object).with(regexp_matches(%r{original/\d+X.*/#{upload.sha1}\.pdf})).returns(s3_object)
s3_object.expects(:acl).returns(s3_object)
s3_object.expects(:put).with(acl: "public-read").returns(s3_object)
expect(store.update_upload_ACL(upload)).to be_truthy
end
it "sets acl to private when upload is marked secure" do
upload.update!(secure: true)
s3_helper.expects(:s3_bucket).returns(s3_bucket)
s3_bucket.expects(:object).with(regexp_matches(%r{original/\d+X.*/#{upload.sha1}\.pdf})).returns(s3_object)
s3_object.expects(:acl).returns(s3_object)
s3_object.expects(:put).with(acl: "private").returns(s3_object)
expect(store.update_upload_ACL(upload)).to be_truthy
end
end
end
describe '.cdn_url' do
it 'supports subfolder' do
SiteSetting.s3_upload_bucket = 's3-upload-bucket/livechat'
SiteSetting.s3_cdn_url = 'https://rainbow.com'
# none of this should matter at all
# subfolder should not leak into uploads
set_subfolder "/community"
url = "//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/livechat/original/gif.png"
expect(store.cdn_url(url)).to eq("https://rainbow.com/original/gif.png")
end
end
describe ".download_url" do
it "returns correct short URL with dl=1 param" do
expect(store.download_url(upload)).to eq("#{upload.short_path}?dl=1")
end
end
describe ".url_for" do
it "returns signed URL with content disposition when requesting to download image" do
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with(regexp_matches(%r{original/\d+X.*/#{upload.sha1}\.png})).returns(s3_object)
opts = {
expires_in: SiteSetting.s3_presigned_get_url_expires_after_seconds,
response_content_disposition: %Q|attachment; filename="#{upload.original_filename}"; filename*=UTF-8''#{upload.original_filename}|
}
s3_object.expects(:presigned_url).with(:get, opts)
expect(store.url_for(upload, force_download: true)).not_to eq(upload.url)
end
end
describe ".signed_url_for_path" do
it "returns signed URL for a given path" do
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("special/optimized/file.png").returns(s3_object)
opts = {
expires_in: SiteSetting.s3_presigned_get_url_expires_after_seconds
}
s3_object.expects(:presigned_url).with(:get, opts)
expect(store.signed_url_for_path("special/optimized/file.png")).not_to eq(upload.url)
end
it "does not prefix the s3_bucket_folder_path onto temporary upload prefixed keys" do
SiteSetting.s3_upload_bucket = "s3-upload-bucket/folder_path"
uri = URI.parse(store.signed_url_for_path("#{FileStore::BaseStore::TEMPORARY_UPLOAD_PREFIX}folder_path/uploads/default/blah/def.xyz"))
expect(uri.path).to eq(
"/#{FileStore::BaseStore::TEMPORARY_UPLOAD_PREFIX}folder_path/uploads/default/blah/def.xyz"
)
uri = URI.parse(store.signed_url_for_path("uploads/default/blah/def.xyz"))
expect(uri.path).to eq(
"/folder_path/uploads/default/blah/def.xyz"
)
end
end
FEATURE: Initial implementation of direct S3 uploads with uppy and stubs (#13787) This adds a few different things to allow for direct S3 uploads using uppy. **These changes are still not the default.** There are hidden `enable_experimental_image_uploader` and `enable_direct_s3_uploads` settings that must be turned on for any of this code to be used, and even if they are turned on only the User Card Background for the user profile actually uses uppy-image-uploader. A new `ExternalUploadStub` model and database table is introduced in this pull request. This is used to keep track of uploads that are uploaded to a temporary location in S3 with the direct to S3 code, and they are eventually deleted a) when the direct upload is completed and b) after a certain time period of not being used. ### Starting a direct S3 upload When an S3 direct upload is initiated with uppy, we first request a presigned PUT URL from the new `generate-presigned-put` endpoint in `UploadsController`. This generates an S3 key in the `temp` folder inside the correct bucket path, along with any metadata from the clientside (e.g. the SHA1 checksum described below). This will also create an `ExternalUploadStub` and store the details of the temp object key and the file being uploaded. Once the clientside has this URL, uppy will upload the file direct to S3 using the presigned URL. Once the upload is complete we go to the next stage. ### Completing a direct S3 upload Once the upload to S3 is done we call the new `complete-external-upload` route with the unique identifier of the `ExternalUploadStub` created earlier. Only the user who made the stub can complete the external upload. One of two paths is followed via the `ExternalUploadManager`. 1. If the object in S3 is too large (currently 100mb defined by `ExternalUploadManager::DOWNLOAD_LIMIT`) we do not download and generate the SHA1 for that file. Instead we create the `Upload` record via `UploadCreator` and simply copy it to its final destination on S3 then delete the initial temp file. Several modifications to `UploadCreator` have been made to accommodate this. 2. If the object in S3 is small enough, we download it. When the temporary S3 file is downloaded, we compare the SHA1 checksum generated by the browser with the actual SHA1 checksum of the file generated by ruby. The browser SHA1 checksum is stored on the object in S3 with metadata, and is generated via the `UppyChecksum` plugin. Keep in mind that some browsers will not generate this due to compatibility or other issues. We then follow the normal `UploadCreator` path with one exception. To cut down on having to re-upload the file again, if there are no changes (such as resizing etc) to the file in `UploadCreator` we follow the same copy + delete temp path that we do for files that are too large. 3. Finally we return the serialized upload record back to the client There are several errors that could happen that are handled by `UploadsController` as well. Also in this PR is some refactoring of `displayErrorForUpload` to handle both uppy and jquery file uploader errors.
2021-07-27 18:42:25 -04:00
def prepare_fake_s3(upload_key, upload)
@fake_s3 = FakeS3.create
@fake_s3_bucket = @fake_s3.bucket(SiteSetting.s3_upload_bucket)
store_fake_s3_object(upload_key, upload)
@fake_s3_bucket
end
def store_fake_s3_object(upload_key, upload)
@fake_s3_bucket.put_object(
key: upload_key,
size: upload.filesize,
last_modified: upload.created_at
FEATURE: Initial implementation of direct S3 uploads with uppy and stubs (#13787) This adds a few different things to allow for direct S3 uploads using uppy. **These changes are still not the default.** There are hidden `enable_experimental_image_uploader` and `enable_direct_s3_uploads` settings that must be turned on for any of this code to be used, and even if they are turned on only the User Card Background for the user profile actually uses uppy-image-uploader. A new `ExternalUploadStub` model and database table is introduced in this pull request. This is used to keep track of uploads that are uploaded to a temporary location in S3 with the direct to S3 code, and they are eventually deleted a) when the direct upload is completed and b) after a certain time period of not being used. ### Starting a direct S3 upload When an S3 direct upload is initiated with uppy, we first request a presigned PUT URL from the new `generate-presigned-put` endpoint in `UploadsController`. This generates an S3 key in the `temp` folder inside the correct bucket path, along with any metadata from the clientside (e.g. the SHA1 checksum described below). This will also create an `ExternalUploadStub` and store the details of the temp object key and the file being uploaded. Once the clientside has this URL, uppy will upload the file direct to S3 using the presigned URL. Once the upload is complete we go to the next stage. ### Completing a direct S3 upload Once the upload to S3 is done we call the new `complete-external-upload` route with the unique identifier of the `ExternalUploadStub` created earlier. Only the user who made the stub can complete the external upload. One of two paths is followed via the `ExternalUploadManager`. 1. If the object in S3 is too large (currently 100mb defined by `ExternalUploadManager::DOWNLOAD_LIMIT`) we do not download and generate the SHA1 for that file. Instead we create the `Upload` record via `UploadCreator` and simply copy it to its final destination on S3 then delete the initial temp file. Several modifications to `UploadCreator` have been made to accommodate this. 2. If the object in S3 is small enough, we download it. When the temporary S3 file is downloaded, we compare the SHA1 checksum generated by the browser with the actual SHA1 checksum of the file generated by ruby. The browser SHA1 checksum is stored on the object in S3 with metadata, and is generated via the `UppyChecksum` plugin. Keep in mind that some browsers will not generate this due to compatibility or other issues. We then follow the normal `UploadCreator` path with one exception. To cut down on having to re-upload the file again, if there are no changes (such as resizing etc) to the file in `UploadCreator` we follow the same copy + delete temp path that we do for files that are too large. 3. Finally we return the serialized upload record back to the client There are several errors that could happen that are handled by `UploadsController` as well. Also in this PR is some refactoring of `displayErrorForUpload` to handle both uppy and jquery file uploader errors.
2021-07-27 18:42:25 -04:00
)
end
end