343 lines
13 KiB
Ruby
343 lines
13 KiB
Ruby
# frozen_string_literal: true
|
|
require "swagger_helper"
|
|
|
|
RSpec.describe "uploads" do
|
|
let(:admin) { Fabricate(:admin) }
|
|
let(:logo_file) { file_from_fixtures("logo.png") }
|
|
let(:logo) { Rack::Test::UploadedFile.new(logo_file) }
|
|
|
|
before do
|
|
Jobs.run_immediately!
|
|
sign_in(admin)
|
|
end
|
|
|
|
path "/uploads.json" do
|
|
post "Creates an upload" do
|
|
tags "Uploads"
|
|
operationId "createUpload"
|
|
consumes "multipart/form-data"
|
|
|
|
expected_request_schema = load_spec_schema("upload_create_request")
|
|
parameter name: :params, in: :body, schema: expected_request_schema
|
|
|
|
let(:params) do
|
|
{ "type" => "avatar", "user_id" => admin.id, "synchronous" => true, "file" => logo }
|
|
end
|
|
|
|
produces "application/json"
|
|
response "200", "file uploaded" do
|
|
expected_response_schema = load_spec_schema("upload_create_response")
|
|
schema(expected_response_schema)
|
|
|
|
# Skipping this test for now until https://github.com/rswag/rswag/issues/348
|
|
# is resolved. This still allows the docs to be generated for this endpoint though.
|
|
xit
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "external and multipart uploads" do
|
|
before do
|
|
setup_s3
|
|
SiteSetting.enable_direct_s3_uploads = true
|
|
end
|
|
|
|
path "/uploads/generate-presigned-put.json" do
|
|
post "Initiates a direct external upload" do
|
|
tags "Uploads"
|
|
operationId "generatePresignedPut"
|
|
consumes "application/json"
|
|
description <<~TEXT
|
|
Direct external uploads bypass the usual method of creating uploads
|
|
via the POST /uploads route, and upload directly to an external provider,
|
|
which by default is S3. This route begins the process, and will return
|
|
a unique identifier for the external upload as well as a presigned URL
|
|
which is where the file binary blob should be uploaded to.
|
|
|
|
Once the upload is complete to the external service, you must call the
|
|
POST /complete-external-upload route using the unique identifier returned
|
|
by this route, which will create any required Upload record in the Discourse
|
|
database and also move file from its temporary location to the final
|
|
destination in the external storage service.
|
|
|
|
#{direct_uploads_disclaimer}
|
|
TEXT
|
|
|
|
expected_request_schema = load_spec_schema("upload_generate_presigned_put_request")
|
|
parameter name: :params, in: :body, schema: expected_request_schema
|
|
|
|
produces "application/json"
|
|
response "200", "external upload initialized" do
|
|
expected_response_schema = load_spec_schema("upload_generate_presigned_put_response")
|
|
schema(expected_response_schema)
|
|
|
|
let(:params) do
|
|
{
|
|
"file_name" => "test.png",
|
|
"type" => "composer",
|
|
"file_size" => 4096,
|
|
"metadata" => {
|
|
"sha1-checksum" => "830869e4ed99128e4352aa72ff5b0ffc26fdc390",
|
|
},
|
|
}
|
|
end
|
|
|
|
it_behaves_like "a JSON endpoint", 200 do
|
|
let(:expected_response_schema) { expected_response_schema }
|
|
let(:expected_request_schema) { expected_request_schema }
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
path "/uploads/complete-external-upload.json" do
|
|
post "Completes a direct external upload" do
|
|
let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" }
|
|
let!(:external_stub) { Fabricate(:external_upload_stub, created_by: admin) }
|
|
let!(:upload) { Fabricate(:upload) }
|
|
|
|
before do
|
|
ExternalUploadManager.any_instance.stubs(:transform!).returns(upload)
|
|
ExternalUploadManager.any_instance.stubs(:destroy!)
|
|
external_stub.update(unique_identifier: unique_identifier)
|
|
end
|
|
|
|
tags "Uploads"
|
|
operationId "completeExternalUpload"
|
|
consumes "application/json"
|
|
description <<~TEXT
|
|
Completes an external upload initialized with /get-presigned-put. The
|
|
file will be moved from its temporary location in external storage to
|
|
a final destination in the S3 bucket. An Upload record will also be
|
|
created in the database in most cases.
|
|
|
|
If a sha1-checksum was provided in the initial request it will also
|
|
be compared with the uploaded file in storage to make sure the same
|
|
file was uploaded. The file size will be compared for the same reason.
|
|
|
|
#{direct_uploads_disclaimer}
|
|
TEXT
|
|
|
|
expected_request_schema = load_spec_schema("upload_complete_external_upload_request")
|
|
parameter name: :params, in: :body, schema: expected_request_schema
|
|
|
|
produces "application/json"
|
|
response "200", "external upload initialized" do
|
|
expected_response_schema = load_spec_schema("upload_create_response")
|
|
schema(expected_response_schema)
|
|
|
|
let(:params) { { "unique_identifier" => unique_identifier } }
|
|
|
|
it_behaves_like "a JSON endpoint", 200 do
|
|
let(:expected_response_schema) { expected_response_schema }
|
|
let(:expected_request_schema) { expected_request_schema }
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
path "/uploads/create-multipart.json" do
|
|
post "Creates a multipart external upload" do
|
|
before do
|
|
ExternalUploadManager.stubs(:create_direct_multipart_upload).returns(
|
|
{
|
|
external_upload_identifier: "66e86218-80d9-4bda-b4d5-2b6def968705",
|
|
key: "temp/site/uploads/default/12345/67890.jpg",
|
|
unique_identifier:
|
|
"84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ",
|
|
},
|
|
)
|
|
end
|
|
|
|
tags "Uploads"
|
|
operationId "createMultipartUpload"
|
|
consumes "application/json"
|
|
description <<~TEXT
|
|
Creates a multipart upload in the external storage provider, storing
|
|
a temporary reference to the external upload similar to /get-presigned-put.
|
|
|
|
#{direct_uploads_disclaimer}
|
|
TEXT
|
|
|
|
expected_request_schema = load_spec_schema("upload_create_multipart_request")
|
|
parameter name: :params, in: :body, schema: expected_request_schema
|
|
|
|
produces "application/json"
|
|
response "200", "external upload initialized" do
|
|
expected_response_schema = load_spec_schema("upload_create_multipart_response")
|
|
schema(expected_response_schema)
|
|
|
|
let(:params) do
|
|
{
|
|
"file_name" => "test.png",
|
|
"upload_type" => "composer",
|
|
"file_size" => 4096,
|
|
"metadata" => {
|
|
"sha1-checksum" => "830869e4ed99128e4352aa72ff5b0ffc26fdc390",
|
|
},
|
|
}
|
|
end
|
|
|
|
it_behaves_like "a JSON endpoint", 200 do
|
|
let(:expected_response_schema) { expected_response_schema }
|
|
let(:expected_request_schema) { expected_request_schema }
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
path "/uploads/batch-presign-multipart-parts.json" do
|
|
post "Generates batches of presigned URLs for multipart parts" do
|
|
let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" }
|
|
let!(:external_stub) { Fabricate(:multipart_external_upload_stub, created_by: admin) }
|
|
let!(:upload) { Fabricate(:upload) }
|
|
|
|
before do
|
|
stub_s3_store
|
|
external_stub.update(unique_identifier: unique_identifier)
|
|
end
|
|
|
|
tags "Uploads"
|
|
operationId "batchPresignMultipartParts"
|
|
consumes "application/json"
|
|
description <<~TEXT
|
|
Multipart uploads are uploaded in chunks or parts to individual presigned
|
|
URLs, similar to the one generated by /generate-presigned-put. The part
|
|
numbers provided must be between 1 and 10000. The total number of parts
|
|
will depend on the chunk size in bytes that you intend to use to upload
|
|
each chunk. For example a 12MB file may have 2 5MB chunks and a final
|
|
2MB chunk, for part numbers 1, 2, and 3.
|
|
|
|
This endpoint will return a presigned URL for each part number provided,
|
|
which you can then use to send PUT requests for the binary chunk corresponding
|
|
to that part. When the part is uploaded, the provider should return an
|
|
ETag for the part, and this should be stored along with the part number,
|
|
because this is needed to complete the multipart upload.
|
|
|
|
#{direct_uploads_disclaimer}
|
|
TEXT
|
|
|
|
expected_request_schema = load_spec_schema("upload_batch_presign_multipart_parts_request")
|
|
parameter name: :params, in: :body, schema: expected_request_schema
|
|
|
|
produces "application/json"
|
|
response "200", "external upload initialized" do
|
|
expected_response_schema =
|
|
load_spec_schema("upload_batch_presign_multipart_parts_response")
|
|
schema(expected_response_schema)
|
|
|
|
let(:params) do
|
|
{
|
|
"part_numbers" => [1, 2, 3],
|
|
"unique_identifier" => "66e86218-80d9-4bda-b4d5-2b6def968705",
|
|
}
|
|
end
|
|
|
|
it_behaves_like "a JSON endpoint", 200 do
|
|
let(:expected_response_schema) { expected_response_schema }
|
|
let(:expected_request_schema) { expected_request_schema }
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
path "/uploads/abort-multipart.json" do
|
|
post "Abort multipart upload" do
|
|
let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" }
|
|
let!(:external_stub) { Fabricate(:multipart_external_upload_stub, created_by: admin) }
|
|
let!(:upload) { Fabricate(:upload) }
|
|
|
|
before do
|
|
stub_s3_store
|
|
external_stub.update(
|
|
unique_identifier: unique_identifier,
|
|
external_upload_identifier:
|
|
"84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ",
|
|
)
|
|
end
|
|
|
|
tags "Uploads"
|
|
operationId "abortMultipart"
|
|
consumes "application/json"
|
|
description <<~TEXT
|
|
This endpoint aborts the multipart upload initiated with /create-multipart.
|
|
This should be used when cancelling the upload. It does not matter if parts
|
|
were already uploaded into the external storage provider.
|
|
|
|
#{direct_uploads_disclaimer}
|
|
TEXT
|
|
|
|
expected_request_schema = load_spec_schema("upload_abort_multipart_request")
|
|
parameter name: :params, in: :body, schema: expected_request_schema
|
|
|
|
produces "application/json"
|
|
response "200", "external upload initialized" do
|
|
expected_response_schema = load_spec_schema("success_ok_response")
|
|
schema(expected_response_schema)
|
|
|
|
let(:params) do
|
|
{
|
|
"external_upload_identifier" =>
|
|
"84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ",
|
|
}
|
|
end
|
|
|
|
it_behaves_like "a JSON endpoint", 200 do
|
|
let(:expected_response_schema) { expected_response_schema }
|
|
let(:expected_request_schema) { expected_request_schema }
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
path "/uploads/complete-multipart.json" do
|
|
post "Complete multipart upload" do
|
|
let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" }
|
|
let!(:external_stub) { Fabricate(:multipart_external_upload_stub, created_by: admin) }
|
|
let!(:upload) { Fabricate(:upload) }
|
|
|
|
before do
|
|
ExternalUploadManager.any_instance.stubs(:transform!).returns(upload)
|
|
ExternalUploadManager.any_instance.stubs(:destroy!)
|
|
stub_s3_store
|
|
external_stub.update(unique_identifier: unique_identifier)
|
|
end
|
|
|
|
tags "Uploads"
|
|
operationId "completeMultipart"
|
|
consumes "application/json"
|
|
description <<~TEXT
|
|
Completes the multipart upload in the external store, and copies the
|
|
file from its temporary location to its final location in the store.
|
|
All of the parts must have been uploaded to the external storage provider.
|
|
An Upload record will be completed in most cases once the file is copied
|
|
to its final location.
|
|
|
|
#{direct_uploads_disclaimer}
|
|
TEXT
|
|
|
|
expected_request_schema = load_spec_schema("upload_complete_multipart_request")
|
|
parameter name: :params, in: :body, schema: expected_request_schema
|
|
|
|
produces "application/json"
|
|
response "200", "external upload initialized" do
|
|
expected_response_schema = load_spec_schema("upload_create_response")
|
|
schema(expected_response_schema)
|
|
|
|
let(:params) do
|
|
{
|
|
"unique_identifier" => unique_identifier,
|
|
"parts" => [{ "part_number" => 1, "etag" => "0c376dcfcc2606f4335bbc732de93344" }],
|
|
}
|
|
end
|
|
|
|
it_behaves_like "a JSON endpoint", 200 do
|
|
let(:expected_response_schema) { expected_response_schema }
|
|
let(:expected_request_schema) { expected_request_schema }
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|