Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DEV: Add API docs for uploads and API doc watcher (#15387) 

This commit adds API documentation for the new upload
endpoints related to direct + multipart external uploads.

Also included is a rake task which watches the files in
the spec/requests/api directory and calls a script file
(spec/regenerate_swagger_docs) whenever one changes. This
script runs rake rswag:specs:swaggerize and then copies
the openapi.yml file over to the discourse_api_docs repo
directory, and hits a script there to convert the YML to
JSON so the API docs are refreshed while the server is
still running. This makes the loop of making a doc change
and seeing it in the local server much faster.

The rake task is rake autospec:swagger
This commit is contained in:
Martin Brennan 2021-12-23 08:40:15 +10:00 committed by GitHub
parent 435562cc70
commit 19089f21d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 571 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lib/tasks/autospec.rake
View File

@ -22,3 +22,21 @@ task "autospec" => :environment do
Autospec::Manager.run(force_polling: force_polling, latency: latency, debug: debug)
end
desc "Regenerate swagger docs on API spec change"
task "autospec:swagger" => :environment do
require 'listen'
require 'open3'
puts "Listening to changes in spec/requests/api to regenerate Swagger docs."
listener = Listen.to("spec/requests/api") do |modified, added, removed|
puts "API doc file changed."
Open3.popen3("spec/regenerate_swagger_docs") do |stdin, stdout, stderr, wait_thr|
while line = stdout.gets
puts line
end
end
end
listener.start
sleep
end

5
spec/fabricators/external_upload_stub_fabricator.rb
View File

@ -22,3 +22,8 @@ Fabricator(:attachment_external_upload_stub, from: :external_upload_stub) do
filesize 1024
key { |attrs| FileStore::BaseStore.temporary_upload_path("file.pdf", folder_prefix: attrs[:folder_prefix] || "") }
end
Fabricator(:multipart_external_upload_stub, from: :external_upload_stub) do
multipart true
external_upload_identifier { "#{SecureRandom.hex(6)}._#{SecureRandom.hex(6)}_#{SecureRandom.hex(6)}.d.ghQ" }
end

10
spec/regenerate_swagger_docs Executable file
View File

@ -0,0 +1,10 @@
#!/bin/sh
if [[ -z "${DISCOURSE_REPO_BASE_DIRECTORY}" ]]; then
echo "Set DISCOURSE_REPO_BASE_DIRECTORY before running this script."
else
discourse_api_docs_dir="${DISCOURSE_REPO_BASE_DIRECTORY}/discourse_api_docs/"
RUBYOPT="W0" rake rswag:specs:swaggerize && cp openapi/openapi.yaml ${discourse_api_docs_dir}openapi.yml
(cd $discourse_api_docs_dir ; sh ${discourse_api_docs_dir}openapi_changed.sh)
echo "Swagger openapi.yml file copied to $discourse_api_docs_dir"
fi

13
spec/requests/api/schemas/json/upload_abort_multipart_request.json Normal file
View File

@ -0,0 +1,13 @@
{
"additionalProperties": false,
"properties": {
"external_upload_identifier": {
"type": "string",
"description": "The identifier of the multipart upload in the external storage provider. This is the multipart upload_id in AWS S3.",
"example": "84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ"
}
},
"required": [
"external_upload_identifier"
]
}

19
spec/requests/api/schemas/json/upload_batch_presign_multipart_parts_request.json Normal file
View File

@ -0,0 +1,19 @@
{
"additionalProperties": false,
"properties": {
"part_numbers": {
"type": "array",
"description": "The part numbers to generate the presigned URLs for, must be between 1 and 10000.",
"example": [1, 2, 3]
},
"unique_identifier": {
"type": "string",
"description": "The unique identifier returned in the original /create-multipart request.",
"example": "66e86218-80d9-4bda-b4d5-2b6def968705"
}
},
"required": [
"part_numbers",
"unique_identifier"
]
}

15
spec/requests/api/schemas/json/upload_batch_presign_multipart_parts_response.json Normal file
View File

@ -0,0 +1,15 @@
{
"additionalProperties": false,
"properties": {
"presigned_urls": {
"type": "object",
"description": "The presigned URLs for each part number, which has the part numbers as keys.",
"example": {
"1": "https://discourse-martin-uploads-test.s3.us-east-2.amazonaws.com/temp/uploads/default/123abc/123abc.jpg?partNumber=1&uploadId=123456abcd&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=test&X-Amz-Date=20211222T012336Z&X-Amz-Expires=600&X-Amz-SignedHeaders=host&X-Amz-Signature=abc123"
}
}
},
"required": [
"presigned_urls"
]
}

28
spec/requests/api/schemas/json/upload_complete_external_upload_request.json Normal file
View File

@ -0,0 +1,28 @@
{
"additionalProperties": false,
"properties": {
"unique_identifier": {
"type": "string",
"example": "66e86218-80d9-4bda-b4d5-2b6def968705",
"description": "The unique identifier returned in the original /generate-presigned-put request."
},
"for_private_message": {
"type": "string",
"example": "true",
"description": "Optionally set this to true if the upload is for a private message."
},
"for_site_setting": {
"type": "string",
"example": "true",
"description": "Optionally set this to true if the upload is for a site setting."
},
"pasted": {
"type": "string",
"example": "true",
"description": "Optionally set this to true if the upload was pasted into the upload area. This will convert PNG files to JPEG."
}
},
"required": [
"unique_identifier"
]
}

28
spec/requests/api/schemas/json/upload_complete_multipart_request.json Normal file
View File

@ -0,0 +1,28 @@
{
"additionalProperties": false,
"properties": {
"unique_identifier": {
"type": "string",
"example": "66e86218-80d9-4bda-b4d5-2b6def968705",
"description": "The unique identifier returned in the original /create-multipart request."
},
"parts": {
"type": "array",
"example": [
{
"part_number": 1,
"etag": "0c376dcfcc2606f4335bbc732de93344"
},
{
"part_number": 2,
"etag": "09ert8cfcc2606f4335bbc732de91122"
}
],
"description": "All of the part numbers and their corresponding ETags that have been uploaded must be provided."
}
},
"required": [
"unique_identifier",
"parts"
]
}

39
spec/requests/api/schemas/json/upload_create_multipart_request.json Normal file
View File

@ -0,0 +1,39 @@
{
"additionalProperties": false,
"properties": {
"upload_type": {
"type": "string",
"enum": [
"avatar",
"profile_background",
"card_background",
"custom_emoji",
"composer"
]
},
"file_name": {
"type": "string",
"example": "IMG_2021.jpeg"
},
"file_size": {
"type": "integer",
"description": "File size should be represented in bytes.",
"example": 4096
},
"metadata": {
"type": "object",
"additionalProperties": false,
"properties": {
"sha1-checksum": {
"type": "string",
"description": "The SHA1 checksum of the upload binary blob. Optionally be provided and serves as an additional security check when later processing the file in complete-external-upload endpoint."
}
}
}
},
"required": [
"upload_type",
"file_name",
"file_size"
]
}

25
spec/requests/api/schemas/json/upload_create_multipart_response.json Normal file
View File

@ -0,0 +1,25 @@
{
"additionalProperties": false,
"properties": {
"key": {
"type": "string",
"description": "The path of the temporary file on the external storage service.",
"example": "temp/site/uploads/default/12345/67890.jpg"
},
"external_upload_identifier": {
"type": "string",
"description": "The identifier of the multipart upload in the external storage provider. This is the multipart upload_id in AWS S3.",
"example": "84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ"
},
"unique_identifier": {
"type": "string",
"description": "A unique string that identifies the external upload. This must be stored and then sent in the /complete-multipart and /batch-presign-multipart-parts endpoints.",
"example": "66e86218-80d9-4bda-b4d5-2b6def968705"
}
},
"required": [
"external_upload_identifier",
"key",
"unique_identifier"
]
}

39
spec/requests/api/schemas/json/upload_generate_presigned_put_request.json Normal file
View File

@ -0,0 +1,39 @@
{
"additionalProperties": false,
"properties": {
"type": {
"type": "string",
"enum": [
"avatar",
"profile_background",
"card_background",
"custom_emoji",
"composer"
]
},
"file_name": {
"type": "string",
"example": "IMG_2021.jpeg"
},
"file_size": {
"type": "integer",
"description": "File size should be represented in bytes.",
"example": 4096
},
"metadata": {
"type": "object",
"additionalProperties": false,
"properties": {
"sha1-checksum": {
"type": "string",
"description": "The SHA1 checksum of the upload binary blob. Optionally be provided and serves as an additional security check when later processing the file in complete-external-upload endpoint."
}
}
}
},
"required": [
"type",
"file_name",
"file_size"
]
}

21
spec/requests/api/schemas/json/upload_generate_presigned_put_response.json Normal file
View File

@ -0,0 +1,21 @@
{
"additionalProperties": false,
"properties": {
"key": {
"type": "string",
"description": "The path of the temporary file on the external storage service.",
"example": "temp/site/uploads/default/12345/67890.jpg"
},
"url": {
"type": "string",
"description": "A presigned PUT URL which must be used to upload the file binary blob to.",
"example": "https://file-uploads.s3.us-west-2.amazonaws.com/temp/site/uploads/default/123/456.jpg?x-amz-acl=private&x-amz-meta-sha1-checksum=sha1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AAAAus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20211221T011246Z&X-Amz-Expires=600&X-Amz-SignedHeaders=host&X-Amz-Signature=12345678"
},
"unique_identifier": {
"type": "string",
"description": "A unique string that identifies the external upload. This must be stored and then sent in the /complete-external-upload endpoint to complete the direct upload.",
"example": "66e86218-80d9-4bda-b4d5-2b6def968705"
}
}
}

303
spec/requests/api/uploads_spec.rb
View File

@ -22,10 +22,10 @@ describe 'uploads' do
parameter name: :params, in: :body, schema: expected_request_schema
let(:params) { {
type: 'avatar',
user_id: admin.id,
synchronous: true,
file: logo
'type' => 'avatar',
'user_id' => admin.id,
'synchronous' => true,
'file' => logo
} }
produces 'application/json'
@ -40,4 +40,299 @@ describe 'uploads' do
end
end
describe "external and multipart uploads" do
before do
setup_s3
SiteSetting.enable_direct_s3_uploads = true
end
path '/uploads/generate-presigned-put.json' do
post 'Initiates a direct external upload' do
tags 'Uploads'
operationId 'generatePresignedPut'
consumes 'application/json'
description <<~HEREDOC
Direct external uploads bypass the usual method of creating uploads
via the POST /uploads route, and upload directly to an external provider,
which by default is S3. This route begins the process, and will return
a unique identifier for the external upload as well as a presigned URL
which is where the file binary blob should be uploaded to.
Once the upload is complete to the external service, you must call the
POST /complete-external-upload route using the unique identifier returned
by this route, which will create any required Upload record in the Discourse
database and also move file from its temporary location to the final
destination in the external storage service.
#{direct_uploads_disclaimer}
HEREDOC
expected_request_schema = load_spec_schema('upload_generate_presigned_put_request')
parameter name: :params, in: :body, schema: expected_request_schema
produces 'application/json'
response '200', 'external upload initialized' do
expected_response_schema = load_spec_schema('upload_generate_presigned_put_response')
schema(expected_response_schema)
let(:params) { {
'file_name' => "test.png",
'type' => "composer",
'file_size' => 4096,
'metadata' => {
'sha1-checksum' => "830869e4ed99128e4352aa72ff5b0ffc26fdc390"
}
} }
it_behaves_like "a JSON endpoint", 200 do
let(:expected_response_schema) { expected_response_schema }
let(:expected_request_schema) { expected_request_schema }
end
end
end
end
path '/uploads/complete-external-upload.json' do
post 'Completes a direct external upload' do
let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" }
let!(:external_stub) { Fabricate(:external_upload_stub, created_by: admin) }
let!(:upload) { Fabricate(:upload) }
before do
ExternalUploadManager.any_instance.stubs(:transform!).returns(upload)
ExternalUploadManager.any_instance.stubs(:destroy!)
external_stub.update(unique_identifier: unique_identifier)
end
tags 'Uploads'
operationId 'completeExternalUpload'
consumes 'application/json'
description <<~HEREDOC
Completes an external upload initialized with /get-presigned-put. The
file will be moved from its temporary location in external storage to
a final destination in the S3 bucket. An Upload record will also be
created in the database in most cases.
If a sha1-checksum was provided in the initial request it will also
be compared with the uploaded file in storage to make sure the same
file was uploaded. The file size will be compared for the same reason.
#{direct_uploads_disclaimer}
HEREDOC
expected_request_schema = load_spec_schema('upload_complete_external_upload_request')
parameter name: :params, in: :body, schema: expected_request_schema
produces 'application/json'
response '200', 'external upload initialized' do
expected_response_schema = load_spec_schema('upload_create_response')
schema(expected_response_schema)
let(:params) { {
'unique_identifier' => unique_identifier,
} }
it_behaves_like "a JSON endpoint", 200 do
let(:expected_response_schema) { expected_response_schema }
let(:expected_request_schema) { expected_request_schema }
end
end
end
end
path '/uploads/create-multipart.json' do
post 'Creates a multipart external upload' do
before do
ExternalUploadManager.stubs(:create_direct_multipart_upload).returns({
external_upload_identifier: "66e86218-80d9-4bda-b4d5-2b6def968705",
key: "temp/site/uploads/default/12345/67890.jpg",
unique_identifier: "84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ"
})
end
tags 'Uploads'
operationId 'createMultipartUpload'
consumes 'application/json'
description <<~HEREDOC
Creates a multipart upload in the external storage provider, storing
a temporary reference to the external upload similar to /get-presigned-put.
#{direct_uploads_disclaimer}
HEREDOC
expected_request_schema = load_spec_schema('upload_create_multipart_request')
parameter name: :params, in: :body, schema: expected_request_schema
produces 'application/json'
response '200', 'external upload initialized' do
expected_response_schema = load_spec_schema('upload_create_multipart_response')
schema(expected_response_schema)
let(:params) { {
'file_name' => "test.png",
'upload_type' => "composer",
'file_size' => 4096,
'metadata' => {
'sha1-checksum' => "830869e4ed99128e4352aa72ff5b0ffc26fdc390"
}
} }
it_behaves_like "a JSON endpoint", 200 do
let(:expected_response_schema) { expected_response_schema }
let(:expected_request_schema) { expected_request_schema }
end
end
end
end
path '/uploads/batch-presign-multipart-parts.json' do
post 'Generates batches of presigned URLs for multipart parts' do
let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" }
let!(:external_stub) { Fabricate(:multipart_external_upload_stub, created_by: admin) }
let!(:upload) { Fabricate(:upload) }
before do
stub_s3_store
external_stub.update(unique_identifier: unique_identifier)
end
tags 'Uploads'
operationId 'batchPresignMultipartParts'
consumes 'application/json'
description <<~HEREDOC
Multipart uploads are uploaded in chunks or parts to individual presigned
URLs, similar to the one genreated by /generate-presigned-put. The part
numbers provided must be between 1 and 10000. The total number of parts
will depend on the chunk size in bytes that you intend to use to upload
each chunk. For example a 12MB file may have 2 5MB chunks and a final
2MB chunk, for part numbers 1, 2, and 3.
This endpoint will return a presigned URL for each part number provided,
which you can then use to send PUT requests for the binary chunk corresponding
to that part. When the part is uploaded, the provider should return an
ETag for the part, and this should be stored along with the part number,
because this is needed to complete the multipart upload.
#{direct_uploads_disclaimer}
HEREDOC
expected_request_schema = load_spec_schema('upload_batch_presign_multipart_parts_request')
parameter name: :params, in: :body, schema: expected_request_schema
produces 'application/json'
response '200', 'external upload initialized' do
expected_response_schema = load_spec_schema('upload_batch_presign_multipart_parts_response')
schema(expected_response_schema)
let(:params) { {
'part_numbers' => [1, 2, 3],
'unique_identifier' => "66e86218-80d9-4bda-b4d5-2b6def968705"
} }
it_behaves_like "a JSON endpoint", 200 do
let(:expected_response_schema) { expected_response_schema }
let(:expected_request_schema) { expected_request_schema }
end
end
end
end
path '/uploads/abort-multipart.json' do
post 'Abort multipart upload' do
let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" }
let!(:external_stub) { Fabricate(:multipart_external_upload_stub, created_by: admin) }
let!(:upload) { Fabricate(:upload) }
before do
stub_s3_store
external_stub.update(
unique_identifier: unique_identifier,
external_upload_identifier: "84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ"
)
end
tags 'Uploads'
operationId 'abortMultipart'
consumes 'application/json'
description <<~HEREDOC
This endpoint aborts the multipart upload initiated with /create-multipart.
This should be used when cancelling the upload. It does not matter if parts
were already uploaded into the external storage provider.
#{direct_uploads_disclaimer}
HEREDOC
expected_request_schema = load_spec_schema('upload_abort_multipart_request')
parameter name: :params, in: :body, schema: expected_request_schema
produces 'application/json'
response '200', 'external upload initialized' do
expected_response_schema = load_spec_schema('success_ok_response')
schema(expected_response_schema)
let(:params) { {
'external_upload_identifier' => "84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ"
} }
it_behaves_like "a JSON endpoint", 200 do
let(:expected_response_schema) { expected_response_schema }
let(:expected_request_schema) { expected_request_schema }
end
end
end
end
path '/uploads/complete-multipart.json' do
post 'Complete multipart upload' do
let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" }
let!(:external_stub) { Fabricate(:multipart_external_upload_stub, created_by: admin) }
let!(:upload) { Fabricate(:upload) }
before do
ExternalUploadManager.any_instance.stubs(:transform!).returns(upload)
ExternalUploadManager.any_instance.stubs(:destroy!)
stub_s3_store
external_stub.update(unique_identifier: unique_identifier)
end
tags 'Uploads'
operationId 'completeMultipart'
consumes 'application/json'
description <<~HEREDOC
Completes the multipart upload in the external store, and copies the
file from its temporary location to its final location in the store.
All of the parts must have been uploaded to the external storage provider.
An Upload record will be completed in most cases once the file is copied
to its final location.
#{direct_uploads_disclaimer}
HEREDOC
expected_request_schema = load_spec_schema('upload_complete_multipart_request')
parameter name: :params, in: :body, schema: expected_request_schema
produces 'application/json'
response '200', 'external upload initialized' do
expected_response_schema = load_spec_schema('upload_create_response')
schema(expected_response_schema)
let(:params) { {
'unique_identifier' => unique_identifier,
'parts' => [
{
'part_number' => 1,
'etag' => '0c376dcfcc2606f4335bbc732de93344'
}
]
} }
it_behaves_like "a JSON endpoint", 200 do
let(:expected_response_schema) { expected_response_schema }
let(:expected_request_schema) { expected_request_schema }
end
end
end
end
end
end

12
spec/swagger_helper.rb
View File

@ -74,6 +74,18 @@ def api_docs_description
HEREDOC
end
def direct_uploads_disclaimer
<<~HEREDOC
You must have the correct permissions and CORS settings configured in your
external provider. We support AWS S3 as the default. See:
https://meta.discourse.org/t/-/210469#s3-multipart-direct-uploads-4.
An external file store must be set up and `enable_direct_s3_uploads` must
be set to true for this endpoint to function.
HEREDOC
end
RSpec.configure do |config|
# Specify a root folder where Swagger JSON files are generated
# NOTE: If you're using the rswag-api to serve API descriptions, you'll need