DEV: Limit list multipart parts to 1 (#14853)

We are only using list_multipart_parts right now in the
uploads controller for multipart uploads to check if the
upload exists; thus we don't need up to 1000 parts.

Also adding a note for future explorers that list_multipart_parts
only gets 1000 parts max, and adding params for max parts
and starting parts.
This commit is contained in:
Martin Brennan 2021-11-10 08:01:28 +10:00 committed by GitHub
parent 25ef395af8
commit 6a68bd4825
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 29 additions and 7 deletions

View File

@ -372,7 +372,9 @@ class UploadsController < ApplicationController
def multipart_upload_exists?(external_upload_stub)
begin
Discourse.store.list_multipart_parts(
upload_id: external_upload_stub.external_upload_identifier, key: external_upload_stub.key
upload_id: external_upload_stub.external_upload_identifier,
key: external_upload_stub.key,
max_parts: 1
)
rescue Aws::S3::Errors::NoSuchUpload => err
debug_upload_error(err, "upload.external_upload_not_found", { additional_detail: "path: #{external_upload_stub.key}" })

View File

@ -347,12 +347,32 @@ module FileStore
)
end
def list_multipart_parts(upload_id:, key:)
s3_helper.s3_client.list_parts(
# Important note from the S3 documentation:
#
# This request returns a default and maximum of 1000 parts.
# You can restrict the number of parts returned by specifying the
# max_parts argument. If your multipart upload consists of more than 1,000
# parts, the response returns an IsTruncated field with the value of true,
# and a NextPartNumberMarker element.
#
# In subsequent ListParts requests you can include the part_number_marker arg
# using the NextPartNumberMarker the field value from the previous response to
# get more parts.
#
# See https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#list_parts-instance_method
def list_multipart_parts(upload_id:, key:, max_parts: 1000, start_from_part_number: nil)
options = {
bucket: s3_bucket_name,
key: key,
upload_id: upload_id
)
upload_id: upload_id,
max_parts: max_parts
}
if start_from_part_number.present?
options[:part_number_marker] = start_from_part_number
end
s3_helper.s3_client.list_parts(options)
end
def complete_multipart(upload_id:, key:, parts:)

View File

@ -974,7 +974,7 @@ describe UploadsController do
<StorageClass>STANDARD</StorageClass>
</ListPartsResult>
BODY
stub_request(:get, "https://s3-upload-bucket.s3.us-west-1.amazonaws.com/#{external_upload_stub.key}?uploadId=#{mock_multipart_upload_id}").to_return({ status: 200, body: list_multipart_result })
stub_request(:get, "https://s3-upload-bucket.s3.us-west-1.amazonaws.com/#{external_upload_stub.key}?max-parts=1&uploadId=#{mock_multipart_upload_id}").to_return({ status: 200, body: list_multipart_result })
end
it "errors if the correct params are not provided" do
@ -1118,7 +1118,7 @@ describe UploadsController do
<StorageClass>STANDARD</StorageClass>
</ListPartsResult>
BODY
stub_request(:get, "#{upload_base_url}/#{external_upload_stub.key}?uploadId=#{mock_multipart_upload_id}").to_return({ status: 200, body: list_multipart_result })
stub_request(:get, "#{upload_base_url}/#{external_upload_stub.key}?max-parts=1&uploadId=#{mock_multipart_upload_id}").to_return({ status: 200, body: list_multipart_result })
end
it "errors if the correct params are not provided" do