discourse/lib/external_upload_helpers.rb

# frozen_string_literal: true

# Extends controllers with the methods required to do direct
# external uploads.
module ExternalUploadHelpers
  extend ActiveSupport::Concern

  class ExternalUploadValidationError < StandardError; end

  PRESIGNED_PUT_RATE_LIMIT_PER_MINUTE = 10
  CREATE_MULTIPART_RATE_LIMIT_PER_MINUTE = 10
  COMPLETE_MULTIPART_RATE_LIMIT_PER_MINUTE = 10

  included do
    before_action :external_store_check, only: [
      :generate_presigned_put,
      :complete_external_upload,
      :create_multipart,
      :batch_presign_multipart_parts,
      :abort_multipart,
      :complete_multipart
    ]
    before_action :direct_s3_uploads_check, only: [
      :generate_presigned_put,
      :complete_external_upload,
      :create_multipart,
      :batch_presign_multipart_parts,
      :abort_multipart,
      :complete_multipart
    ]
    before_action :can_upload_external?, only: [:create_multipart, :generate_presigned_put]
  end

  def generate_presigned_put
    RateLimiter.new(
      current_user, "generate-presigned-put-upload-stub", ExternalUploadHelpers::PRESIGNED_PUT_RATE_LIMIT_PER_MINUTE, 1.minute
    ).performed!

    file_name = params.require(:file_name)
    file_size = params.require(:file_size).to_i
    type = params.require(:type)

    begin
      validate_before_create_direct_upload(
        file_name: file_name,
        file_size: file_size,
        upload_type: type
      )
    rescue ExternalUploadValidationError => err
      return render_json_error(err.message, status: 422)
    end

    external_upload_data = ExternalUploadManager.create_direct_upload(
      current_user: current_user,
      file_name: file_name,
      file_size: file_size,
      upload_type: type,
      metadata: parse_allowed_metadata(params[:metadata])
    )

    render json: external_upload_data
  end

  def complete_external_upload
    unique_identifier = params.require(:unique_identifier)
    external_upload_stub = ExternalUploadStub.find_by(
      unique_identifier: unique_identifier, created_by: current_user
    )
    return render_404 if external_upload_stub.blank?

    complete_external_upload_via_manager(external_upload_stub)
  end

  def create_multipart
    RateLimiter.new(
      current_user, "create-multipart-upload", ExternalUploadHelpers::CREATE_MULTIPART_RATE_LIMIT_PER_MINUTE, 1.minute
    ).performed!

    file_name = params.require(:file_name)
    file_size = params.require(:file_size).to_i
    upload_type = params.require(:upload_type)

    begin
      validate_before_create_multipart(
        file_name: file_name,
        file_size: file_size,
        upload_type: upload_type
      )
    rescue ExternalUploadValidationError => err
      return render_json_error(err.message, status: 422)
    end

    begin
      external_upload_data = create_direct_multipart_upload do
        ExternalUploadManager.create_direct_multipart_upload(
          current_user: current_user,
          file_name: file_name,
          file_size: file_size,
          upload_type: upload_type,
          metadata: parse_allowed_metadata(params[:metadata])
        )
      end
    rescue ExternalUploadHelpers::ExternalUploadValidationError => err
      return render_json_error(err.message, status: 422)
    end

    render json: external_upload_data
  end

  def batch_presign_multipart_parts
    part_numbers = params.require(:part_numbers)
    unique_identifier = params.require(:unique_identifier)

    ##
    # NOTE: This is configurable by hidden site setting because this really is heavily
    # dependent on upload speed. We request 5-10 URLs at a time with this endpoint; for
    # a 1.5GB upload with 5mb parts this could mean 60 requests to the server to get all
    # the part URLs. If the user's upload speed is super fast they may request all 60
    # batches in a minute, if it is slow they may request 5 batches in a minute.
    #
    # The other external upload endpoints are not hit as often, so they can stay as constant
    # values for now.
    RateLimiter.new(
      current_user, "batch-presign", SiteSetting.max_batch_presign_multipart_per_minute, 1.minute
    ).performed!

    part_numbers = part_numbers.map do |part_number|
      validate_part_number(part_number)
    end

    external_upload_stub = ExternalUploadStub.find_by(
      unique_identifier: unique_identifier, created_by: current_user
    )
    return render_404 if external_upload_stub.blank?

    if !multipart_upload_exists?(external_upload_stub)
      return render_404
    end

    store = multipart_store(external_upload_stub.upload_type)

    presigned_urls = {}
    part_numbers.each do |part_number|
      presigned_urls[part_number] = store.presign_multipart_part(
        upload_id: external_upload_stub.external_upload_identifier,
        key: external_upload_stub.key,
        part_number: part_number
      )
    end

    render json: { presigned_urls: presigned_urls }
  end

  def multipart_upload_exists?(external_upload_stub)
    store = multipart_store(external_upload_stub.upload_type)
    begin
      store.list_multipart_parts(
        upload_id: external_upload_stub.external_upload_identifier,
        key: external_upload_stub.key,
        max_parts: 1
      )
    rescue Aws::S3::Errors::NoSuchUpload => err
      debug_upload_error(err, I18n.t("upload.external_upload_not_found", additional_detail: "path: #{external_upload_stub.key}"))
      return false
    end
    true
  end

  def abort_multipart
    external_upload_identifier = params.require(:external_upload_identifier)
    external_upload_stub = ExternalUploadStub.find_by(
      external_upload_identifier: external_upload_identifier
    )

    # The stub could have already been deleted by an earlier error via
    # ExternalUploadManager, so we consider this a great success if the
    # stub is already gone.
    return render json: success_json if external_upload_stub.blank?

    return render_404 if external_upload_stub.created_by_id != current_user.id
    store = multipart_store(external_upload_stub.upload_type)

    begin
      store.abort_multipart(
        upload_id: external_upload_stub.external_upload_identifier,
        key: external_upload_stub.key
      )
    rescue Aws::S3::Errors::ServiceError => err
      return render_json_error(
        debug_upload_error(err, I18n.t("upload.abort_multipart_failure", additional_detail: "external upload stub id: #{external_upload_stub.id}")),
        status: 422
      )
    end

    external_upload_stub.destroy!

    render json: success_json
  end

  def complete_multipart
    unique_identifier = params.require(:unique_identifier)
    parts = params.require(:parts)

    RateLimiter.new(
      current_user, "complete-multipart-upload", ExternalUploadHelpers::COMPLETE_MULTIPART_RATE_LIMIT_PER_MINUTE, 1.minute
    ).performed!

    external_upload_stub = ExternalUploadStub.find_by(
      unique_identifier: unique_identifier, created_by: current_user
    )
    return render_404 if external_upload_stub.blank?

    if !multipart_upload_exists?(external_upload_stub)
      return render_404
    end

    store = multipart_store(external_upload_stub.upload_type)
    parts = parts.map do |part|
      part_number = part[:part_number]
      etag = part[:etag]
      part_number = validate_part_number(part_number)

      if etag.blank?
        raise Discourse::InvalidParameters.new("All parts must have an etag and a valid part number")
      end

      # this is done so it's an array of hashes rather than an array of
      # ActionController::Parameters
      { part_number: part_number, etag: etag }
    end.sort_by do |part|
      part[:part_number]
    end

    begin
      complete_response = store.complete_multipart(
        upload_id: external_upload_stub.external_upload_identifier,
        key: external_upload_stub.key,
        parts: parts
      )
    rescue Aws::S3::Errors::ServiceError => err
      return render_json_error(
        debug_upload_error(err, I18n.t("upload.complete_multipart_failure", additional_detail: "external upload stub id: #{external_upload_stub.id}")),
        status: 422
      )
    end

    complete_external_upload_via_manager(external_upload_stub)
  end

  private

  def complete_external_upload_via_manager(external_upload_stub)
    opts = {
      for_private_message: params[:for_private_message]&.to_s == "true",
      for_site_setting: params[:for_site_setting]&.to_s == "true",
      pasted: params[:pasted]&.to_s == "true",
    }

    external_upload_manager = ExternalUploadManager.new(external_upload_stub, opts)
    hijack do
      begin
        upload = external_upload_manager.transform!

        if upload.errors.empty?
          response_serialized = self.class.serialize_upload(upload)
          external_upload_stub.destroy!
          render json: response_serialized, status: 200
        else
          render_json_error(upload.errors.to_hash.values.flatten, status: 422)
        end
      rescue ExternalUploadManager::SizeMismatchError => err
        render_json_error(
          debug_upload_error(err, I18n.t("upload.size_mismatch_failure", additional_detail: err.message)),
          status: 422
        )
      rescue ExternalUploadManager::ChecksumMismatchError => err
        render_json_error(
          debug_upload_error(err, I18n.t("upload.checksum_mismatch_failure", additional_detail: err.message)),
          status: 422
        )
      rescue ExternalUploadManager::CannotPromoteError => err
        render_json_error(
          debug_upload_error(err, I18n.t("upload.cannot_promote_failure", additional_detail: err.message)),
          status: 422
        )
      rescue ExternalUploadManager::DownloadFailedError, Aws::S3::Errors::NotFound => err
        render_json_error(
          debug_upload_error(err, I18n.t("upload.download_failure", additional_detail: err.message)),
          status: 422
        )
      rescue => err
        Discourse.warn_exception(
          err, message: "Complete external upload failed unexpectedly for user #{current_user.id}"
        )

        render_json_error(I18n.t("upload.failed"), status: 422)
      end
    end
  end

  def validate_before_create_direct_upload(file_name:, file_size:, upload_type:)
    # noop, should be overridden
  end

  def validate_before_create_multipart(file_name:, file_size:, upload_type:)
    # noop, should be overridden
  end

  def validate_part_number(part_number)
    part_number = part_number.to_i
    if !part_number.between?(1, 10000)
      raise Discourse::InvalidParameters.new(
        "Each part number should be between 1 and 10000"
      )
    end
    part_number
  end

  def debug_upload_error(err, friendly_message)
    return if !SiteSetting.enable_upload_debug_mode
    Discourse.warn_exception(err, message: friendly_message)
    Rails.env.development? ? friendly_message : I18n.t("upload.failed")
  end

  def multipart_store(upload_type)
    ExternalUploadManager.store_for_upload_type(upload_type)
  end

  def external_store_check
    return render_404 if !Discourse.store.external?
  end

  def direct_s3_uploads_check
    return render_404 if !SiteSetting.enable_direct_s3_uploads
  end

  def can_upload_external?
    raise Discourse::InvalidAccess if !guardian.can_upload_external?
  end

  # don't want people posting arbitrary S3 metadata so we just take the
  # one we need. all of these will be converted to x-amz-meta- metadata
  # fields in S3 so it's best to use dashes in the names for consistency
  #
  # this metadata is baked into the presigned url and is not altered when
  # sending the PUT from the clientside to the presigned url
  def parse_allowed_metadata(metadata)
    return if metadata.blank?
    metadata.permit("sha1-checksum").to_h
  end

  def render_404
    raise Discourse::NotFound
  end
end