Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FEATURE: Add support for secure media (#7888) 

This PR introduces a new secure media setting. When enabled, it prevent unathorized access to media uploads (files of type image, video and audio). When the `login_required` setting is enabled, then all media uploads will be protected from unauthorized (anonymous) access. When `login_required`is disabled, only media in private messages will be protected from unauthorized access. 

A few notes: 

- the `prevent_anons_from_downloading_files` setting no longer applies to audio and video uploads
- the `secure_media` setting can only be enabled if S3 uploads are already enabled and configured
- upload records have a new column, `secure`, which is a boolean `true/false` of the upload's secure status
- when creating a public post with an upload that has already been uploaded and is marked as secure, the post creator will raise an error
- when enabling or disabling the setting on a site with existing uploads, the rake task `uploads:ensure_correct_acl` should be used to update all uploads' secure status and their ACL on S3
This commit is contained in:
Penar Musaraj 2019-11-17 20:25:42 -05:00 committed by Martin Brennan
parent 56b19ba740
commit 102909edb3
40 changed files with 1157 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/assets/javascripts/discourse/lib/uploads.js.es6
View File

@ -242,7 +242,6 @@ export function getUploadMarkdown(upload) {
upload.thumbnail_height
}](${upload.short_url || upload.url})`;
} else if (
!Discourse.SiteSettings.prevent_anons_from_downloading_files &&
/\.(mov|mp4|webm|ogv|mp3|ogg|wav|m4a)$/i.test(upload.original_filename)
) {
return uploadLocation(upload.url);

13
app/controllers/uploads_controller.rb
View File

@ -5,7 +5,7 @@ require "mini_mime"
class UploadsController < ApplicationController
requires_login except: [:show, :show_short]
skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required, only: [:show, :show_short]
skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required, only: [:show, :show_short, :show_secure]
protect_from_forgery except: :show
def create
@ -110,6 +110,17 @@ class UploadsController < ApplicationController
end
end
def show_secure
# do not serve uploads requested via XHR to prevent XSS
return xhr_not_allowed if request.xhr?
if SiteSetting.secure_media?
redirect_to Discourse.store.signed_url_for_path("#{params[:path]}.#{params[:extension]}")
else
render_404
end
end
def metadata
params.require(:url)
upload = Upload.get_from_url(params[:url])

4
app/jobs/regular/update_private_uploads_acl.rb
View File

@ -7,8 +7,8 @@ module Jobs
return if !SiteSetting.enable_s3_uploads
Upload.find_each do |upload|
if !FileHelper.is_supported_image?(upload.original_filename)
Discourse.store.update_upload_ACL(upload)
if !FileHelper.is_supported_media?(upload.original_filename)
upload.update_secure_status
end
end
end

15
app/mailers/user_notifications.rb
View File

@ -349,13 +349,26 @@ class UserNotifications < ActionMailer::Base
end
def email_post_markdown(post, add_posted_by = false)
result = +"#{post.raw}\n\n"
result = +"#{post.with_secure_media? ? strip_secure_urls(post.raw) : post.raw}\n\n"
if add_posted_by
result << "#{I18n.t('user_notifications.posted_by', username: post.username, post_date: post.created_at.strftime("%m/%d/%Y"))}\n\n"
end
result
end
def strip_secure_urls(raw)
urls = Set.new
raw.scan(URI.regexp(%w{http https})) { urls << $& }
urls.each do |url|
if (url.start_with?(Discourse.store.s3_upload_host) && FileHelper.is_supported_media?(url))
raw = raw.sub(url, "<p class='secure-media-notice'>#{I18n.t("emails.secure_media_placeholder")}</p>")
end
end
raw
end
def self.get_context_posts(post, topic_user, user)
if (user.user_option.email_previous_replies == UserOption.previous_replies_type[:never]) ||
SiteSetting.private_email?

28
app/models/post.rb
View File

@ -300,6 +300,15 @@ class Post < ActiveRecord::Base
options[:user_id] = post_user.id if post_user
options[:omit_nofollow] = true if omit_nofollow?
if self.with_secure_media?
each_upload_url do |url|
uri = URI.parse(url)
if FileHelper.is_supported_media?(File.basename(uri.path))
raw = raw.sub(Discourse.store.s3_upload_host, "#{Discourse.base_url}/secure-media-uploads")
end
end
end
cooked = post_analyzer.cook(raw, options)
new_cooked = Plugin::Filter.apply(:after_post_cook, self, cooked)
@ -492,6 +501,11 @@ class Post < ActiveRecord::Base
ReviewableFlaggedPost.pending.find_by(target: self)
end
def with_secure_media?
return false unless SiteSetting.secure_media?
topic&.private_message? || SiteSetting.login_required?
end
def hide!(post_action_type_id, reason = nil)
return if hidden?
@ -882,6 +896,13 @@ class Post < ActiveRecord::Base
end
upload_ids |= Upload.where(id: downloaded_images.values).pluck(:id)
disallowed_uploads = []
if SiteSetting.secure_media? && !topic&.private_message?
disallowed_uploads = Upload.where(id: upload_ids, secure: true).pluck(:original_filename)
end
return disallowed_uploads if disallowed_uploads.count > 0
values = upload_ids.map! { |upload_id| "(#{self.id},#{upload_id})" }.join(",")
PostUpload.transaction do
@ -893,6 +914,12 @@ class Post < ActiveRecord::Base
end
end
def update_uploads_secure_status
if Discourse.store.external?
self.uploads.each { |upload| upload.update_secure_status }
end
end
def downloaded_images
JSON.parse(self.custom_fields[Post::DOWNLOADED_IMAGES].presence || "{}")
rescue JSON::ParserError
@ -909,6 +936,7 @@ class Post < ActiveRecord::Base
]
fragments ||= Nokogiri::HTML::fragment(self.cooked)
links = fragments.css("a/@href", "img/@src").map do |media|
src = media.value
next if src.blank?

10
app/models/topic_converter.rb
View File

@ -30,9 +30,9 @@ class TopicConverter
)
update_user_stats
update_post_uploads_secure_status
Jobs.enqueue(:topic_action_converter, topic_id: @topic.id)
Jobs.enqueue(:delete_inaccessible_notifications, topic_id: @topic.id)
watch_topic(topic)
end
@topic
@ -49,6 +49,7 @@ class TopicConverter
)
add_allowed_users
update_post_uploads_secure_status
Jobs.enqueue(:topic_action_converter, topic_id: @topic.id)
Jobs.enqueue(:delete_inaccessible_notifications, topic_id: @topic.id)
@ -97,4 +98,11 @@ class TopicConverter
end
end
def update_post_uploads_secure_status
@topic.posts.each do |post|
next if post.uploads.empty?
post.update_uploads_secure_status
post.rebake!
end
end
end

2
app/models/topic_link.rb
View File

@ -176,7 +176,7 @@ class TopicLink < ActiveRecord::Base
if upload = Upload.get_from_url(url)
internal = Discourse.store.internal?
# Store the same URL that will be used in the cooked version of the post
url = UrlHelper.cook_url(upload.url)
url = UrlHelper.cook_url(upload.url, secure: upload.secure?)
elsif route = Discourse.route_for(parsed)
internal = true

34
app/models/upload.rb
View File

@ -140,11 +140,6 @@ class Upload < ActiveRecord::Base
!(url =~ /^(https?:)?\/\//)
end
def private?
return false if self.for_theme || self.for_site_setting
SiteSetting.prevent_anons_from_downloading_files && !FileHelper.is_supported_image?(self.original_filename)
end
def fix_dimensions!
return if !FileHelper.is_supported_image?("image.#{extension}")
@ -235,6 +230,34 @@ class Upload < ActiveRecord::Base
self.posts.where("cooked LIKE '%/_optimized/%'").find_each(&:rebake!)
end
def update_secure_status
return false if self.for_theme || self.for_site_setting
mark_secure = should_be_secure?
self.update_column("secure", mark_secure)
Discourse.store.update_upload_ACL(self) if Discourse.store.external?
end
def should_be_secure?
mark_secure = false
if FileHelper.is_supported_media?(self.original_filename)
if SiteSetting.secure_media?
mark_secure = true if SiteSetting.login_required?
unless SiteSetting.login_required?
# first post associated with upload determines secure status
# i.e. an already public upload will stay public even if added to a new PM
first_post_with_upload = self.posts.order(sort_order: :asc).first
mark_secure = first_post_with_upload ? first_post_with_upload.with_secure_media? : false
end
else
mark_secure = false
end
else
mark_secure = SiteSetting.prevent_anons_from_downloading_files?
end
mark_secure
end
def self.migrate_to_new_scheme(limit: nil)
problems = []
@ -385,6 +408,7 @@ end
# thumbnail_width :integer
# thumbnail_height :integer
# etag :string
# secure :boolean default(FALSE), not null
#
# Indexes
#

5
config/locales/server.en.yml
View File

@ -140,6 +140,7 @@ en:
unsubscribe_not_allowed: "Happens when unsubscribing via email is not allowed for this user."
email_not_allowed: "Happens when the email address is not on the whitelist or is on the blacklist."
unrecognized_error: "Unrecognized Error"
secure_media_placeholder: "Redacted: this site has secure media enabled, visit the topic to see the attached image/audio/video."
errors: &errors
format: ! "%{attribute} %{message}"
@ -204,6 +205,7 @@ en:
enable_s3_uploads_is_required: "You cannot enable inventory to S3 unless you've enabled the S3 uploads."
s3_backup_requires_s3_settings: "You cannot use S3 as backup location unless you've provided the '%{setting_name}'."
s3_bucket_reused: "You cannot use the same bucket for 's3_upload_bucket' and 's3_backup_bucket'. Choose a different bucket or use a different path for each bucket."
secure_media_requirements: "S3 uploads must be enabled before enabling secure media."
second_factor_cannot_be_enforced_with_disabled_local_login: "You cannot enforce 2FA if local logins are disabled."
local_login_cannot_be_disabled_if_second_factor_enforced: "You cannot disable local login if 2FA is enforced. Disable enforced 2FA before disabling local logins."
conflicting_google_user_id: 'The Google Account ID for this account has changed; staff intervention is required for security reasons. Please contact staff and point them to <br><a href="https://meta.discourse.org/t/76575">https://meta.discourse.org/t/76575</a>'
@ -333,6 +335,7 @@ en:
max_pm_recepients: "Sorry, you can send a message to maximum %{recipients_limit} recipients."
pm_reached_recipients_limit: "Sorry, you can't have more than %{recipients_limit} recipients in a message."
removed_direct_reply_full_quotes: "Automatically removed quote of whole previous post."
secure_upload_not_allowed_in_public_topic: "Sorry, the following secure upload(s) cannot be used in a public topic: %{upload_filenames}."
just_posted_that: "is too similar to what you recently posted"
invalid_characters: "contains invalid characters"
@ -2017,7 +2020,7 @@ en:
bootstrap_mode_min_users: "Minimum number of users required to disable bootstrap mode (set to 0 to disable)"
prevent_anons_from_downloading_files: "Prevent anonymous users from downloading attachments. WARNING: this will prevent any non-image site assets posted as attachments from working."
secure_media: 'Limits access to media uploads (images, video, audio). If "login required" is enabled, only logged-in users can access media uploads. Otherwise, access will be limited only for media uploads in private messages. Note: S3 uploads must be enabled prior to enabling this setting.'
slug_generation_method: "Choose a slug generation method. 'encoded' will generate percent encoding string. 'none' will disable slug at all."
enable_emoji: "Enable emoji"

1
config/routes.rb
View File

@ -502,6 +502,7 @@ Discourse::Application.routes.draw do
get "uploads/:site/original/:tree:sha(.:extension)" => "uploads#show", constraints: { site: /\w+/, tree: /([a-z0-9]+\/)+/i, sha: /\h{40}/, extension: /[a-z0-9\.]+/i }
# used to download attachments (old route)
get "uploads/:site/:id/:sha" => "uploads#show", constraints: { site: /\w+/, id: /\d+/, sha: /\h{16}/, format: /.*/ }
get "secure-media-uploads/*path(.:extension)" => "uploads#show_secure", constraints: { extension: /[a-z0-9\.]+/i }
get "posts" => "posts#latest", id: "latest_posts", constraints: { format: /(json|rss)/ }
get "private-posts" => "posts#latest", id: "private_posts", constraints: { format: /(json|rss)/ }

3
config/site_settings.yml
View File

@ -1127,6 +1127,9 @@ files:
prevent_anons_from_downloading_files:
default: false
client: true
secure_media:
default: false
client: true
enable_s3_uploads:
default: false
client: true

26
db/migrate/20190716173854_add_secure_to_uploads.rb Normal file
View File

@ -0,0 +1,26 @@
# frozen_string_literal: true
class AddSecureToUploads < ActiveRecord::Migration[5.2]
def up
add_column :uploads, :secure, :boolean, default: false, null: false
prevent_anons_from_downloading_files = \
DB.query_single("SELECT value FROM site_settings WHERE name = 'prevent_anons_from_downloading_files'").first == 't'
if prevent_anons_from_downloading_files
execute(
<<-SQL
UPDATE uploads SET secure = 't' WHERE id IN (
SELECT DISTINCT(uploads.id) FROM uploads
INNER JOIN post_uploads ON post_uploads.upload_id = uploads.id
WHERE LOWER(original_filename) NOT SIMILAR TO '%\.(jpg|jpeg|png|gif|svg|ico)'
)
SQL
)
end
end
def down
remove_column :uploads, :secure
end
end

12
lib/cooked_post_processor.rb
View File

@ -281,6 +281,10 @@ class CookedPostProcessor
absolute_url = url
absolute_url = Discourse.base_url_no_prefix + absolute_url if absolute_url =~ /^\/[^\/]/
if url&.start_with?("/secure-media-uploads/")
absolute_url = Discourse.store.signed_url_for_path(url.sub("/secure-media-uploads/", ""))
end
return unless absolute_url
# FastImage fails when there's no scheme
@ -400,14 +404,14 @@ class CookedPostProcessor
resized_h = (h * ratio).to_i
if !cropped && upload.width && resized_w > upload.width
cooked_url = UrlHelper.cook_url(upload.url)
cooked_url = UrlHelper.cook_url(upload.url, secure: upload.secure?)
srcset << ", #{cooked_url} #{ratio.to_s.sub(/\.0$/, "")}x"
elsif t = upload.thumbnail(resized_w, resized_h)
cooked_url = UrlHelper.cook_url(t.url)
cooked_url = UrlHelper.cook_url(t.url, secure: upload.secure?)
srcset << ", #{cooked_url} #{ratio.to_s.sub(/\.0$/, "")}x"
end
img["srcset"] = "#{UrlHelper.cook_url(img["src"])}#{srcset}" if srcset.present?
img["srcset"] = "#{UrlHelper.cook_url(img["src"], secure: upload.secure?)}#{srcset}" if srcset.present?
end
else
img["src"] = upload.url
@ -595,7 +599,7 @@ class CookedPostProcessor
%w{src data-small-upload}.each do |selector|
@doc.css("img[#{selector}]").each do |img|
img[selector] = UrlHelper.cook_url(img[selector].to_s)
img[selector] = UrlHelper.cook_url(img[selector].to_s, secure: @post.with_secure_media?)
end
end
end

19
lib/email/styles.rb
View File

@ -198,6 +198,7 @@ module Email
style('code', 'background-color: #f1f1ff; padding: 2px 5px;')
style('pre code', 'display: block; background-color: #f1f1ff; padding: 5px;')
style('.featured-topic a', "text-decoration: none; font-weight: bold; color: #{SiteSetting.email_link_color}; line-height:1.5em;")
style('.secure-image-notice', 'font-style: italic; background-color: #f1f1ff; padding: 5px;')
style('.summary-email', "-moz-box-sizing:border-box;-ms-text-size-adjust:100%;-webkit-box-sizing:border-box;-webkit-text-size-adjust:100%;box-sizing:border-box;color:#0a0a0a;font-family:Helvetica,Arial,sans-serif;font-size:14px;font-weight:400;line-height:1.3;margin:0;min-width:100%;padding:0;width:100%")
style('.previous-discussion', 'font-size: 17px; color: #444; margin-bottom:10px;')
@ -237,6 +238,7 @@ module Email
def to_html
strip_classes_and_ids
replace_relative_urls
replace_secure_media_urls
@fragment.to_html
end
@ -284,6 +286,23 @@ module Email
end
end
def replace_secure_media_urls
@fragment.css('[href]').each do |a|
if a['href'][/secure-media-uploads/]
a.add_next_sibling "<p class='secure-media-notice'>#{I18n.t("emails.secure_media_placeholder")}</p>"
a.remove
end
end
@fragment.search('img').each do |img|
next unless img['src']
if img['src'][/secure-media-uploads/]
img.add_next_sibling "<p class='secure-media-notice'>#{I18n.t("emails.secure_media_placeholder")}</p>"
img.remove
end
end
end
def correct_first_body_margin
@fragment.css('div.body p').each do |element|
element['style'] = "margin-top:0; border: 0;"

16
lib/file_helper.rb
View File

@ -17,6 +17,10 @@ class FileHelper
filename =~ supported_images_regexp
end
def self.is_supported_media?(filename)
filename =~ supported_media_regexp
end
class FakeIO
attr_accessor :status
end
@ -132,8 +136,20 @@ class FileHelper
@@supported_images ||= Set.new %w{jpg jpeg png gif svg ico}
end
def self.supported_audio
@@supported_audio ||= Set.new %w{mp3 ogg wav m4a}
end
def self.supported_video
@@supported_video ||= Set.new %w{mov mp4 webm ogv}
end
def self.supported_images_regexp
@@supported_images_regexp ||= /\.(#{supported_images.to_a.join("|")})$/i
end
def self.supported_media_regexp
media = supported_images | supported_audio | supported_video
@@supported_media_regexp ||= /\.(#{media.to_a.join("|")})$/i
end
end

10
lib/file_store/base_store.rb
View File

@ -54,6 +54,10 @@ module FileStore
not_implemented
end
def s3_upload_host
not_implemented
end
def external?
not_implemented
end
@ -77,7 +81,11 @@ module FileStore
if !file
max_file_size_kb = [SiteSetting.max_image_size_kb, SiteSetting.max_attachment_size_kb].max.kilobytes
url = Discourse.store.cdn_url(upload.url)
url = upload.secure? ?
Discourse.store.signed_url_for_path(upload.url) :
Discourse.store.cdn_url(upload.url)
url = SiteSetting.scheme + ":" + url if url =~ /^\/\//
file = FileHelper.download(
url,

69
lib/file_store/s3_store.rb
View File

@ -21,13 +21,13 @@ module FileStore
def store_upload(file, upload, content_type = nil)
path = get_path_for_upload(upload)
url, upload.etag = store_file(file, path, filename: upload.original_filename, content_type: content_type, cache_locally: true, private: upload.private?)
url, upload.etag = store_file(file, path, filename: upload.original_filename, content_type: content_type, cache_locally: true, private_acl: upload.secure?)
url
end
def store_optimized_image(file, optimized_image, content_type = nil)
def store_optimized_image(file, optimized_image, content_type = nil, secure: false)
path = get_path_for_optimized_image(optimized_image)
url, optimized_image.etag = store_file(file, path, content_type: content_type)
url, optimized_image.etag = store_file(file, path, content_type: content_type, private_acl: secure)
url
end
@ -42,12 +42,12 @@ module FileStore
# cache file locally when needed
cache_file(file, File.basename(path)) if opts[:cache_locally]
options = {
acl: opts[:private] ? "private" : "public-read",
acl: opts[:private_acl] ? "private" : "public-read",
cache_control: 'max-age=31556952, public, immutable',
content_type: opts[:content_type].presence || MiniMime.lookup_by_filename(filename)&.content_type
}
# add a "content disposition" header for "attachments"
options[:content_disposition] = "attachment; filename=\"#{filename}\"" unless FileHelper.is_supported_image?(filename)
options[:content_disposition] = "attachment; filename=\"#{filename}\"" unless FileHelper.is_supported_media?(filename)
path.prepend(File.join(upload_path, "/")) if Rails.configuration.multisite
@ -88,6 +88,10 @@ module FileStore
@absolute_base_url ||= SiteSetting.Upload.absolute_base_url
end
def s3_upload_host
SiteSetting.Upload.s3_cdn_url.present? ? SiteSetting.Upload.s3_cdn_url : "https:#{absolute_base_url}"
end
def external?
true
end
@ -111,22 +115,9 @@ module FileStore
end
def url_for(upload, force_download: false)
if upload.private? || force_download
opts = { expires_in: S3Helper::DOWNLOAD_URL_EXPIRES_AFTER_SECONDS }
if force_download
opts[:response_content_disposition] = ActionDispatch::Http::ContentDisposition.format(
disposition: "attachment", filename: upload.original_filename
)
end
obj = @s3_helper.object(get_upload_key(upload))
url = obj.presigned_url(:get, opts)
else
url = upload.url
end
url
upload.secure? || force_download ?
presigned_url(get_upload_key(upload), force_download: force_download, filename: upload.original_filename) :
upload.url
end
def cdn_url(url)
@ -136,6 +127,11 @@ module FileStore
url.sub(File.join("#{schema}#{absolute_base_url}", folder), File.join(SiteSetting.Upload.s3_cdn_url, "/"))
end
def signed_url_for_path(path)
key = path.sub(absolute_base_url + "/", "")
presigned_url(key)
end
def cache_avatar(avatar, user_id)
source = avatar.url.sub(absolute_base_url + "/", "")
destination = avatar_template(avatar, user_id).sub(absolute_base_url + "/", "")
@ -163,14 +159,15 @@ module FileStore
end
def update_upload_ACL(upload)
private_uploads = SiteSetting.prevent_anons_from_downloading_files
key = get_upload_key(upload)
update_ACL(key, upload.secure?)
begin
@s3_helper.object(key).acl.put(acl: private_uploads ? "private" : "public-read")
rescue Aws::S3::Errors::NoSuchKey
Rails.logger.warn("Could not update ACL on upload with key: '#{key}'. Upload is missing.")
upload.optimized_images.find_each do |optimized_image|
optimized_image_key = get_path_for_optimized_image(optimized_image)
update_ACL(optimized_image_key, upload.secure?)
end
true
end
def download_file(upload, destination_path)
@ -179,6 +176,18 @@ module FileStore
private
def presigned_url(url, force_download: false, filename: false)
opts = { expires_in: S3Helper::DOWNLOAD_URL_EXPIRES_AFTER_SECONDS }
if force_download && filename
opts[:response_content_disposition] = ActionDispatch::Http::ContentDisposition.format(
disposition: "attachment", filename: filename
)
end
obj = @s3_helper.object(url)
obj.presigned_url(:get, opts)
end
def get_upload_key(upload)
if Rails.configuration.multisite
File.join(upload_path, "/", get_path_for_upload(upload))
@ -187,6 +196,14 @@ module FileStore
end
end
def update_ACL(key, secure)
begin
@s3_helper.object(key).acl.put(acl: secure ? "private" : "public-read")
rescue Aws::S3::Errors::NoSuchKey
Rails.logger.warn("Could not update ACL on upload with key: '#{key}'. Upload is missing.")
end
end
def list_missing(model, prefix)
connection = ActiveRecord::Base.connection.raw_connection
connection.exec('CREATE TEMP TABLE verified_ids(val integer PRIMARY KEY)')

10
lib/oneboxer.rb
View File

@ -178,7 +178,15 @@ module Oneboxer
def self.local_upload_html(url)
case File.extname(URI(url).path || "")
when VIDEO_REGEX
"<video width='100%' height='100%' controls><source src='#{url}'><a href='#{url}'>#{url}</a></video>"
<<~HTML
<div class="onebox video-onebox">
<video width="100%" height="100%" controls="">
<source src='#{url}'>
<a href='#{url}'>#{url}</a>
</source>
</video>
</div>
HTML
when AUDIO_REGEX
"<audio controls><source src='#{url}'><a href='#{url}'>#{url}</a></audio>"
end

11
lib/post_creator.rb
View File

@ -177,6 +177,7 @@ class PostCreator
update_user_counts
create_embedded_topic
link_post_uploads
update_uploads_secure_status
ensure_in_allowed_users if guardian.is_staff?
unarchive_message
@post.advance_draft_sequence unless @opts[:import_mode]
@ -366,7 +367,15 @@ class PostCreator
end
def link_post_uploads
@post.link_post_uploads
disallowed_uploads = @post.link_post_uploads
if disallowed_uploads.is_a? Array
@post.errors.add(:base, I18n.t('secure_upload_not_allowed_in_public_topic', upload_filenames: disallowed_uploads.join(", ")))
rollback_from_errors!(@post)
end
end
def update_uploads_secure_status
@post.update_uploads_secure_status
end
def handle_spam

10
lib/pretty_text.rb
View File

@ -381,9 +381,19 @@ module PrettyText
end
end
def self.strip_secure_media(doc)
doc.css("a[href]").each do |a|
if a["href"].include?("/secure-media-uploads/") && FileHelper.is_supported_media?(a["href"])
target = %w(video audio).include?(a&.parent&.parent&.name) ? a.parent.parent : a
target.replace "<p class='secure-media-notice'>#{I18n.t("emails.secure_media_placeholder")}</p>"
end
end
end
def self.format_for_email(html, post = nil)
doc = Nokogiri::HTML.fragment(html)
DiscourseEvent.trigger(:reduce_cooked, doc, post)
strip_secure_media(doc) if post&.with_secure_media?
strip_image_wrapping(doc)
convert_vimeo_iframes(doc)
make_all_links_absolute(doc)

12
lib/pretty_text/helpers.rb
View File

@ -64,13 +64,15 @@ module PrettyText
reverse_map[value] << key
end
Upload.where(sha1: map.values).pluck(:sha1, :url, :extension).each do |row|
sha1, url, extension = row
Upload.where(sha1: map.values).pluck(:sha1, :url, :extension, :original_filename, :secure).each do |row|
sha1, url, extension, original_filename, secure = row
if short_urls = reverse_map[sha1]
secure_media = FileHelper.is_supported_media?(original_filename) && SiteSetting.secure_media? && secure
short_urls.each do |short_url|
result[short_url] = {
url: Discourse.store.cdn_url(url),
url: secure_media ? secure_media_url(url) : Discourse.store.cdn_url(url),
short_path: Upload.short_path(sha1: sha1, extension: extension),
base62_sha1: Upload.base62_sha1(sha1)
}
@ -82,6 +84,10 @@ module PrettyText
result
end
def secure_media_url(url)
url.sub(SiteSetting.Upload.absolute_base_url, "/secure-media-uploads")
end
def get_topic_info(topic_id)
return unless topic_id.is_a?(Integer)
# TODO this only handles public topics, secured one do not get this

11
lib/rake_helpers.rb Normal file
View File

@ -0,0 +1,11 @@
# frozen_string_literal: true
class RakeHelpers
def self.print_status_with_label(label, current, max)
print "\r\033[K%s%9d / %d (%5.1f%%)" % [label, current, max, ((current.to_f / max.to_f) * 100).round(1)]
end
def self.print_status(current, max)
print "\r\033[K%9d / %d (%5.1f%%)" % [current, max, ((current.to_f / max.to_f) * 100).round(1)]
end
end

4
lib/site_settings/validations.rb
View File

@ -121,6 +121,10 @@ module SiteSettings::Validations
validate_error :s3_upload_bucket_is_required if new_val == "t" && SiteSetting.s3_upload_bucket.blank?
end
def validate_secure_media(new_val)
validate_error :secure_media_requirements if new_val == "t" && !SiteSetting.enable_s3_uploads?
end
def validate_enable_s3_inventory(new_val)
validate_error :enable_s3_uploads_is_required if new_val == "t" && !SiteSetting.Upload.enable_s3_uploads
end

16
lib/tasks/topics.rake
View File

@ -1,8 +1,6 @@
# frozen_string_literal: true
def print_status_with_label(label, current, max)
print "\r%s%9d / %d (%5.1f%%)" % [label, current, max, ((current.to_f / max.to_f) * 100).round(1)]
end
require_dependency "rake_helpers"
def close_old_topics(category)
topics = Topic.where(closed: false, category_id: category.id)
@ -23,7 +21,7 @@ def close_old_topics(category)
topics.find_each do |topic|
topic.update_status("closed", true, Discourse.system_user)
print_status_with_label(" closing old topics: ", topics_closed += 1, total)
RakeHelpers.print_status_with_label(" closing old topics: ", topics_closed += 1, total)
end
end
@ -49,7 +47,7 @@ def apply_auto_close(category)
topics.find_each do |topic|
topic.inherit_auto_close_from_category
print_status_with_label(" applying auto-close to topics: ", topics_closed += 1, total)
RakeHelpers.print_status_with_label(" applying auto-close to topics: ", topics_closed += 1, total)
end
end
@ -77,7 +75,7 @@ task "topics:watch_all_replied_topics" => :environment do
t.topic_users.where(posted: true).find_each do |tp|
tp.update!(notification_level: TopicUser.notification_levels[:watching], notifications_reason_id: TopicUser.notification_reasons[:created_post])
end
print_status(count += 1, total)
RakeHelpers.print_status(count += 1, total)
end
puts "", "Done"
@ -96,12 +94,8 @@ task "topics:update_fancy_titles" => :environment do
Topic.find_each do |topic|
topic.fancy_title
print_status(count += 1, total)
RakeHelpers.print_status(count += 1, total)
end
puts "", "Done"
end
def print_status(current, max)
print "\r%9d / %d (%5.1f%%)" % [current, max, ((current.to_f / max.to_f) * 100).round(1)]
end

106
lib/tasks/uploads.rake
View File

@ -8,6 +8,8 @@ require "base62"
# gather #
################################################################################
require_dependency "rake_helpers"
task "uploads:gather" => :environment do
ENV["RAILS_DB"] ? gather_uploads : gather_uploads_for_all_sites
end
@ -426,7 +428,7 @@ def migrate_to_s3
%Q{attachment; filename="#{upload.original_filename}"}
end
if upload&.private?
if upload.secure
options[:acl] = "private"
end
end
@ -907,6 +909,108 @@ task "uploads:recover" => :environment do
end
end
##
# Run this task whenever the secure_media or login_required
# settings are changed for a Discourse instance to update
# the upload secure flag and S3 upload ACLs.
task "uploads:ensure_correct_acl" => :environment do
RailsMultisite::ConnectionManagement.each_connection do |db|
unless Discourse.store.external?
puts "This task only works for external storage."
exit 1
end
puts "Ensuring correct ACL for uploads in #{db}...", ""
Upload.transaction do
mark_secure_in_loop_because_no_login_required = false
# First of all only get relevant uploads (supported media).
#
# Also only get uploads that are not for a theme or a site setting, so only
# get post related uploads.
uploads_with_supported_media = Upload.includes(:posts, :optimized_images).where(
"LOWER(original_filename) SIMILAR TO '%\.(jpg|jpeg|png|gif|svg|ico|mp3|ogg|wav|m4a|mov|mp4|webm|ogv)'"
).joins(:post_uploads)
puts "There are #{uploads_with_supported_media.count} upload(s) with supported media that could be marked secure.", ""
# Simply mark all these uploads as secure if login_required because no anons will be able to access them
if SiteSetting.login_required?
mark_all_as_secure_login_required(uploads_with_supported_media)
else
# If NOT login_required, then we have to go for the other slower flow, where in the loop
# we mark the upload as secure if the first post it is used in is with_secure_media?
mark_secure_in_loop_because_no_login_required = true
puts "Marking posts as secure in the next step because login_required is false."
end
puts "", "Rebaking #{uploads_with_supported_media.count} upload posts and updating ACLs in S3.", ""
upload_ids_to_mark_as_secure, uploads_skipped_because_of_error = update_acls_and_rebake_upload_posts(
uploads_with_supported_media, mark_secure_in_loop_because_no_login_required
)
log_rebake_errors(uploads_skipped_because_of_error)
mark_specific_uploads_as_secure_no_login_required(upload_ids_to_mark_as_secure)
end
end
puts "", "Done"
end
def mark_all_as_secure_login_required(uploads_with_supported_media)
puts "Marking #{uploads_with_supported_media.count} upload(s) as secure because login_required is true.", ""
uploads_with_supported_media.update_all(secure: true)
puts "Finished marking upload(s) as secure."
end
def log_rebake_errors(uploads_skipped_because_of_error)
return if uploads_skipped_because_of_error.empty?
puts "Skipped the following uploads due to error:", ""
uploads_skipped_because_of_error.each do |message|
puts message
end
end
def mark_specific_uploads_as_secure_no_login_required(upload_ids_to_mark_as_secure)
return if upload_ids_to_mark_as_secure.empty?
puts "Marking #{upload_ids_to_mark_as_secure.length} uploads as secure because their first post contains secure media."
Upload.where(id: upload_ids_to_mark_as_secure).update_all(secure: true)
puts "Finished marking uploads as secure."
end
def update_acls_and_rebake_upload_posts(uploads_with_supported_media, mark_secure_in_loop_because_no_login_required)
upload_ids_to_mark_as_secure = []
uploads_skipped_because_of_error = []
i = 0
uploads_with_supported_media.find_each(batch_size: 50) do |upload_with_supported_media|
RakeHelpers.print_status_with_label("Updating ACL for upload.......", i, uploads_with_supported_media.count)
Discourse.store.update_upload_ACL(upload_with_supported_media)
RakeHelpers.print_status_with_label("Rebaking posts for upload.....", i, uploads_with_supported_media.count)
begin
upload_with_supported_media.posts.each { |post| post.rebake! }
if mark_secure_in_loop_because_no_login_required
first_post_with_upload = upload_with_supported_media.posts.order(sort_order: :asc).first
mark_secure = first_post_with_upload ? first_post_with_upload.with_secure_media? : false
upload_ids_to_mark_as_secure << upload_with_supported_media.id if mark_secure
end
rescue => e
uploads_skipped_because_of_error << "#{upload_with_supported_media.original_filename} (#{upload_with_supported_media.url}) #{e.message}"
end
i += 1
end
RakeHelpers.print_status_with_label("Rebaking complete! ", i, uploads_with_supported_media.count)
puts ""
[upload_ids_to_mark_as_secure, uploads_skipped_because_of_error]
end
def inline_uploads(post)
replaced = false

7
lib/upload_creator.rb
View File

@ -118,6 +118,13 @@ class UploadCreator
@upload.for_site_setting = true if @opts[:for_site_setting]
@upload.for_gravatar = true if @opts[:for_gravatar]
if !FileHelper.is_supported_media?(@filename) &&
!@upload.for_theme &&
!@upload.for_site_setting &&
SiteSetting.prevent_anons_from_downloading_files
@upload.secure = true
end
return @upload unless @upload.save
# store the file and update its url

11
lib/url_helper.rb
View File

@ -38,6 +38,11 @@ class UrlHelper
url.sub(/^http:/i, "")
end
def self.secure_proxy_without_cdn(url)
url = url.sub(SiteSetting.Upload.absolute_base_url, "/secure-media-uploads")
self.absolute(url, nil)
end
DOUBLE_ESCAPED_REGEXP ||= /%25([0-9a-f]{2})/i
# Prevents double URL encode
@ -48,16 +53,16 @@ class UrlHelper
encoded
end
def self.cook_url(url)
def self.cook_url(url, secure: false)
return url unless is_local(url)
uri = URI.parse(url)
filename = File.basename(uri.path)
is_attachment = !FileHelper.is_supported_image?(filename)
is_attachment = !FileHelper.is_supported_media?(filename)
no_cdn = SiteSetting.login_required || SiteSetting.prevent_anons_from_downloading_files
url = absolute_without_cdn(url)
url = secure ? secure_proxy_without_cdn(url) : absolute_without_cdn(url)
unless is_attachment && no_cdn
url = Discourse.store.cdn_url(url)

189
spec/components/cooked_post_processor_spec.rb
View File

@ -812,8 +812,8 @@ describe CookedPostProcessor do
it "is always allowed to crawl our own images" do
store = stub
Discourse.expects(:store).returns(store).at_least_once
store.expects(:has_been_uploaded?).returns(true)
Discourse.expects(:store).returns(store)
FastImage.expects(:size).returns([100, 200])
expect(cpp.get_size("http://foo.bar/image2.png")).to eq([100, 200])
end
@ -1104,40 +1104,175 @@ describe CookedPostProcessor do
HTML
end
it "uses the right CDN when uploads are on S3" do
Rails.configuration.action_controller.stubs(:asset_host).returns("https://local.cdn.com")
context "s3_uploads" do
before do
Rails.configuration.action_controller.stubs(:asset_host).returns("https://local.cdn.com")
SiteSetting.s3_upload_bucket = "some-bucket-on-s3"
SiteSetting.s3_access_key_id = "s3-access-key-id"
SiteSetting.s3_secret_access_key = "s3-secret-access-key"
SiteSetting.s3_cdn_url = "https://s3.cdn.com"
SiteSetting.enable_s3_uploads = true
SiteSetting.s3_upload_bucket = "some-bucket-on-s3"
SiteSetting.s3_access_key_id = "s3-access-key-id"
SiteSetting.s3_secret_access_key = "s3-secret-access-key"
SiteSetting.s3_cdn_url = "https://s3.cdn.com"
SiteSetting.enable_s3_uploads = true
SiteSetting.authorized_extensions = "png|jpg|gif|mov|ogg|"
uploaded_file = file_from_fixtures("smallest.png")
upload_sha1 = Digest::SHA1.hexdigest(File.read(uploaded_file))
uploaded_file = file_from_fixtures("smallest.png")
upload_sha1 = Digest::SHA1.hexdigest(File.read(uploaded_file))
upload.update!(
original_filename: "smallest.png",
width: 10,
height: 20,
sha1: upload_sha1,
extension: "png",
)
end
upload.update!(
original_filename: "smallest.png",
width: 10,
height: 20,
sha1: upload_sha1,
extension: "png",
)
it "uses the right CDN when uploads are on S3" do
stored_path = Discourse.store.get_path_for_upload(upload)
upload.update_column(:url, "#{SiteSetting.Upload.absolute_base_url}/#{stored_path}")
stored_path = Discourse.store.get_path_for_upload(upload)
upload.update_column(:url, "#{SiteSetting.Upload.absolute_base_url}/#{stored_path}")
the_post = Fabricate(:post, raw: %Q{This post has a local emoji :+1: and an external upload\n\n![smallest.png|10x20](#{upload.short_url})})
the_post = Fabricate(:post, raw: %Q{This post has a local emoji :+1: and an external upload\n\n![smallest.png|10x20](#{upload.short_url})})
cpp = CookedPostProcessor.new(the_post)
cpp.optimize_urls
cpp = CookedPostProcessor.new(the_post)
cpp.optimize_urls
expect(cpp.html).to match_html <<~HTML
<p>This post has a local emoji <img src="https://local.cdn.com/images/emoji/twitter/+1.png?v=#{Emoji::EMOJI_VERSION}" title=":+1:" class="emoji" alt=":+1:"> and an external upload</p>
<p><img src="https://s3.cdn.com/#{stored_path}" alt="smallest.png" data-base62-sha1="#{upload.base62_sha1}" width="10" height="20"></p>
HTML
end
expect(cpp.html).to match_html <<~HTML
<p>This post has a local emoji <img src="https://local.cdn.com/images/emoji/twitter/+1.png?v=#{Emoji::EMOJI_VERSION}" title=":+1:" class="emoji" alt=":+1:"> and an external upload</p>
<p><img src="https://s3.cdn.com/#{stored_path}" alt="smallest.png" data-base62-sha1="#{upload.base62_sha1}" width="10" height="20"></p>
HTML
it "doesn't use CDN for secure media" do
SiteSetting.secure_media = true
stored_path = Discourse.store.get_path_for_upload(upload)
upload.update_column(:url, "#{SiteSetting.Upload.absolute_base_url}/#{stored_path}")
upload.update_column(:secure, true)
the_post = Fabricate(:post, raw: %Q{This post has a local emoji :+1: and an external upload\n\n![smallest.png|10x20](#{upload.short_url})})
cpp = CookedPostProcessor.new(the_post)
cpp.optimize_urls
expect(cpp.html).to match_html <<~HTML
<p>This post has a local emoji <img src="https://local.cdn.com/images/emoji/twitter/+1.png?v=#{Emoji::EMOJI_VERSION}" title=":+1:" class="emoji" alt=":+1:"> and an external upload</p>
<p><img src="/secure-media-uploads/#{stored_path}" alt="smallest.png" data-base62-sha1="#{upload.base62_sha1}" width="10" height="20"></p>
HTML
end
context "media uploads" do
fab!(:image_upload) { Fabricate(:upload) }
fab!(:audio_upload) { Fabricate(:upload, extension: "ogg") }
fab!(:video_upload) { Fabricate(:upload, extension: "mov") }
before do
video_upload.update!(url: "#{SiteSetting.s3_cdn_url}/#{Discourse.store.get_path_for_upload(video_upload)}")
stub_request(:head, video_upload.url)
end
it "ignores prevent_anons_from_downloading_files and oneboxes video uploads" do
SiteSetting.prevent_anons_from_downloading_files = true
the_post = Fabricate(:post, raw: "This post has an S3 video onebox:\n#{video_upload.url}")
cpp = CookedPostProcessor.new(the_post)
cpp.post_process_oneboxes
expect(cpp.html).to match_html <<~HTML
<p>This post has an S3 video onebox:<br></p>
<div class="onebox video-onebox">
<video width="100%" height="100%" controls="">
<source src="#{video_upload.url}">
<a href="#{video_upload.url}" rel="nofollow ugc noopener">#{video_upload.url}</a>
</source>
</video>
</div>
HTML
end
it "oneboxes video using secure url when secure_media is enabled" do
SiteSetting.login_required = true
SiteSetting.secure_media = true
video_upload.update_column(:secure, true)
the_post = Fabricate(:post, raw: "This post has an S3 video onebox:\n#{video_upload.url}")
cpp = CookedPostProcessor.new(the_post)
cpp.post_process_oneboxes
secure_url = video_upload.url.sub(SiteSetting.s3_cdn_url, "#{Discourse.base_url}/secure-media-uploads")
expect(cpp.html).to match_html <<~HTML
<p>This post has an S3 video onebox:<br>
<div class="onebox video-onebox">
<video width="100%" height="100%" controls="">
<source src="#{secure_url}">
<a href="#{secure_url}">#{secure_url}</a>
</source>
</video>
</div>
</p>
HTML
end
it "oneboxes only audio/video and not images when secure_media is enabled" do
SiteSetting.login_required = true
SiteSetting.secure_media = true
video_upload.update_column(:secure, true)
audio_upload.update!(
url: "#{SiteSetting.s3_cdn_url}/#{Discourse.store.get_path_for_upload(audio_upload)}",
secure: true
)
image_upload.update!(
url: "#{SiteSetting.s3_cdn_url}/#{Discourse.store.get_path_for_upload(image_upload)}",
secure: true
)
stub_request(:head, audio_upload.url)
stub_request(:get, image_upload.url)
raw = <<~RAW
This post has a video upload.
#{video_upload.url}
This post has an audio upload.
#{audio_upload.url}
And an image upload.
![logo.png](upload://#{image_upload.base62_sha1}.#{image_upload.extension})
RAW
the_post = Fabricate(:post, raw: raw)
cpp = CookedPostProcessor.new(the_post)
cpp.post_process_oneboxes
secure_video_url = video_upload.url.sub(SiteSetting.s3_cdn_url, "#{Discourse.base_url}/secure-media-uploads")
secure_audio_url = audio_upload.url.sub(SiteSetting.s3_cdn_url, "#{Discourse.base_url}/secure-media-uploads")
expect(cpp.html).to match_html <<~HTML
<p>This post has a video upload.<br></p>
<div class="onebox video-onebox">
<video width="100%" height="100%" controls="">
<source src="#{secure_video_url}">
<a href="#{secure_video_url}">#{secure_video_url}</a>
</source>
</video>
</div>
<p>This post has an audio upload.<br>
<audio controls><source src="#{secure_audio_url}"><a href="#{secure_audio_url}">#{secure_audio_url}</a></source></audio>
</p>
<p>And an image upload.<br>
<img src="#{image_upload.url}" alt="#{image_upload.original_filename}" data-base62-sha1="#{image_upload.base62_sha1}"></p>
HTML
end
end
end
end
end

15
spec/components/email/styles_spec.rb
View File

@ -199,4 +199,19 @@ describe Email::Styles do
end
end
context "replace_relative_urls" do
it "replaces secure media within a link with a placeholder" do
frag = html_fragment("<a href=\"#{Discourse.base_url}\/secure-media-uploads/original/1X/testimage.png\"><img src=\"/secure-media-uploads/original/1X/testimage.png\"></a>")
expect(frag.at('p.secure-media-notice')).to be_present
expect(frag.at('img')).not_to be_present
expect(frag.at('a')).not_to be_present
end
it "replaces secure images with a placeholder" do
frag = html_fragment("<img src=\"/secure-media-uploads/original/1X/testimage.png\">")
expect(frag.at('p.secure-media-notice')).to be_present
expect(frag.at('img')).not_to be_present
end
end
end

13
spec/components/file_store/base_store_spec.rb
View File

@ -86,5 +86,18 @@ RSpec.describe FileStore::BaseStore do
expect(file.class).to eq(File)
end
it "should return the file when secure media are enabled" do
SiteSetting.login_required = true
SiteSetting.secure_media = true
stub_request(:head, "https://s3-upload-bucket.s3.amazonaws.com/")
signed_url = Discourse.store.signed_url_for_path(upload_s3.url)
stub_request(:get, signed_url).to_return(status: 200, body: "Hello world")
file = store.download(upload_s3)
expect(file.class).to eq(File)
end
end
end

78
spec/components/file_store/s3_store_spec.rb
View File

@ -43,16 +43,16 @@ describe FileStore::S3Store do
let(:s3_object) { stub }
let(:etag) { "etag" }
before do
s3_object.stubs(:put).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
end
describe "#store_upload" do
it "returns an absolute schemaless url" do
store.expects(:get_depth_for).with(upload.id).returns(0)
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.png").returns(s3_object)
s3_object.expects(:put).with(
acl: "public-read",
cache_control: "max-age=31556952, public, immutable",
content_type: "image/png",
body: uploaded_file).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
expect(store.store_upload(uploaded_file, upload)).to eq(
"//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/original/1X/#{upload.sha1}.png"
@ -62,6 +62,7 @@ describe FileStore::S3Store do
describe "when s3_upload_bucket includes folders path" do
before do
s3_object.stubs(:put).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
SiteSetting.s3_upload_bucket = "s3-upload-bucket/discourse-uploads"
end
@ -78,28 +79,36 @@ describe FileStore::S3Store do
end
end
describe "when private uploads are enabled" do
it "returns signed URL for eligible private upload" do
describe "when secure uploads are enabled" do
it "saves secure attachment using private ACL" do
SiteSetting.prevent_anons_from_downloading_files = true
SiteSetting.authorized_extensions = "pdf|png|jpg|gif"
upload.update!(original_filename: "small.pdf", extension: "pdf")
upload.update!(original_filename: "small.pdf", extension: "pdf", secure: true)
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.pdf").returns(s3_object).at_least_once
s3_object.expects(:presigned_url).with(:get, expires_in: S3Helper::DOWNLOAD_URL_EXPIRES_AFTER_SECONDS)
s3_helper.expects(:s3_bucket).returns(s3_bucket)
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.pdf").returns(s3_object)
s3_object.expects(:put).with(
acl: "private",
cache_control: "max-age=31556952, public, immutable",
content_type: "application/pdf",
content_disposition: "attachment; filename=\"#{upload.original_filename}\"",
body: uploaded_file).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
expect(store.store_upload(uploaded_file, upload)).to eq(
"//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/original/1X/#{upload.sha1}.pdf"
)
expect(store.url_for(upload)).not_to eq(upload.url)
end
it "returns regular URL for ineligible private upload" do
it "saves image upload using public ACL" do
SiteSetting.prevent_anons_from_downloading_files = true
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.png").returns(s3_object).at_least_once
s3_object.expects(:put).with(
acl: "public-read",
cache_control: "max-age=31556952, public, immutable",
content_type: "image/png",
body: uploaded_file).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
expect(store.store_upload(uploaded_file, upload)).to eq(
"//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/original/1X/#{upload.sha1}.png"
@ -111,6 +120,10 @@ describe FileStore::S3Store do
end
describe "#store_optimized_image" do
before do
s3_object.stubs(:put).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
end
it "returns an absolute schemaless url" do
store.expects(:get_depth_for).with(optimized_image.upload.id).returns(0)
s3_helper.expects(:s3_bucket).returns(s3_bucket)
@ -355,23 +368,27 @@ describe FileStore::S3Store do
include_context "s3 helpers"
let(:s3_object) { stub }
before do
SiteSetting.authorized_extensions = "pdf|png"
end
describe ".update_upload_ACL" do
it "sets acl to private when private uploads are enabled" do
SiteSetting.prevent_anons_from_downloading_files = true
it "sets acl to public by default" do
upload.update!(original_filename: "small.pdf", extension: "pdf")
s3_helper.expects(:s3_bucket).returns(s3_bucket)
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.png").returns(s3_object)
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.pdf").returns(s3_object)
s3_object.expects(:acl).returns(s3_object)
s3_object.expects(:put).with(acl: "private").returns(s3_object)
s3_object.expects(:put).with(acl: "public-read").returns(s3_object)
expect(store.update_upload_ACL(upload)).to be_truthy
end
it "sets acl to public when private uploads are disabled" do
SiteSetting.prevent_anons_from_downloading_files = false
it "sets acl to private when upload is marked secure" do
upload.update!(original_filename: "small.pdf", extension: "pdf", secure: true)
s3_helper.expects(:s3_bucket).returns(s3_bucket)
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.png").returns(s3_object)
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.pdf").returns(s3_object)
s3_object.expects(:acl).returns(s3_object)
s3_object.expects(:put).with(acl: "public-read").returns(s3_object)
s3_object.expects(:put).with(acl: "private").returns(s3_object)
expect(store.update_upload_ACL(upload)).to be_truthy
end
@ -421,4 +438,21 @@ describe FileStore::S3Store do
end
end
describe ".signed_url_for_path" do
include_context "s3 helpers"
let(:s3_object) { stub }
it "returns signed URL for a given path" do
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("special/optimized/file.png").returns(s3_object)
opts = {
expires_in: S3Helper::DOWNLOAD_URL_EXPIRES_AFTER_SECONDS
}
s3_object.expects(:presigned_url).with(:get, opts)
expect(store.signed_url_for_path("special/optimized/file.png")).not_to eq(upload.url)
end
end
end

59
spec/components/post_creator_spec.rb
View File

@ -1388,4 +1388,63 @@ describe PostCreator do
end
end
end
context "secure media uploads" do
fab!(:image_upload) { Fabricate(:upload, secure: true) }
fab!(:user2) { Fabricate(:user) }
fab!(:public_topic) { Fabricate(:topic) }
before do
SiteSetting.enable_s3_uploads = true
SiteSetting.authorized_extensions = "png|jpg|gif|mp4"
SiteSetting.s3_upload_bucket = "s3-upload-bucket"
SiteSetting.s3_access_key_id = "some key"
SiteSetting.s3_secret_access_key = "some secret key"
SiteSetting.secure_media = true
stub_request(:head, "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/")
stub_request(
:put,
"https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{image_upload.sha1}.#{image_upload.extension}?acl"
)
end
it "does not allow a secure image to be used in a public topic" do
public_post = PostCreator.create(
user,
topic_id: public_topic.id,
raw: "A public post with an image.\n![](#{image_upload.short_path})"
)
expect(public_post.errors.count).to be(1)
expect(public_post.errors.full_messages).to include(I18n.t('secure_upload_not_allowed_in_public_topic', upload_filenames: image_upload.original_filename))
# secure upload CAN be used in another PM
pm = PostCreator.create(
user,
title: 'this is another private message',
raw: "with an upload: \n![](#{image_upload.short_path})",
archetype: Archetype.private_message,
target_usernames: [user2.username].join(',')
)
expect(pm.errors).to be_blank
end
it "does not allow a secure video to be used in a public topic" do
video_upload = Fabricate(:upload_s3, extension: 'mp4', original_filename: "video.mp4", secure: true)
public_post = PostCreator.create(
user,
topic_id: public_topic.id,
raw: "A public post with a video onebox:\n#{video_upload.url}"
)
expect(public_post.errors.count).to be(1)
expect(public_post.errors.full_messages).to include(I18n.t('secure_upload_not_allowed_in_public_topic', upload_filenames: video_upload.original_filename))
end
end
end

44
spec/components/pretty_text_spec.rb
View File

@ -810,6 +810,50 @@ describe PrettyText do
html = "<p>Check out this video – <iframe src='https://player.vimeo.com/video/329875646' data-original-href='https://vimeo.com/329875646/> <script>alert(1)</script>'></iframe>.</p>"
expect(PrettyText.format_for_email(html, post)).to match(Regexp.escape("https://vimeo.com/329875646/%3E%20%3Cscript%3Ealert(1)%3C/script%3E"))
end
describe "#strip_secure_media" do
before do
SiteSetting.s3_upload_bucket = "some-bucket-on-s3"
SiteSetting.s3_access_key_id = "s3-access-key-id"
SiteSetting.s3_secret_access_key = "s3-secret-access-key"
SiteSetting.s3_cdn_url = "https://s3.cdn.com"
SiteSetting.enable_s3_uploads = true
SiteSetting.secure_media = true
SiteSetting.login_required = true
end
it "replaces secure video content" do
html = <<~HTML
<video width="100%" height="100%" controls="">
<source src="#{base_url}/secure-media-uploads/original/1X/some-video.mp4">
<a href="#{base_url}/secure-media-uploads/original/1X/some-video.mp4">Video label</a>
</source>
</video>
HTML
md = PrettyText.format_for_email(html, post)
expect(md).not_to include('<video')
expect(md.to_s).to match(I18n.t("emails.secure_media_placeholder"))
expect(md.to_s).not_to match(SiteSetting.Upload.s3_cdn_url)
end
it "replaces secure audio content" do
html = <<~HTML
<audio controls>
<source src="#{base_url}/secure-media-uploads/original/1X/some-audio.mp3">
<a href="#{base_url}/secure-media-uploads/original/1X/some-audio.mp3">Audio label</a>
</source>
</audio>
HTML
md = PrettyText.format_for_email(html, post)
expect(md).not_to include('<video')
expect(md.to_s).to match(I18n.t("emails.secure_media_placeholder"))
expect(md.to_s).not_to match(SiteSetting.Upload.s3_cdn_url)
end
end
end
it 'Is smart about linebreaks and IMG tags' do

20
spec/lib/upload_creator_spec.rb
View File

@ -170,7 +170,7 @@ RSpec.describe UploadCreator do
end
end
describe 'private uploads' do
describe 'secure attachments' do
let(:filename) { "small.pdf" }
let(:file) { file_from_fixtures(filename, "pdf") }
@ -179,31 +179,31 @@ RSpec.describe UploadCreator do
SiteSetting.authorized_extensions = 'pdf|svg|jpg'
end
it 'should mark uploads as private' do
it 'should mark attachments as secure' do
upload = UploadCreator.new(file, filename).create_for(user.id)
stored_upload = Upload.last
expect(stored_upload.private?).to eq(true)
expect(stored_upload.secure?).to eq(true)
end
it 'should not mark theme uploads as private' do
it 'should not mark theme uploads as secure' do
fname = "custom-theme-icon-sprite.svg"
upload = UploadCreator.new(file_from_fixtures(fname), fname, for_theme: true).create_for(-1)
expect(upload.private?).to eq(false)
expect(upload.secure?).to eq(false)
end
it 'should not mark image uploads as private' do
it 'should not apply prevent_anons_from_downloading_files to image uploads' do
fname = "logo.jpg"
upload = UploadCreator.new(file_from_fixtures(fname), fname).create_for(user.id)
stored_upload = Upload.last
expect(stored_upload.original_filename).to eq(fname)
expect(stored_upload.private?).to eq(false)
expect(stored_upload.secure?).to eq(false)
end
end
describe 'uploading to s3' do
context 'uploading to s3' do
let(:filename) { "should_be_jpeg.png" }
let(:file) { file_from_fixtures(filename) }
let(:pdf_filename) { "small.pdf" }
@ -233,7 +233,7 @@ RSpec.describe UploadCreator do
expect(upload.etag).to eq('ETag')
end
it 'should return signed URL for private uploads in S3' do
it 'should return signed URL for secure attachments in S3' do
SiteSetting.prevent_anons_from_downloading_files = true
SiteSetting.authorized_extensions = 'pdf'
@ -241,7 +241,7 @@ RSpec.describe UploadCreator do
stored_upload = Upload.last
signed_url = Discourse.store.url_for(stored_upload)
expect(stored_upload.private?).to eq(true)
expect(stored_upload.secure?).to eq(true)
expect(stored_upload.url).not_to eq(signed_url)
expect(signed_url).to match(/Amz-Credential/)
end

44
spec/mailers/user_notifications_spec.rb
View File

@ -674,6 +674,50 @@ describe UserNotifications do
expect(mail.body.to_s).to match(I18n.t("user_notifications.reached_limit", count: 2))
end
describe "secure media" do
let(:video_upload) { Fabricate(:upload, extension: "mov") }
let(:user) { Fabricate(:user) }
let(:post) { Fabricate(:post) }
before do
SiteSetting.s3_upload_bucket = "some-bucket-on-s3"
SiteSetting.s3_access_key_id = "s3-access-key-id"
SiteSetting.s3_secret_access_key = "s3-secret-access-key"
SiteSetting.s3_cdn_url = "https://s3.cdn.com"
SiteSetting.enable_s3_uploads = true
SiteSetting.secure_media = true
SiteSetting.login_required = true
video_upload.update!(url: "#{SiteSetting.s3_cdn_url}/#{Discourse.store.get_path_for_upload(video_upload)}")
user.email_logs.create!(
email_type: 'blah',
to_address: user.email,
user_id: user.id
)
end
it "replaces secure audio/video with placeholder" do
reply = Fabricate(:post, topic_id: post.topic_id, raw: "Video: #{video_upload.url}")
notification = Fabricate(
:notification,
topic_id: post.topic_id,
post_number: reply.post_number,
user: post.user,
data: { original_username: 'bob' }.to_json
)
mail = UserNotifications.user_replied(
user,
post: reply,
notification_type: notification.notification_type,
notification_data_hash: notification.data_hash
)
expect(mail.body.to_s).to match(I18n.t("emails.secure_media_placeholder"))
end
end
def expects_build_with(condition)
UserNotifications.any_instance.expects(:build_email).with(user.email, condition)
mailer = UserNotifications.public_send(

133
spec/models/post_spec.rb
View File

@ -1245,7 +1245,7 @@ describe Post do
expect(post.revisions.pluck(:number)).to eq([1, 2])
end
describe '#link_post_uploads' do
describe 'uploads' do
fab!(:video_upload) { Fabricate(:upload, extension: "mp4") }
fab!(:image_upload) { Fabricate(:upload) }
fab!(:audio_upload) { Fabricate(:upload, extension: "ogg") }
@ -1257,7 +1257,7 @@ describe Post do
let(:video_url) { "#{base_url}#{video_upload.url}" }
let(:audio_url) { "#{base_url}#{audio_upload.url}" }
let(:raw) do
let(:raw_multiple) do
<<~RAW
<a href="#{attachment_upload.url}">Link</a>
[test|attachment](#{attachment_upload_2.short_url})
@ -1276,36 +1276,121 @@ describe Post do
RAW
end
let(:post) { Fabricate(:post, raw: raw) }
let(:post) { Fabricate(:post, raw: raw_multiple) }
it "finds all the uploads in the post" do
post.custom_fields[Post::DOWNLOADED_IMAGES] = {
"/uploads/default/original/1X/1/1234567890123456.csv": attachment_upload.id
}
context "#link_post_uploads" do
it "finds all the uploads in the post" do
post.custom_fields[Post::DOWNLOADED_IMAGES] = {
"/uploads/default/original/1X/1/1234567890123456.csv": attachment_upload.id
}
post.save_custom_fields
post.link_post_uploads
post.save_custom_fields
post.link_post_uploads
expect(PostUpload.where(post: post).pluck(:upload_id)).to contain_exactly(
video_upload.id,
image_upload.id,
audio_upload.id,
attachment_upload.id,
attachment_upload_2.id,
attachment_upload_3.id
)
expect(PostUpload.where(post: post).pluck(:upload_id)).to contain_exactly(
video_upload.id,
image_upload.id,
audio_upload.id,
attachment_upload.id,
attachment_upload_2.id,
attachment_upload_3.id
)
end
it "cleans the reverse index up for the current post" do
post.link_post_uploads
post_uploads_ids = post.post_uploads.pluck(:id)
post.link_post_uploads
expect(post.reload.post_uploads.pluck(:id)).to_not contain_exactly(
post_uploads_ids
)
end
end
it "cleans the reverse index up for the current post" do
post.link_post_uploads
context '#update_uploads_secure_status' do
fab!(:user) { Fabricate(:user, trust_level: 0) }
post_uploads_ids = post.post_uploads.pluck(:id)
let(:raw) do
<<~RAW
<a href="#{attachment_upload.url}">Link</a>
<img src="#{image_upload.url}">
RAW
end
post.link_post_uploads
before do
SiteSetting.authorized_extensions = "pdf|png|jpg|csv"
SiteSetting.enable_s3_uploads = true
SiteSetting.s3_upload_bucket = "s3-upload-bucket"
SiteSetting.s3_access_key_id = "some key"
SiteSetting.s3_secret_access_key = "some secret key"
SiteSetting.secure_media = true
attachment_upload.update!(original_filename: "hello.csv")
expect(post.reload.post_uploads.pluck(:id)).to_not contain_exactly(
post_uploads_ids
)
stub_request(:head, "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/")
stub_request(
:put,
"https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{attachment_upload.sha1}.#{attachment_upload.extension}?acl"
)
stub_request(
:put,
"https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{image_upload.sha1}.#{image_upload.extension}?acl"
)
end
it "marks image uploads as secure in PMs when secure_media is ON" do
post = Fabricate(:post, raw: raw, user: user, topic: Fabricate(:private_message_topic, user: user))
post.link_post_uploads
post.update_uploads_secure_status
expect(PostUpload.where(post: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
[attachment_upload.id, false],
[image_upload.id, true]
)
end
it "marks image uploads as not secure in PMs when when secure_media is ON" do
SiteSetting.secure_media = false
post = Fabricate(:post, raw: raw, user: user, topic: Fabricate(:private_message_topic, user: user))
post.link_post_uploads
post.update_uploads_secure_status
expect(PostUpload.where(post: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
[attachment_upload.id, false],
[image_upload.id, false]
)
end
it "marks attachments as secure when relevant setting is enabled" do
SiteSetting.prevent_anons_from_downloading_files = true
post = Fabricate(:post, raw: raw, user: user, topic: Fabricate(:topic, user: user))
post.link_post_uploads
post.update_uploads_secure_status
expect(PostUpload.where(post: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
[attachment_upload.id, true],
[image_upload.id, false]
)
end
it "does not mark an upload as secure if it has already been used in a public topic" do
post = Fabricate(:post, raw: raw, user: user, topic: Fabricate(:topic, user: user))
post.link_post_uploads
post.update_uploads_secure_status
pm = Fabricate(:post, raw: raw, user: user, topic: Fabricate(:private_message_topic, user: user))
pm.link_post_uploads
pm.update_uploads_secure_status
expect(PostUpload.where(post: pm).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
[attachment_upload.id, false],
[image_upload.id, false]
)
end
end
end

41
spec/models/topic_converter_spec.rb
View File

@ -102,6 +102,47 @@ describe TopicConverter do
expect(Notification.exists?(user_notification.id)).to eq(false)
expect(Notification.exists?(admin_notification.id)).to eq(true)
end
context "secure uploads" do
fab!(:image_upload) { Fabricate(:upload) }
fab!(:public_topic) { Fabricate(:topic, user: author) }
before do
SiteSetting.enable_s3_uploads = true
SiteSetting.s3_upload_bucket = "s3-upload-bucket"
SiteSetting.s3_access_key_id = "some key"
SiteSetting.s3_secret_access_key = "some secret key"
SiteSetting.secure_media = true
stub_request(:head, "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/")
stub_request(
:put,
"https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{image_upload.sha1}.#{image_upload.extension}?acl"
)
end
it "converts regular uploads to secure when making a public post a PM" do
public_reply = Fabricate(:post, raw: "<img src='#{image_upload.url}'>", user: other_user, topic: public_topic)
public_reply.link_post_uploads
public_reply.update_uploads_secure_status
expect(public_reply.uploads[0].secure).to eq(false)
public_topic.convert_to_private_message(admin)
expect(public_topic.reload.posts.find(public_reply.id).uploads[0].secure).to eq(true)
end
it "converts secure uploads back to public" do
first_post
second_post = Fabricate(:post, raw: "<img src='#{image_upload.url}'>", user: other_user, topic: private_message)
second_post.link_post_uploads
second_post.update_uploads_secure_status
expect(second_post.uploads[0].secure).to eq(true)
private_message.convert_to_public_topic(admin)
expect(private_message.reload.posts.find(second_post.id).uploads[0].secure).to eq(false)
end
end
end
end

78
spec/models/upload_spec.rb
View File

@ -288,6 +288,84 @@ describe Upload do
end
end
describe '.update_secure_status' do
it 'marks a local upload as not secure with default settings' do
upload.update!(secure: true)
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.secure).to eq(false)
end
it 'marks a local attachment as secure if prevent_anons_from_downloading_files is enabled' do
SiteSetting.prevent_anons_from_downloading_files = true
SiteSetting.authorized_extensions = "pdf"
upload.update!(original_filename: "small.pdf", extension: "pdf")
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.secure).to eq(true)
end
it 'marks a local attachment as not secure if prevent_anons_from_downloading_files is disabled' do
SiteSetting.prevent_anons_from_downloading_files = false
SiteSetting.authorized_extensions = "pdf"
upload.update!(original_filename: "small.pdf", extension: "pdf", secure: true)
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.secure).to eq(false)
end
it 'does not change secure status of a non-attachment when prevent_anons_from_downloading_files is enabled' do
SiteSetting.prevent_anons_from_downloading_files = true
SiteSetting.authorized_extensions = "mp4"
upload.update!(original_filename: "small.mp4", extension: "mp4")
expect { upload.update_secure_status }
.not_to change { upload.secure }
expect(upload.secure).to eq(false)
end
context "secure media enabled" do
before do
SiteSetting.enable_s3_uploads = true
SiteSetting.s3_upload_bucket = "s3-upload-bucket"
SiteSetting.s3_access_key_id = "some key"
SiteSetting.s3_secret_access_key = "some secret key"
SiteSetting.secure_media = true
stub_request(:head, "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/")
stub_request(
:put,
"https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{upload.sha1}.#{upload.extension}?acl"
)
end
it 'marks an image upload as not secure when not associated with a post' do
upload.update!(secure: true)
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.secure).to eq(false)
end
it 'marks an image upload as secure if login_required is enabled' do
SiteSetting.login_required = true
upload.update!(secure: false)
expect { upload.update_secure_status }
.to change { upload.secure }
expect(upload.secure).to eq(true)
end
end
end
describe '.reset_unknown_extensions!' do
it 'should reset the extension of uploads when it is "unknown"' do
upload1 = Fabricate(:upload, extension: "unknown")

41
spec/multisite/s3_store_spec.rb
View File

@ -119,7 +119,7 @@ RSpec.describe 'Multisite s3 uploads', type: :multisite do
end
end
context 'private uploads' do
context 'secure uploads' do
let(:store) { FileStore::S3Store.new }
let(:client) { Aws::S3::Client.new(stub_responses: true) }
let(:resource) { Aws::S3::Resource.new(client: client) }
@ -133,18 +133,18 @@ RSpec.describe 'Multisite s3 uploads', type: :multisite do
SiteSetting.s3_secret_access_key = "s3-secret-access-key"
SiteSetting.enable_s3_uploads = true
SiteSetting.prevent_anons_from_downloading_files = true
SiteSetting.authorized_extensions = "pdf|png|jpg|gif"
end
before do
s3_object.stubs(:put).returns(Aws::S3::Types::PutObjectOutput.new(etag: "etag"))
end
describe "when private uploads are enabled" do
describe "when secure attachments are enabled" do
it "returns signed URL with correct path" do
test_multisite_connection('default') do
SiteSetting.authorized_extensions = "pdf|png|jpg|gif"
upload = build_upload
upload.update!(original_filename: "small.pdf", extension: "pdf")
upload.update!(original_filename: "small.pdf", extension: "pdf", secure: true)
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("uploads/default/original/1X/#{upload.sha1}.pdf").returns(s3_object).at_least_once
@ -159,13 +159,41 @@ RSpec.describe 'Multisite s3 uploads', type: :multisite do
end
end
describe "when secure media are enabled" do
before do
SiteSetting.login_required = true
SiteSetting.secure_media = true
s3_helper.stubs(:s3_client).returns(client)
Discourse.stubs(:store).returns(store)
end
it "returns signed URL with correct path" do
test_multisite_connection('default') do
upload = Fabricate.build(:upload_s3, sha1: upload_sha1, id: 1)
signed_url = Discourse.store.signed_url_for_path(upload.url)
expect(signed_url).to match(/Amz-Expires/)
expect(signed_url).to match("uploads/default")
end
test_multisite_connection('second') do
upload = Fabricate.build(:upload_s3, sha1: upload_sha1, id: 1)
signed_url = Discourse.store.signed_url_for_path(upload.url)
expect(signed_url).to match(/Amz-Expires/)
expect(signed_url).to match("uploads/second")
end
end
end
describe "#update_upload_ACL" do
it "updates correct file for default and second multisite db" do
test_multisite_connection('default') do
upload = build_upload
upload.update!(original_filename: "small.pdf", extension: "pdf", secure: true)
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("uploads/default/original/1X/#{upload.sha1}.png").returns(s3_object)
s3_bucket.expects(:object).with("uploads/default/original/1X/#{upload.sha1}.pdf").returns(s3_object)
s3_object.expects(:acl).returns(s3_object)
s3_object.expects(:put).with(acl: "private").returns(s3_object)
@ -174,9 +202,10 @@ RSpec.describe 'Multisite s3 uploads', type: :multisite do
test_multisite_connection('second') do
upload = build_upload
upload.update!(original_filename: "small.pdf", extension: "pdf", secure: true)
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("uploads/second/original/1X/#{upload.sha1}.png").returns(s3_object)
s3_bucket.expects(:object).with("uploads/second/original/1X/#{upload.sha1}.pdf").returns(s3_object)
s3_object.expects(:acl).returns(s3_object)
s3_object.expects(:put).with(acl: "private").returns(s3_object)

89
spec/requests/uploads_controller_spec.rb
View File

@ -361,6 +361,59 @@ describe UploadsController do
end
end
describe "#show_secure" do
describe "local store" do
fab!(:image_upload) { upload_file("smallest.png") }
it "does not return secure media when using local store" do
secure_url = image_upload.url.sub("/uploads", "/secure-media-uploads")
get secure_url
expect(response.status).to eq(404)
end
end
describe "s3 store" do
let(:upload) { Fabricate(:upload_s3) }
before do
SiteSetting.enable_s3_uploads = true
SiteSetting.s3_upload_bucket = "s3-upload-bucket"
SiteSetting.s3_access_key_id = "fakeid7974664"
SiteSetting.s3_secret_access_key = "fakesecretid7974664"
SiteSetting.secure_media = true
end
it "should return 404 for anonymous requests requests" do
secure_url = upload.url.sub(SiteSetting.Upload.absolute_base_url, "/secure-media-uploads")
get secure_url
expect(response.status).to eq(404)
end
it "should return signed url for legitimate request" do
secure_url = upload.url.sub(SiteSetting.Upload.absolute_base_url, "/secure-media-uploads")
sign_in(user)
stub_request(:head, "https://s3-upload-bucket.s3.amazonaws.com/")
get secure_url
expect(response.status).to eq(302)
expect(response.redirect_url).to match("Amz-Expires")
end
it "should return secure media URL when looking up urls" do
upload.update_column(:secure, true)
sign_in(user)
post "/uploads/lookup-urls.json", params: { short_urls: [upload.short_url] }
expect(response.status).to eq(200)
result = JSON.parse(response.body)
expect(result[0]["url"]).to match("secure-media-uploads")
end
end
end
describe '#lookup_urls' do
it 'can look up long urls' do
sign_in(user)
@ -373,6 +426,42 @@ describe UploadsController do
expect(result[0]["url"]).to eq(upload.url)
expect(result[0]["short_path"]).to eq(upload.short_path)
end
describe 'secure media' do
let(:upload) { Fabricate(:upload_s3, secure: true) }
before do
SiteSetting.authorized_extensions = "pdf|png"
SiteSetting.s3_upload_bucket = "s3-upload-bucket"
SiteSetting.s3_access_key_id = "s3-access-key-id"
SiteSetting.s3_secret_access_key = "s3-secret-access-key"
SiteSetting.enable_s3_uploads = true
SiteSetting.secure_media = true
end
it 'returns secure url for a secure media upload' do
sign_in(user)
post "/uploads/lookup-urls.json", params: { short_urls: [upload.short_url] }
expect(response.status).to eq(200)
result = JSON.parse(response.body)
expect(result[0]["url"]).to match("/secure-media-uploads")
expect(result[0]["short_path"]).to eq(upload.short_path)
end
it 'does not return secure urls for non-media uploads' do
upload.update!(original_filename: "not-an-image.pdf", extension: "pdf")
sign_in(user)
post "/uploads/lookup-urls.json", params: { short_urls: [upload.short_url] }
expect(response.status).to eq(200)
result = JSON.parse(response.body)
expect(result[0]["url"]).not_to match("/secure-media-uploads")
expect(result[0]["short_path"]).to eq(upload.short_path)
end
end
end
describe '#metadata' do