discourse/app/models/upload.rb

685 lines
18 KiB
Ruby

# frozen_string_literal: true
require "digest/sha1"
class Upload < ActiveRecord::Base
include ActionView::Helpers::NumberHelper
include HasUrl
SHA1_LENGTH = 40
SEEDED_ID_THRESHOLD = 0
URL_REGEX ||= %r{(/original/\dX[/\.\w]*/(\h+)[\.\w]*)}
MAX_IDENTIFY_SECONDS = 5
DOMINANT_COLOR_COMMAND_TIMEOUT_SECONDS = 5
belongs_to :user
belongs_to :access_control_post, class_name: "Post"
# when we access this post we don't care if the post
# is deleted
def access_control_post
Post.unscoped { super }
end
has_many :post_hotlinked_media, dependent: :destroy, class_name: "PostHotlinkedMedia"
has_many :optimized_images, dependent: :destroy
has_many :user_uploads, dependent: :destroy
has_many :upload_references, dependent: :destroy
has_many :posts, through: :upload_references, source: :target, source_type: "Post"
has_many :topic_thumbnails
attr_accessor :for_group_message
attr_accessor :for_theme
attr_accessor :for_private_message
attr_accessor :for_export
attr_accessor :for_site_setting
attr_accessor :for_gravatar
attr_accessor :validate_file_size
validates_presence_of :filesize
validates_presence_of :original_filename
validates :dominant_color, length: { is: 6 }, allow_blank: true, allow_nil: true
validates_with UploadValidator
before_destroy do
UserProfile.where(card_background_upload_id: self.id).update_all(card_background_upload_id: nil)
UserProfile.where(profile_background_upload_id: self.id).update_all(
profile_background_upload_id: nil,
)
end
after_destroy do
User.where(uploaded_avatar_id: self.id).update_all(uploaded_avatar_id: nil)
UserAvatar.where(gravatar_upload_id: self.id).update_all(gravatar_upload_id: nil)
UserAvatar.where(custom_upload_id: self.id).update_all(custom_upload_id: nil)
end
scope :by_users, -> { where("uploads.id > ?", SEEDED_ID_THRESHOLD) }
def self.verification_statuses
@verification_statuses ||= Enum.new(unchecked: 1, verified: 2, invalid_etag: 3)
end
def self.add_unused_callback(&block)
(@unused_callbacks ||= []) << block
end
def self.unused_callbacks
@unused_callbacks
end
def self.reset_unused_callbacks
@unused_callbacks = []
end
def self.add_in_use_callback(&block)
(@in_use_callbacks ||= []) << block
end
def self.in_use_callbacks
@in_use_callbacks
end
def self.reset_in_use_callbacks
@in_use_callbacks = []
end
def self.with_no_non_post_relations
self.joins(
"LEFT JOIN upload_references ur ON ur.upload_id = uploads.id AND ur.target_type != 'Post'",
).where("ur.upload_id IS NULL")
end
def initialize(*args)
super
self.validate_file_size = true
end
def to_s
self.url
end
def to_markdown
UploadMarkdown.new(self).to_markdown
end
def thumbnail(width = self.thumbnail_width, height = self.thumbnail_height)
optimized_images.find_by(width: width, height: height)
end
def has_thumbnail?(width, height)
thumbnail(width, height).present?
end
def create_thumbnail!(width, height, opts = nil)
return unless SiteSetting.create_thumbnails?
opts ||= {}
save(validate: false) if get_optimized_image(width, height, opts)
end
# this method attempts to correct old incorrect extensions
def get_optimized_image(width, height, opts = nil)
opts ||= {}
fix_image_extension if (!extension || extension.length == 0)
opts = opts.merge(raise_on_error: true)
begin
OptimizedImage.create_for(self, width, height, opts)
rescue => ex
Rails.logger.info ex if Rails.env.development?
opts = opts.merge(raise_on_error: false)
if fix_image_extension
OptimizedImage.create_for(self, width, height, opts)
else
nil
end
end
end
def content
original_path = Discourse.store.path_for(self)
external_copy = nil
if original_path.blank?
external_copy = Discourse.store.download!(self)
original_path = external_copy.path
end
File.read(original_path)
ensure
File.unlink(external_copy.path) if external_copy
end
def fix_image_extension
return false if extension == "unknown"
begin
# this is relatively cheap once cached
original_path = Discourse.store.path_for(self)
if original_path.blank?
external_copy = Discourse.store.download_safe(self)
original_path = external_copy&.path
end
image_info =
begin
FastImage.new(original_path)
rescue StandardError
nil
end
new_extension = image_info&.type&.to_s || "unknown"
if new_extension != self.extension
self.update_columns(extension: new_extension)
true
end
rescue StandardError
self.update_columns(extension: "unknown")
true
end
end
def destroy
Upload.transaction do
Discourse.store.remove_upload(self)
super
end
end
def short_url
"upload://#{short_url_basename}"
end
def uploaded_before_secure_uploads_enabled?
original_sha1.blank?
end
def matching_access_control_post?(post)
access_control_post_id == post.id
end
def copied_from_other_post?(post)
return false if access_control_post_id.blank?
!matching_access_control_post?(post)
end
def short_path
self.class.short_path(sha1: self.sha1, extension: self.extension)
end
def self.consider_for_reuse(upload, post)
return upload if !SiteSetting.secure_uploads? || upload.blank? || post.blank?
if !upload.matching_access_control_post?(post) || upload.uploaded_before_secure_uploads_enabled?
return nil
end
upload
end
def self.secure_uploads_url?(url)
# we do not want to exclude topic links that for whatever reason
# have secure-uploads in the URL e.g. /t/secure-uploads-are-cool/223452
route = UrlHelper.rails_route_from_url(url)
return false if route.blank?
route[:action] == "show_secure" && route[:controller] == "uploads" &&
FileHelper.is_supported_media?(url)
rescue ActionController::RoutingError
false
end
def self.signed_url_from_secure_uploads_url(url)
route = UrlHelper.rails_route_from_url(url)
url = Rails.application.routes.url_for(route.merge(only_path: true))
secure_upload_s3_path = url[url.index(route[:path])..-1]
Discourse.store.signed_url_for_path(secure_upload_s3_path)
end
def self.secure_uploads_url_from_upload_url(url)
return url if !url.include?(SiteSetting.Upload.absolute_base_url)
uri = URI.parse(url)
Rails.application.routes.url_for(
controller: "uploads",
action: "show_secure",
path: uri.path[1..-1],
only_path: true,
)
end
def self.short_path(sha1:, extension:)
@url_helpers ||= Rails.application.routes.url_helpers
@url_helpers.upload_short_path(base62: self.base62_sha1(sha1), extension: extension)
end
def self.base62_sha1(sha1)
Base62.encode(sha1.hex)
end
def base62_sha1
Upload.base62_sha1(self.sha1)
end
def local?
!(url =~ %r{\A(https?:)?//})
end
def fix_dimensions!
return if !FileHelper.is_supported_image?("image.#{extension}")
begin
path =
if local?
Discourse.store.path_for(self)
else
Discourse.store.download!(self).path
end
if extension == "svg"
w, h =
begin
Discourse::Utils.execute_command(
"identify",
"-format",
"%w %h",
path,
timeout: MAX_IDENTIFY_SECONDS,
).split(" ")
rescue StandardError
[0, 0]
end
else
w, h = FastImage.new(path, raise_on_failure: true).size
end
self.width = w || 0
self.height = h || 0
self.thumbnail_width, self.thumbnail_height = ImageSizer.resize(w, h)
self.update_columns(
width: width,
height: height,
thumbnail_width: thumbnail_width,
thumbnail_height: thumbnail_height,
)
rescue => e
Discourse.warn_exception(e, message: "Error getting image dimensions")
end
nil
end
# on demand image size calculation, this allows us to null out image sizes
# and still handle as needed
def get_dimension(key)
if v = read_attribute(key)
return v
end
fix_dimensions!
read_attribute(key)
end
def width
get_dimension(:width)
end
def height
get_dimension(:height)
end
def thumbnail_width
get_dimension(:thumbnail_width)
end
def thumbnail_height
get_dimension(:thumbnail_height)
end
def dominant_color(calculate_if_missing: false)
val = read_attribute(:dominant_color)
if val.nil? && calculate_if_missing
calculate_dominant_color!
read_attribute(:dominant_color)
else
val
end
end
def calculate_dominant_color!(local_path = nil)
color = nil
color = "" if !FileHelper.is_supported_image?("image.#{extension}") || extension == "svg"
if color.nil?
local_path ||=
if local?
Discourse.store.path_for(self)
else
Discourse.store.download_safe(self)&.path
end
if local_path.nil?
# Download failed. Could be too large to download, or file could be missing in s3
color = ""
end
color ||=
begin
data =
Discourse::Utils.execute_command(
"nice",
"-n",
"10",
"convert",
local_path,
"-depth",
"8",
"-resize",
"1x1",
"-define",
"histogram:unique-colors=true",
"-format",
"%c",
"histogram:info:",
timeout: DOMINANT_COLOR_COMMAND_TIMEOUT_SECONDS,
)
# Output format:
# 1: (110.873,116.226,93.8821) #6F745E srgb(43.4798%,45.5789%,36.8165%)
color = data[/#([0-9A-F]{6})/, 1]
raise "Calculated dominant color but unable to parse output:\n#{data}" if color.nil?
color
rescue Discourse::Utils::CommandError => e
# Timeout or unable to parse image
# This can happen due to bad user input - ignore and save
# an empty string to prevent re-evaluation
""
end
end
if persisted?
self.update_column(:dominant_color, color)
else
self.dominant_color = color
end
end
def target_image_quality(local_path, test_quality)
@file_quality ||=
begin
Discourse::Utils.execute_command(
"identify",
"-format",
"%Q",
local_path,
timeout: MAX_IDENTIFY_SECONDS,
).to_i
rescue StandardError
0
end
test_quality if @file_quality == 0 || @file_quality > test_quality
end
def self.sha1_from_short_path(path)
self.sha1_from_base62_encoded($2) if path =~ %r{(/uploads/short-url/)([a-zA-Z0-9]+)(\..*)?}
end
def self.sha1_from_short_url(url)
self.sha1_from_base62_encoded($2) if url =~ %r{(upload://)?([a-zA-Z0-9]+)(\..*)?}
end
def self.sha1_from_base62_encoded(encoded_sha1)
sha1 = Base62.decode(encoded_sha1).to_s(16)
if sha1.length > SHA1_LENGTH
nil
else
sha1.rjust(SHA1_LENGTH, "0")
end
end
def self.generate_digest(path)
Digest::SHA1.file(path).hexdigest
end
def human_filesize
number_to_human_size(self.filesize)
end
def rebake_posts_on_old_scheme
self.posts.where("cooked LIKE '%/_optimized/%'").find_each(&:rebake!)
end
def update_secure_status(source: "unknown", override: nil)
if override.nil?
mark_secure, reason = UploadSecurity.new(self).should_be_secure_with_reason
else
mark_secure = override
reason = "manually overridden"
end
secure_status_did_change = self.secure? != mark_secure
self.update(secure_params(mark_secure, reason, source))
if Discourse.store.external?
begin
Discourse.store.update_upload_ACL(self)
rescue Aws::S3::Errors::NotImplemented => err
Discourse.warn_exception(
err,
message: "The file store object storage provider does not support setting ACLs",
)
end
end
secure_status_did_change
end
def secure_params(secure, reason, source = "unknown")
{
secure: secure,
security_last_changed_reason: reason + " | source: #{source}",
security_last_changed_at: Time.zone.now,
}
end
def self.migrate_to_new_scheme(limit: nil)
problems = []
DistributedMutex.synchronize("migrate_upload_to_new_scheme") do
if SiteSetting.migrate_to_new_scheme
max_file_size_kb = [
SiteSetting.max_image_size_kb,
SiteSetting.max_attachment_size_kb,
].max.kilobytes
local_store = FileStore::LocalStore.new
db = RailsMultisite::ConnectionManagement.current_db
scope =
Upload
.by_users
.where("url NOT LIKE '%/original/_X/%' AND url LIKE ?", "%/uploads/#{db}%")
.order(id: :desc)
scope = scope.limit(limit) if limit
if scope.count == 0
SiteSetting.migrate_to_new_scheme = false
return problems
end
remap_scope = nil
scope.each do |upload|
begin
# keep track of the url
previous_url = upload.url.dup
# where is the file currently stored?
external = previous_url =~ %r{\A//}
# download if external
if external
url = SiteSetting.scheme + ":" + previous_url
begin
retries ||= 0
file =
FileHelper.download(
url,
max_file_size: max_file_size_kb,
tmp_file_name: "discourse",
follow_redirect: true,
)
rescue OpenURI::HTTPError
retry if (retries += 1) < 1
next
end
path = file.path
else
path = local_store.path_for(upload)
end
# compute SHA if missing
upload.sha1 = Upload.generate_digest(path) if upload.sha1.blank?
# store to new location & update the filesize
File.open(path) do |f|
upload.url = Discourse.store.store_upload(f, upload)
upload.filesize = f.size
upload.save!(validate: false)
end
# remap the URLs
DbHelper.remap(UrlHelper.absolute(previous_url), upload.url) unless external
DbHelper.remap(
previous_url,
upload.url,
excluded_tables: %w[
posts
post_search_data
incoming_emails
notifications
single_sign_on_records
stylesheet_cache
topic_search_data
users
user_emails
draft_sequences
optimized_images
],
)
remap_scope ||=
begin
Post
.with_deleted
.where(
"raw ~ '/uploads/#{db}/\\d+/' OR raw ~ '/uploads/#{db}/original/(\\d|[a-z])/'",
)
.select(:id, :raw, :cooked)
.all
end
remap_scope.each do |post|
post.raw.gsub!(previous_url, upload.url)
post.cooked.gsub!(previous_url, upload.url)
if post.changed?
Post.with_deleted.where(id: post.id).update_all(raw: post.raw, cooked: post.cooked)
end
end
upload.optimized_images.find_each(&:destroy!)
upload.rebake_posts_on_old_scheme
# remove the old file (when local)
FileUtils.rm(path, force: true) unless external
rescue => e
problems << { upload: upload, ex: e }
ensure
file&.unlink
file&.close
end
end
end
end
problems
end
def self.extract_upload_ids(raw)
return [] if raw.blank?
sha1s = []
raw.scan(/\/(\h{40})/).each { |match| sha1s << match[0] }
raw
.scan(%r{/([a-zA-Z0-9]+)})
.each { |match| sha1s << Upload.sha1_from_base62_encoded(match[0]) }
Upload.where(sha1: sha1s.uniq).pluck(:id)
end
def self.backfill_dominant_colors!(count)
Upload
.where(dominant_color: nil)
.order("id desc")
.first(count)
.each { |upload| upload.calculate_dominant_color! }
end
private
def short_url_basename
"#{Upload.base62_sha1(sha1)}#{extension.present? ? ".#{extension}" : ""}"
end
end
# == Schema Information
#
# Table name: uploads
#
# id :integer not null, primary key
# user_id :integer not null
# original_filename :string not null
# filesize :bigint not null
# width :integer
# height :integer
# url :string not null
# created_at :datetime not null
# updated_at :datetime not null
# sha1 :string(40)
# origin :string(1000)
# retain_hours :integer
# extension :string(10)
# thumbnail_width :integer
# thumbnail_height :integer
# etag :string
# secure :boolean default(FALSE), not null
# access_control_post_id :bigint
# original_sha1 :string
# animated :boolean
# verification_status :integer default(1), not null
# security_last_changed_at :datetime
# security_last_changed_reason :string
# dominant_color :text
#
# Indexes
#
# idx_uploads_on_verification_status (verification_status)
# index_uploads_on_access_control_post_id (access_control_post_id)
# index_uploads_on_etag (etag)
# index_uploads_on_extension (lower((extension)::text))
# index_uploads_on_id (id) WHERE (dominant_color IS NULL)
# index_uploads_on_id_and_url (id,url)
# index_uploads_on_original_sha1 (original_sha1)
# index_uploads_on_sha1 (sha1) UNIQUE
# index_uploads_on_url (url)
# index_uploads_on_user_id (user_id)
#