# frozen_string_literal: true require "digest/sha1" class Upload < ActiveRecord::Base self.ignored_columns = [ "verified" # TODO(2020-12-10): remove ] include ActionView::Helpers::NumberHelper include HasUrl SHA1_LENGTH = 40 SEEDED_ID_THRESHOLD = 0 URL_REGEX ||= /(\/original\/\dX[\/\.\w]*\/(\h+)[\.\w]*)/ MAX_IDENTIFY_SECONDS = 5 belongs_to :user belongs_to :access_control_post, class_name: 'Post' # when we access this post we don't care if the post # is deleted def access_control_post Post.unscoped { super } end has_many :post_uploads, dependent: :destroy has_many :posts, through: :post_uploads has_many :optimized_images, dependent: :destroy has_many :user_uploads, dependent: :destroy has_many :topic_thumbnails attr_accessor :for_group_message attr_accessor :for_theme attr_accessor :for_private_message attr_accessor :for_export attr_accessor :for_site_setting attr_accessor :for_gravatar validates_presence_of :filesize validates_presence_of :original_filename validates_with UploadValidator before_destroy do UserProfile.where(card_background_upload_id: self.id).update_all(card_background_upload_id: nil) UserProfile.where(profile_background_upload_id: self.id).update_all(profile_background_upload_id: nil) end after_destroy do User.where(uploaded_avatar_id: self.id).update_all(uploaded_avatar_id: nil) UserAvatar.where(gravatar_upload_id: self.id).update_all(gravatar_upload_id: nil) UserAvatar.where(custom_upload_id: self.id).update_all(custom_upload_id: nil) end scope :by_users, -> { where("uploads.id > ?", SEEDED_ID_THRESHOLD) } def self.verification_statuses @verification_statuses ||= Enum.new( unchecked: 1, verified: 2, invalid_etag: 3 ) end def self.add_unused_callback(&block) (@unused_callbacks ||= []) << block end def self.unused_callbacks @unused_callbacks end def self.reset_unused_callbacks @unused_callbacks = [] end def self.add_in_use_callback(&block) (@in_use_callbacks ||= []) << block end def self.in_use_callbacks @in_use_callbacks end def self.reset_in_use_callbacks @in_use_callbacks = [] end def self.with_no_non_post_relations scope = self .joins(<<~SQL) LEFT JOIN site_settings ss ON NULLIF(ss.value, '')::integer = uploads.id AND ss.data_type = #{SiteSettings::TypeSupervisor.types[:upload].to_i} SQL .where("ss.value IS NULL") .joins("LEFT JOIN users u ON u.uploaded_avatar_id = uploads.id") .where("u.uploaded_avatar_id IS NULL") .joins("LEFT JOIN user_avatars ua ON ua.gravatar_upload_id = uploads.id OR ua.custom_upload_id = uploads.id") .where("ua.gravatar_upload_id IS NULL AND ua.custom_upload_id IS NULL") .joins("LEFT JOIN user_profiles up ON up.profile_background_upload_id = uploads.id OR up.card_background_upload_id = uploads.id") .where("up.profile_background_upload_id IS NULL AND up.card_background_upload_id IS NULL") .joins("LEFT JOIN categories c ON c.uploaded_logo_id = uploads.id OR c.uploaded_background_id = uploads.id") .where("c.uploaded_logo_id IS NULL AND c.uploaded_background_id IS NULL") .joins("LEFT JOIN custom_emojis ce ON ce.upload_id = uploads.id") .where("ce.upload_id IS NULL") .joins("LEFT JOIN theme_fields tf ON tf.upload_id = uploads.id") .where("tf.upload_id IS NULL") .joins("LEFT JOIN user_exports ue ON ue.upload_id = uploads.id") .where("ue.upload_id IS NULL") .joins("LEFT JOIN groups g ON g.flair_upload_id = uploads.id") .where("g.flair_upload_id IS NULL") .joins("LEFT JOIN badges b ON b.image_upload_id = uploads.id") .where("b.image_upload_id IS NULL") .joins(<<~SQL) LEFT JOIN theme_settings ts ON NULLIF(ts.value, '')::integer = uploads.id AND ts.data_type = #{ThemeSetting.types[:upload].to_i} SQL .where("ts.value IS NULL") if SiteSetting.selectable_avatars.present? scope = scope.where.not(id: SiteSetting.selectable_avatars.map(&:id)) end scope end def to_s self.url end def thumbnail(width = self.thumbnail_width, height = self.thumbnail_height) optimized_images.find_by(width: width, height: height) end def has_thumbnail?(width, height) thumbnail(width, height).present? end def create_thumbnail!(width, height, opts = nil) return unless SiteSetting.create_thumbnails? opts ||= {} if get_optimized_image(width, height, opts) save(validate: false) end end # this method attempts to correct old incorrect extensions def get_optimized_image(width, height, opts = nil) opts ||= {} if (!extension || extension.length == 0) fix_image_extension end opts = opts.merge(raise_on_error: true) begin OptimizedImage.create_for(self, width, height, opts) rescue => ex Rails.logger.info ex if Rails.env.development? opts = opts.merge(raise_on_error: false) if fix_image_extension OptimizedImage.create_for(self, width, height, opts) else nil end end end def content original_path = Discourse.store.path_for(self) external_copy = nil if original_path.blank? external_copy = Discourse.store.download(self) original_path = external_copy.path end File.read(original_path) ensure if external_copy File.unlink(external_copy.path) end end def fix_image_extension return false if extension == "unknown" begin # this is relatively cheap once cached original_path = Discourse.store.path_for(self) if original_path.blank? external_copy = Discourse.store.download(self) rescue nil original_path = external_copy.try(:path) end image_info = FastImage.new(original_path) rescue nil new_extension = image_info&.type&.to_s || "unknown" if new_extension != self.extension self.update_columns(extension: new_extension) true end rescue self.update_columns(extension: "unknown") true end end def destroy Upload.transaction do Discourse.store.remove_upload(self) super end end def short_url "upload://#{short_url_basename}" end def uploaded_before_secure_media_enabled? original_sha1.blank? end def matching_access_control_post?(post) access_control_post_id == post.id end def copied_from_other_post?(post) return false if access_control_post_id.blank? !matching_access_control_post?(post) end def short_path self.class.short_path(sha1: self.sha1, extension: self.extension) end def self.consider_for_reuse(upload, post) return upload if !SiteSetting.secure_media? || upload.blank? || post.blank? return nil if !upload.matching_access_control_post?(post) || upload.uploaded_before_secure_media_enabled? upload end def self.secure_media_url?(url) # we do not want to exclude topic links that for whatever reason # have secure-media-uploads in the URL e.g. /t/secure-media-uploads-are-cool/223452 route = UrlHelper.rails_route_from_url(url) return false if route.blank? route[:action] == "show_secure" && route[:controller] == "uploads" && FileHelper.is_supported_media?(url) rescue ActionController::RoutingError false end def self.signed_url_from_secure_media_url(url) route = UrlHelper.rails_route_from_url(url) url = Rails.application.routes.url_for(route.merge(only_path: true)) secure_upload_s3_path = url[url.index(route[:path])..-1] Discourse.store.signed_url_for_path(secure_upload_s3_path) end def self.secure_media_url_from_upload_url(url) return url if !url.include?(SiteSetting.Upload.absolute_base_url) uri = URI.parse(url) Rails.application.routes.url_for( controller: "uploads", action: "show_secure", path: uri.path[1..-1], only_path: true ) end def self.short_path(sha1:, extension:) @url_helpers ||= Rails.application.routes.url_helpers @url_helpers.upload_short_path( base62: self.base62_sha1(sha1), extension: extension ) end def self.base62_sha1(sha1) Base62.encode(sha1.hex) end def base62_sha1 Upload.base62_sha1(self.sha1) end def local? !(url =~ /^(https?:)?\/\//) end def fix_dimensions! return if !FileHelper.is_supported_image?("image.#{extension}") path = if local? Discourse.store.path_for(self) else Discourse.store.download(self).path end begin if extension == 'svg' w, h = Discourse::Utils.execute_command("identify", "-format", "%w %h", path, timeout: MAX_IDENTIFY_SECONDS).split(' ') rescue [0, 0] else w, h = FastImage.new(path, raise_on_failure: true).size end self.width = w || 0 self.height = h || 0 self.thumbnail_width, self.thumbnail_height = ImageSizer.resize(w, h) self.update_columns( width: width, height: height, thumbnail_width: thumbnail_width, thumbnail_height: thumbnail_height ) rescue => e Discourse.warn_exception(e, message: "Error getting image dimensions") end nil end # on demand image size calculation, this allows us to null out image sizes # and still handle as needed def get_dimension(key) if v = read_attribute(key) return v end fix_dimensions! read_attribute(key) end def width get_dimension(:width) end def height get_dimension(:height) end def thumbnail_width get_dimension(:thumbnail_width) end def thumbnail_height get_dimension(:thumbnail_height) end def target_image_quality(local_path, test_quality) @file_quality ||= Discourse::Utils.execute_command("identify", "-format", "%Q", local_path, timeout: MAX_IDENTIFY_SECONDS).to_i rescue 0 if @file_quality == 0 || @file_quality > test_quality test_quality end end def self.sha1_from_short_path(path) if path =~ /(\/uploads\/short-url\/)([a-zA-Z0-9]+)(\..*)?/ self.sha1_from_base62_encoded($2) end end def self.sha1_from_short_url(url) if url =~ /(upload:\/\/)?([a-zA-Z0-9]+)(\..*)?/ self.sha1_from_base62_encoded($2) end end def self.sha1_from_base62_encoded(encoded_sha1) sha1 = Base62.decode(encoded_sha1).to_s(16) if sha1.length > SHA1_LENGTH nil else sha1.rjust(SHA1_LENGTH, '0') end end def self.generate_digest(path) Digest::SHA1.file(path).hexdigest end def human_filesize number_to_human_size(self.filesize) end def rebake_posts_on_old_scheme self.posts.where("cooked LIKE '%/_optimized/%'").find_each(&:rebake!) end def update_secure_status(source: "unknown", override: nil) if override.nil? mark_secure, reason = UploadSecurity.new(self).should_be_secure_with_reason else mark_secure = override reason = "manually overridden" end secure_status_did_change = self.secure? != mark_secure self.update(secure_params(mark_secure, reason, source)) if Discourse.store.external? begin Discourse.store.update_upload_ACL(self) rescue Aws::S3::Errors::NotImplemented => err Discourse.warn_exception(err, message: "The file store object storage provider does not support setting ACLs") end end secure_status_did_change end def secure_params(secure, reason, source = "unknown") { secure: secure, security_last_changed_reason: reason + " | source: #{source}", security_last_changed_at: Time.zone.now } end def self.migrate_to_new_scheme(limit: nil) problems = [] DistributedMutex.synchronize("migrate_upload_to_new_scheme") do if SiteSetting.migrate_to_new_scheme max_file_size_kb = [ SiteSetting.max_image_size_kb, SiteSetting.max_attachment_size_kb ].max.kilobytes local_store = FileStore::LocalStore.new db = RailsMultisite::ConnectionManagement.current_db scope = Upload.by_users .where("url NOT LIKE '%/original/_X/%' AND url LIKE '%/uploads/#{db}%'") .order(id: :desc) scope = scope.limit(limit) if limit if scope.count == 0 SiteSetting.migrate_to_new_scheme = false return problems end remap_scope = nil scope.each do |upload| begin # keep track of the url previous_url = upload.url.dup # where is the file currently stored? external = previous_url =~ /^\/\// # download if external if external url = SiteSetting.scheme + ":" + previous_url begin retries ||= 0 file = FileHelper.download( url, max_file_size: max_file_size_kb, tmp_file_name: "discourse", follow_redirect: true ) rescue OpenURI::HTTPError retry if (retries += 1) < 1 next end path = file.path else path = local_store.path_for(upload) end # compute SHA if missing if upload.sha1.blank? upload.sha1 = Upload.generate_digest(path) end # store to new location & update the filesize File.open(path) do |f| upload.url = Discourse.store.store_upload(f, upload) upload.filesize = f.size upload.save!(validate: false) end # remap the URLs DbHelper.remap(UrlHelper.absolute(previous_url), upload.url) unless external DbHelper.remap( previous_url, upload.url, excluded_tables: %w{ posts post_search_data incoming_emails notifications single_sign_on_records stylesheet_cache topic_search_data users user_emails draft_sequences optimized_images } ) remap_scope ||= begin Post.with_deleted .where("raw ~ '/uploads/#{db}/\\d+/' OR raw ~ '/uploads/#{db}/original/(\\d|[a-z])/'") .select(:id, :raw, :cooked) .all end remap_scope.each do |post| post.raw.gsub!(previous_url, upload.url) post.cooked.gsub!(previous_url, upload.url) Post.with_deleted.where(id: post.id).update_all(raw: post.raw, cooked: post.cooked) if post.changed? end upload.optimized_images.find_each(&:destroy!) upload.rebake_posts_on_old_scheme # remove the old file (when local) unless external FileUtils.rm(path, force: true) end rescue => e problems << { upload: upload, ex: e } ensure file&.unlink file&.close end end end end problems end private def short_url_basename "#{Upload.base62_sha1(sha1)}#{extension.present? ? ".#{extension}" : ""}" end end # == Schema Information # # Table name: uploads # # id :integer not null, primary key # user_id :integer not null # original_filename :string not null # filesize :bigint not null # width :integer # height :integer # url :string not null # created_at :datetime not null # updated_at :datetime not null # sha1 :string(40) # origin :string(1000) # retain_hours :integer # extension :string(10) # thumbnail_width :integer # thumbnail_height :integer # etag :string # secure :boolean default(FALSE), not null # access_control_post_id :bigint # original_sha1 :string # verification_status :integer default(1), not null # animated :boolean # security_last_changed_at :datetime # security_last_changed_reason :string # # Indexes # # idx_uploads_on_verification_status (verification_status) # index_uploads_on_access_control_post_id (access_control_post_id) # index_uploads_on_etag (etag) # index_uploads_on_extension (lower((extension)::text)) # index_uploads_on_id_and_url (id,url) # index_uploads_on_original_sha1 (original_sha1) # index_uploads_on_sha1 (sha1) UNIQUE # index_uploads_on_url (url) # index_uploads_on_user_id (user_id) #