2019-05-02 18:17:27 -04:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2013-11-05 13:04:47 -05:00
|
|
|
require "digest/sha1"
|
2013-02-05 14:16:51 -05:00
|
|
|
|
|
|
|
class Upload < ActiveRecord::Base
|
2020-09-16 23:35:29 -04:00
|
|
|
self.ignored_columns = [
|
|
|
|
"verified" # TODO(2020-12-10): remove
|
|
|
|
]
|
|
|
|
|
2019-02-20 21:13:37 -05:00
|
|
|
include ActionView::Helpers::NumberHelper
|
2019-04-08 16:37:35 -04:00
|
|
|
include HasUrl
|
2019-02-20 21:13:37 -05:00
|
|
|
|
2018-09-09 22:10:39 -04:00
|
|
|
SHA1_LENGTH = 40
|
2019-01-02 02:29:17 -05:00
|
|
|
SEEDED_ID_THRESHOLD = 0
|
2020-07-03 13:16:54 -04:00
|
|
|
URL_REGEX ||= /(\/original\/\dX[\/\.\w]*\/(\h+)[\.\w]*)/
|
2021-04-11 23:55:54 -04:00
|
|
|
MAX_IDENTIFY_SECONDS = 5
|
2018-09-09 22:10:39 -04:00
|
|
|
|
2013-02-05 14:16:51 -05:00
|
|
|
belongs_to :user
|
2020-01-15 22:50:27 -05:00
|
|
|
belongs_to :access_control_post, class_name: 'Post'
|
2013-02-05 14:16:51 -05:00
|
|
|
|
2020-03-02 18:03:58 -05:00
|
|
|
# when we access this post we don't care if the post
|
|
|
|
# is deleted
|
|
|
|
def access_control_post
|
|
|
|
Post.unscoped { super }
|
|
|
|
end
|
|
|
|
|
2013-11-05 13:04:47 -05:00
|
|
|
has_many :post_uploads, dependent: :destroy
|
2013-06-13 17:44:24 -04:00
|
|
|
has_many :posts, through: :post_uploads
|
2013-06-12 19:43:50 -04:00
|
|
|
|
2013-06-21 03:34:02 -04:00
|
|
|
has_many :optimized_images, dependent: :destroy
|
2018-09-20 01:33:10 -04:00
|
|
|
has_many :user_uploads, dependent: :destroy
|
FEATURE: Include optimized thumbnails for topics (#9215)
This introduces new APIs for obtaining optimized thumbnails for topics. There are a few building blocks required for this:
- Introduces new `image_upload_id` columns on the `posts` and `topics` table. This replaces the old `image_url` column, which means that thumbnails are now restricted to uploads. Hotlinked thumbnails are no longer possible. In normal use (with pull_hotlinked_images enabled), this has no noticeable impact
- A migration attempts to match existing urls to upload records. If a match cannot be found then the posts will be queued for rebake
- Optimized thumbnails are generated during post_process_cooked. If thumbnails are missing when serializing a topic list, then a sidekiq job is queued
- Topic lists and topics now include a `thumbnails` key, which includes all the available images:
```
"thumbnails": [
{
"max_width": null,
"max_height": null,
"url": "//example.com/original-image.png",
"width": 1380,
"height": 1840
},
{
"max_width": 1024,
"max_height": 1024,
"url": "//example.com/optimized-image.png",
"width": 768,
"height": 1024
}
]
```
- Themes can request additional thumbnail sizes by using a modifier in their `about.json` file:
```
"modifiers": {
"topic_thumbnail_sizes": [
[200, 200],
[800, 800]
],
...
```
Remember that these are generated asynchronously, so your theme should include logic to fallback to other available thumbnails if your requested size has not yet been generated
- Two new raw plugin outlets are introduced, to improve the customisability of the topic list. `topic-list-before-columns` and `topic-list-before-link`
2020-05-05 04:07:50 -04:00
|
|
|
has_many :topic_thumbnails
|
2013-06-16 04:39:48 -04:00
|
|
|
|
2017-06-12 16:41:29 -04:00
|
|
|
attr_accessor :for_group_message
|
2017-05-09 17:20:28 -04:00
|
|
|
attr_accessor :for_theme
|
2017-06-12 16:41:29 -04:00
|
|
|
attr_accessor :for_private_message
|
2018-04-19 07:30:31 -04:00
|
|
|
attr_accessor :for_export
|
2018-11-14 02:03:02 -05:00
|
|
|
attr_accessor :for_site_setting
|
2019-07-30 23:16:03 -04:00
|
|
|
attr_accessor :for_gravatar
|
2016-02-29 16:39:24 -05:00
|
|
|
|
2013-02-05 14:16:51 -05:00
|
|
|
validates_presence_of :filesize
|
|
|
|
validates_presence_of :original_filename
|
|
|
|
|
2019-10-02 00:01:53 -04:00
|
|
|
validates_with UploadValidator
|
2014-04-14 16:55:57 -04:00
|
|
|
|
2020-08-17 20:55:16 -04:00
|
|
|
before_destroy do
|
|
|
|
UserProfile.where(card_background_upload_id: self.id).update_all(card_background_upload_id: nil)
|
|
|
|
UserProfile.where(profile_background_upload_id: self.id).update_all(profile_background_upload_id: nil)
|
|
|
|
end
|
|
|
|
|
2018-08-31 00:46:22 -04:00
|
|
|
after_destroy do
|
|
|
|
User.where(uploaded_avatar_id: self.id).update_all(uploaded_avatar_id: nil)
|
|
|
|
UserAvatar.where(gravatar_upload_id: self.id).update_all(gravatar_upload_id: nil)
|
|
|
|
UserAvatar.where(custom_upload_id: self.id).update_all(custom_upload_id: nil)
|
|
|
|
end
|
|
|
|
|
2019-01-02 02:29:17 -05:00
|
|
|
scope :by_users, -> { where("uploads.id > ?", SEEDED_ID_THRESHOLD) }
|
|
|
|
|
2020-09-16 23:35:29 -04:00
|
|
|
def self.verification_statuses
|
|
|
|
@verification_statuses ||= Enum.new(
|
|
|
|
unchecked: 1,
|
|
|
|
verified: 2,
|
|
|
|
invalid_etag: 3
|
|
|
|
)
|
|
|
|
end
|
|
|
|
|
2021-06-23 18:09:40 -04:00
|
|
|
def self.with_no_non_post_relations
|
|
|
|
scope = self
|
|
|
|
.joins(<<~SQL)
|
|
|
|
LEFT JOIN site_settings ss
|
|
|
|
ON NULLIF(ss.value, '')::integer = uploads.id
|
|
|
|
AND ss.data_type = #{SiteSettings::TypeSupervisor.types[:upload].to_i}
|
|
|
|
SQL
|
|
|
|
.where("ss.value IS NULL")
|
|
|
|
.joins("LEFT JOIN users u ON u.uploaded_avatar_id = uploads.id")
|
|
|
|
.where("u.uploaded_avatar_id IS NULL")
|
|
|
|
.joins("LEFT JOIN user_avatars ua ON ua.gravatar_upload_id = uploads.id OR ua.custom_upload_id = uploads.id")
|
|
|
|
.where("ua.gravatar_upload_id IS NULL AND ua.custom_upload_id IS NULL")
|
|
|
|
.joins("LEFT JOIN user_profiles up ON up.profile_background_upload_id = uploads.id OR up.card_background_upload_id = uploads.id")
|
|
|
|
.where("up.profile_background_upload_id IS NULL AND up.card_background_upload_id IS NULL")
|
|
|
|
.joins("LEFT JOIN categories c ON c.uploaded_logo_id = uploads.id OR c.uploaded_background_id = uploads.id")
|
|
|
|
.where("c.uploaded_logo_id IS NULL AND c.uploaded_background_id IS NULL")
|
|
|
|
.joins("LEFT JOIN custom_emojis ce ON ce.upload_id = uploads.id")
|
|
|
|
.where("ce.upload_id IS NULL")
|
|
|
|
.joins("LEFT JOIN theme_fields tf ON tf.upload_id = uploads.id")
|
|
|
|
.where("tf.upload_id IS NULL")
|
|
|
|
.joins("LEFT JOIN user_exports ue ON ue.upload_id = uploads.id")
|
|
|
|
.where("ue.upload_id IS NULL")
|
|
|
|
.joins("LEFT JOIN groups g ON g.flair_upload_id = uploads.id")
|
|
|
|
.where("g.flair_upload_id IS NULL")
|
|
|
|
.joins("LEFT JOIN badges b ON b.image_upload_id = uploads.id")
|
|
|
|
.where("b.image_upload_id IS NULL")
|
|
|
|
|
|
|
|
if SiteSetting.selectable_avatars.present?
|
|
|
|
scope = scope.where.not(id: SiteSetting.selectable_avatars.map(&:id))
|
|
|
|
end
|
|
|
|
|
|
|
|
scope
|
|
|
|
end
|
|
|
|
|
2018-11-14 02:03:02 -05:00
|
|
|
def to_s
|
|
|
|
self.url
|
|
|
|
end
|
|
|
|
|
2018-08-27 22:48:43 -04:00
|
|
|
def thumbnail(width = self.thumbnail_width, height = self.thumbnail_height)
|
2014-05-06 09:41:59 -04:00
|
|
|
optimized_images.find_by(width: width, height: height)
|
2013-06-16 19:00:25 -04:00
|
|
|
end
|
|
|
|
|
2013-11-05 13:04:47 -05:00
|
|
|
def has_thumbnail?(width, height)
|
2013-09-27 04:55:50 -04:00
|
|
|
thumbnail(width, height).present?
|
2013-06-16 19:00:25 -04:00
|
|
|
end
|
|
|
|
|
2018-12-14 16:50:28 -05:00
|
|
|
def create_thumbnail!(width, height, opts = nil)
|
2013-06-16 19:00:25 -04:00
|
|
|
return unless SiteSetting.create_thumbnails?
|
2018-12-14 16:50:28 -05:00
|
|
|
opts ||= {}
|
2015-09-20 16:01:03 -04:00
|
|
|
|
2018-08-17 00:00:27 -04:00
|
|
|
if get_optimized_image(width, height, opts)
|
2015-06-26 19:26:16 -04:00
|
|
|
save(validate: false)
|
2013-09-27 04:55:50 -04:00
|
|
|
end
|
2013-06-16 19:00:25 -04:00
|
|
|
end
|
|
|
|
|
2018-08-17 00:00:27 -04:00
|
|
|
# this method attempts to correct old incorrect extensions
|
2020-10-16 06:41:27 -04:00
|
|
|
def get_optimized_image(width, height, opts = nil)
|
|
|
|
opts ||= {}
|
|
|
|
|
2018-08-17 00:00:27 -04:00
|
|
|
if (!extension || extension.length == 0)
|
|
|
|
fix_image_extension
|
|
|
|
end
|
|
|
|
|
|
|
|
opts = opts.merge(raise_on_error: true)
|
|
|
|
begin
|
|
|
|
OptimizedImage.create_for(self, width, height, opts)
|
2018-12-14 17:44:38 -05:00
|
|
|
rescue => ex
|
|
|
|
Rails.logger.info ex if Rails.env.development?
|
2018-08-17 00:00:27 -04:00
|
|
|
opts = opts.merge(raise_on_error: false)
|
|
|
|
if fix_image_extension
|
|
|
|
OptimizedImage.create_for(self, width, height, opts)
|
|
|
|
else
|
|
|
|
nil
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def fix_image_extension
|
|
|
|
return false if extension == "unknown"
|
|
|
|
|
|
|
|
begin
|
|
|
|
# this is relatively cheap once cached
|
|
|
|
original_path = Discourse.store.path_for(self)
|
|
|
|
if original_path.blank?
|
|
|
|
external_copy = Discourse.store.download(self) rescue nil
|
|
|
|
original_path = external_copy.try(:path)
|
|
|
|
end
|
|
|
|
|
|
|
|
image_info = FastImage.new(original_path) rescue nil
|
|
|
|
new_extension = image_info&.type&.to_s || "unknown"
|
|
|
|
|
|
|
|
if new_extension != self.extension
|
|
|
|
self.update_columns(extension: new_extension)
|
|
|
|
true
|
|
|
|
end
|
|
|
|
rescue
|
|
|
|
self.update_columns(extension: "unknown")
|
|
|
|
true
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2013-06-21 03:34:02 -04:00
|
|
|
def destroy
|
2013-06-19 15:51:41 -04:00
|
|
|
Upload.transaction do
|
2013-08-13 16:08:29 -04:00
|
|
|
Discourse.store.remove_upload(self)
|
2013-06-19 15:51:41 -04:00
|
|
|
super
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-08-22 11:46:15 -04:00
|
|
|
def short_url
|
2019-05-28 21:00:25 -04:00
|
|
|
"upload://#{short_url_basename}"
|
|
|
|
end
|
|
|
|
|
2020-04-23 20:29:02 -04:00
|
|
|
def uploaded_before_secure_media_enabled?
|
|
|
|
original_sha1.blank?
|
|
|
|
end
|
|
|
|
|
|
|
|
def matching_access_control_post?(post)
|
|
|
|
access_control_post_id == post.id
|
|
|
|
end
|
|
|
|
|
|
|
|
def copied_from_other_post?(post)
|
|
|
|
return false if access_control_post_id.blank?
|
|
|
|
!matching_access_control_post?(post)
|
|
|
|
end
|
|
|
|
|
2019-05-28 21:00:25 -04:00
|
|
|
def short_path
|
|
|
|
self.class.short_path(sha1: self.sha1, extension: self.extension)
|
|
|
|
end
|
|
|
|
|
2020-01-28 19:11:38 -05:00
|
|
|
def self.consider_for_reuse(upload, post)
|
|
|
|
return upload if !SiteSetting.secure_media? || upload.blank? || post.blank?
|
2020-04-23 20:29:02 -04:00
|
|
|
return nil if !upload.matching_access_control_post?(post) || upload.uploaded_before_secure_media_enabled?
|
2020-01-28 19:11:38 -05:00
|
|
|
upload
|
|
|
|
end
|
|
|
|
|
2020-01-23 20:59:30 -05:00
|
|
|
def self.secure_media_url?(url)
|
2020-01-30 01:19:14 -05:00
|
|
|
# we do not want to exclude topic links that for whatever reason
|
|
|
|
# have secure-media-uploads in the URL e.g. /t/secure-media-uploads-are-cool/223452
|
2020-08-27 21:28:11 -04:00
|
|
|
route = UrlHelper.rails_route_from_url(url)
|
2020-09-30 01:20:00 -04:00
|
|
|
return false if route.blank?
|
2020-08-27 21:28:11 -04:00
|
|
|
route[:action] == "show_secure" && route[:controller] == "uploads" && FileHelper.is_supported_media?(url)
|
|
|
|
rescue ActionController::RoutingError
|
|
|
|
false
|
2020-01-23 20:59:30 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
def self.signed_url_from_secure_media_url(url)
|
2020-08-27 21:28:11 -04:00
|
|
|
route = UrlHelper.rails_route_from_url(url)
|
|
|
|
url = Rails.application.routes.url_for(route.merge(only_path: true))
|
|
|
|
secure_upload_s3_path = url[url.index(route[:path])..-1]
|
2020-01-23 20:59:30 -05:00
|
|
|
Discourse.store.signed_url_for_path(secure_upload_s3_path)
|
|
|
|
end
|
|
|
|
|
|
|
|
def self.secure_media_url_from_upload_url(url)
|
2020-08-27 21:28:11 -04:00
|
|
|
return url if !url.include?(SiteSetting.Upload.absolute_base_url)
|
|
|
|
uri = URI.parse(url)
|
|
|
|
Rails.application.routes.url_for(
|
|
|
|
controller: "uploads",
|
|
|
|
action: "show_secure",
|
|
|
|
path: uri.path[1..-1],
|
|
|
|
only_path: true
|
|
|
|
)
|
2020-01-23 20:59:30 -05:00
|
|
|
end
|
|
|
|
|
2019-05-28 21:00:25 -04:00
|
|
|
def self.short_path(sha1:, extension:)
|
|
|
|
@url_helpers ||= Rails.application.routes.url_helpers
|
|
|
|
|
|
|
|
@url_helpers.upload_short_path(
|
|
|
|
base62: self.base62_sha1(sha1),
|
|
|
|
extension: extension
|
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
def self.base62_sha1(sha1)
|
|
|
|
Base62.encode(sha1.hex)
|
|
|
|
end
|
|
|
|
|
|
|
|
def base62_sha1
|
2019-06-04 02:10:46 -04:00
|
|
|
Upload.base62_sha1(self.sha1)
|
2017-08-22 11:46:15 -04:00
|
|
|
end
|
|
|
|
|
2018-08-27 22:48:43 -04:00
|
|
|
def local?
|
|
|
|
!(url =~ /^(https?:)?\/\//)
|
|
|
|
end
|
|
|
|
|
|
|
|
def fix_dimensions!
|
2018-09-09 22:22:45 -04:00
|
|
|
return if !FileHelper.is_supported_image?("image.#{extension}")
|
2018-08-27 22:48:43 -04:00
|
|
|
|
|
|
|
path =
|
|
|
|
if local?
|
|
|
|
Discourse.store.path_for(self)
|
|
|
|
else
|
|
|
|
Discourse.store.download(self).path
|
|
|
|
end
|
|
|
|
|
2018-12-03 10:19:49 -05:00
|
|
|
begin
|
2021-03-01 11:44:00 -05:00
|
|
|
if extension == 'svg'
|
2021-04-11 23:55:54 -04:00
|
|
|
w, h = Discourse::Utils.execute_command("identify", "-format", "%w %h", path, timeout: MAX_IDENTIFY_SECONDS).split(' ') rescue [0, 0]
|
2021-03-01 11:44:00 -05:00
|
|
|
else
|
|
|
|
w, h = FastImage.new(path, raise_on_failure: true).size
|
|
|
|
end
|
2018-12-26 10:17:08 -05:00
|
|
|
|
|
|
|
self.width = w || 0
|
|
|
|
self.height = h || 0
|
|
|
|
|
|
|
|
self.thumbnail_width, self.thumbnail_height = ImageSizer.resize(w, h)
|
|
|
|
|
|
|
|
self.update_columns(
|
|
|
|
width: width,
|
|
|
|
height: height,
|
|
|
|
thumbnail_width: thumbnail_width,
|
|
|
|
thumbnail_height: thumbnail_height
|
|
|
|
)
|
2018-12-03 10:19:49 -05:00
|
|
|
rescue => e
|
|
|
|
Discourse.warn_exception(e, message: "Error getting image dimensions")
|
|
|
|
end
|
2018-08-27 22:48:43 -04:00
|
|
|
nil
|
|
|
|
end
|
|
|
|
|
|
|
|
# on demand image size calculation, this allows us to null out image sizes
|
|
|
|
# and still handle as needed
|
|
|
|
def get_dimension(key)
|
|
|
|
if v = read_attribute(key)
|
|
|
|
return v
|
|
|
|
end
|
|
|
|
fix_dimensions!
|
|
|
|
read_attribute(key)
|
|
|
|
end
|
|
|
|
|
|
|
|
def width
|
|
|
|
get_dimension(:width)
|
|
|
|
end
|
|
|
|
|
|
|
|
def height
|
|
|
|
get_dimension(:height)
|
|
|
|
end
|
|
|
|
|
|
|
|
def thumbnail_width
|
|
|
|
get_dimension(:thumbnail_width)
|
|
|
|
end
|
|
|
|
|
|
|
|
def thumbnail_height
|
|
|
|
get_dimension(:thumbnail_height)
|
|
|
|
end
|
|
|
|
|
2020-10-23 12:38:28 -04:00
|
|
|
def target_image_quality(local_path, test_quality)
|
2021-04-11 23:55:54 -04:00
|
|
|
@file_quality ||= Discourse::Utils.execute_command("identify", "-format", "%Q", local_path, timeout: MAX_IDENTIFY_SECONDS).to_i rescue 0
|
2020-10-23 12:38:28 -04:00
|
|
|
|
|
|
|
if @file_quality == 0 || @file_quality > test_quality
|
|
|
|
test_quality
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2019-05-28 21:00:25 -04:00
|
|
|
def self.sha1_from_short_path(path)
|
|
|
|
if path =~ /(\/uploads\/short-url\/)([a-zA-Z0-9]+)(\..*)?/
|
|
|
|
self.sha1_from_base62_encoded($2)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-08-22 11:46:15 -04:00
|
|
|
def self.sha1_from_short_url(url)
|
|
|
|
if url =~ /(upload:\/\/)?([a-zA-Z0-9]+)(\..*)?/
|
2019-05-28 21:00:25 -04:00
|
|
|
self.sha1_from_base62_encoded($2)
|
|
|
|
end
|
|
|
|
end
|
2019-05-28 11:18:21 -04:00
|
|
|
|
2019-05-28 21:00:25 -04:00
|
|
|
def self.sha1_from_base62_encoded(encoded_sha1)
|
|
|
|
sha1 = Base62.decode(encoded_sha1).to_s(16)
|
|
|
|
|
|
|
|
if sha1.length > SHA1_LENGTH
|
|
|
|
nil
|
|
|
|
else
|
|
|
|
sha1.rjust(SHA1_LENGTH, '0')
|
2017-08-22 11:46:15 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2016-09-02 02:50:13 -04:00
|
|
|
def self.generate_digest(path)
|
|
|
|
Digest::SHA1.file(path).hexdigest
|
|
|
|
end
|
|
|
|
|
2019-02-20 21:13:37 -05:00
|
|
|
def human_filesize
|
|
|
|
number_to_human_size(self.filesize)
|
|
|
|
end
|
|
|
|
|
2019-04-03 00:37:50 -04:00
|
|
|
def rebake_posts_on_old_scheme
|
|
|
|
self.posts.where("cooked LIKE '%/_optimized/%'").find_each(&:rebake!)
|
|
|
|
end
|
|
|
|
|
2021-01-28 18:03:44 -05:00
|
|
|
def update_secure_status(source: "unknown", override: nil)
|
|
|
|
if override.nil?
|
|
|
|
mark_secure, reason = UploadSecurity.new(self).should_be_secure_with_reason
|
|
|
|
else
|
|
|
|
mark_secure = override
|
|
|
|
reason = "manually overridden"
|
|
|
|
end
|
2019-11-17 20:25:42 -05:00
|
|
|
|
2020-01-22 21:01:10 -05:00
|
|
|
secure_status_did_change = self.secure? != mark_secure
|
2021-01-28 18:03:44 -05:00
|
|
|
self.update(secure_params(mark_secure, reason, source))
|
|
|
|
|
2021-07-08 21:31:44 -04:00
|
|
|
if Discourse.store.external?
|
|
|
|
begin
|
|
|
|
Discourse.store.update_upload_ACL(self)
|
|
|
|
rescue Aws::S3::Errors::NotImplemented => err
|
|
|
|
Discourse.warn_exception(err, message: "The file store object storage provider does not support setting ACLs")
|
|
|
|
end
|
|
|
|
end
|
2020-01-22 21:01:10 -05:00
|
|
|
|
|
|
|
secure_status_did_change
|
2019-11-17 20:25:42 -05:00
|
|
|
end
|
|
|
|
|
2021-01-28 18:03:44 -05:00
|
|
|
def secure_params(secure, reason, source = "unknown")
|
|
|
|
{
|
|
|
|
secure: secure,
|
|
|
|
security_last_changed_reason: reason + " | source: #{source}",
|
|
|
|
security_last_changed_at: Time.zone.now
|
|
|
|
}
|
|
|
|
end
|
|
|
|
|
2019-04-23 23:56:48 -04:00
|
|
|
def self.migrate_to_new_scheme(limit: nil)
|
2015-06-12 06:02:36 -04:00
|
|
|
problems = []
|
|
|
|
|
2019-04-24 05:07:10 -04:00
|
|
|
DistributedMutex.synchronize("migrate_upload_to_new_scheme") do
|
|
|
|
if SiteSetting.migrate_to_new_scheme
|
|
|
|
max_file_size_kb = [
|
|
|
|
SiteSetting.max_image_size_kb,
|
|
|
|
SiteSetting.max_attachment_size_kb
|
|
|
|
].max.kilobytes
|
2015-06-12 06:02:36 -04:00
|
|
|
|
2019-04-24 05:07:10 -04:00
|
|
|
local_store = FileStore::LocalStore.new
|
|
|
|
db = RailsMultisite::ConnectionManagement.current_db
|
2019-03-14 00:38:16 -04:00
|
|
|
|
2019-04-24 05:07:10 -04:00
|
|
|
scope = Upload.by_users
|
|
|
|
.where("url NOT LIKE '%/original/_X/%' AND url LIKE '%/uploads/#{db}%'")
|
|
|
|
.order(id: :desc)
|
2019-04-24 01:59:23 -04:00
|
|
|
|
2019-04-24 05:07:10 -04:00
|
|
|
scope = scope.limit(limit) if limit
|
2019-04-24 01:59:23 -04:00
|
|
|
|
2019-04-24 05:07:10 -04:00
|
|
|
if scope.count == 0
|
|
|
|
SiteSetting.migrate_to_new_scheme = false
|
|
|
|
return problems
|
|
|
|
end
|
2019-03-28 03:58:42 -04:00
|
|
|
|
2019-04-24 05:07:10 -04:00
|
|
|
remap_scope = nil
|
|
|
|
|
|
|
|
scope.each do |upload|
|
|
|
|
begin
|
|
|
|
# keep track of the url
|
|
|
|
previous_url = upload.url.dup
|
|
|
|
# where is the file currently stored?
|
|
|
|
external = previous_url =~ /^\/\//
|
|
|
|
# download if external
|
|
|
|
if external
|
|
|
|
url = SiteSetting.scheme + ":" + previous_url
|
|
|
|
|
|
|
|
begin
|
|
|
|
retries ||= 0
|
|
|
|
|
|
|
|
file = FileHelper.download(
|
|
|
|
url,
|
|
|
|
max_file_size: max_file_size_kb,
|
|
|
|
tmp_file_name: "discourse",
|
|
|
|
follow_redirect: true
|
|
|
|
)
|
|
|
|
rescue OpenURI::HTTPError
|
2020-06-19 13:28:54 -04:00
|
|
|
retry if (retries += 1) < 1
|
2019-04-24 05:07:10 -04:00
|
|
|
next
|
|
|
|
end
|
|
|
|
|
|
|
|
path = file.path
|
|
|
|
else
|
|
|
|
path = local_store.path_for(upload)
|
|
|
|
end
|
|
|
|
# compute SHA if missing
|
|
|
|
if upload.sha1.blank?
|
|
|
|
upload.sha1 = Upload.generate_digest(path)
|
|
|
|
end
|
2019-03-26 03:07:50 -04:00
|
|
|
|
2019-04-24 05:07:10 -04:00
|
|
|
# store to new location & update the filesize
|
|
|
|
File.open(path) do |f|
|
|
|
|
upload.url = Discourse.store.store_upload(f, upload)
|
|
|
|
upload.filesize = f.size
|
|
|
|
upload.save!(validate: false)
|
|
|
|
end
|
|
|
|
# remap the URLs
|
|
|
|
DbHelper.remap(UrlHelper.absolute(previous_url), upload.url) unless external
|
|
|
|
|
|
|
|
DbHelper.remap(
|
|
|
|
previous_url,
|
|
|
|
upload.url,
|
|
|
|
excluded_tables: %w{
|
|
|
|
posts
|
|
|
|
post_search_data
|
2019-04-24 23:55:48 -04:00
|
|
|
incoming_emails
|
|
|
|
notifications
|
|
|
|
single_sign_on_records
|
|
|
|
stylesheet_cache
|
|
|
|
topic_search_data
|
|
|
|
users
|
|
|
|
user_emails
|
|
|
|
draft_sequences
|
|
|
|
optimized_images
|
2019-04-24 05:07:10 -04:00
|
|
|
}
|
|
|
|
)
|
|
|
|
|
|
|
|
remap_scope ||= begin
|
|
|
|
Post.with_deleted
|
2019-04-24 06:19:25 -04:00
|
|
|
.where("raw ~ '/uploads/#{db}/\\d+/' OR raw ~ '/uploads/#{db}/original/(\\d|[a-z])/'")
|
2019-04-25 02:26:40 -04:00
|
|
|
.select(:id, :raw, :cooked)
|
2019-04-24 05:07:10 -04:00
|
|
|
.all
|
|
|
|
end
|
2019-04-23 23:56:48 -04:00
|
|
|
|
2019-04-24 05:07:10 -04:00
|
|
|
remap_scope.each do |post|
|
|
|
|
post.raw.gsub!(previous_url, upload.url)
|
|
|
|
post.cooked.gsub!(previous_url, upload.url)
|
2019-04-25 02:26:40 -04:00
|
|
|
Post.with_deleted.where(id: post.id).update_all(raw: post.raw, cooked: post.cooked) if post.changed?
|
2019-04-24 05:07:10 -04:00
|
|
|
end
|
2019-04-02 19:38:57 -04:00
|
|
|
|
2019-04-24 05:07:10 -04:00
|
|
|
upload.optimized_images.find_each(&:destroy!)
|
|
|
|
upload.rebake_posts_on_old_scheme
|
|
|
|
# remove the old file (when local)
|
|
|
|
unless external
|
|
|
|
FileUtils.rm(path, force: true)
|
|
|
|
end
|
|
|
|
rescue => e
|
|
|
|
problems << { upload: upload, ex: e }
|
|
|
|
ensure
|
|
|
|
file&.unlink
|
|
|
|
file&.close
|
2015-06-12 06:02:36 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
problems
|
|
|
|
end
|
|
|
|
|
2019-05-28 21:00:25 -04:00
|
|
|
private
|
|
|
|
|
|
|
|
def short_url_basename
|
2019-06-18 21:10:50 -04:00
|
|
|
"#{Upload.base62_sha1(sha1)}#{extension.present? ? ".#{extension}" : ""}"
|
2019-05-28 21:00:25 -04:00
|
|
|
end
|
|
|
|
|
2013-02-05 14:16:51 -05:00
|
|
|
end
|
2013-05-23 22:48:32 -04:00
|
|
|
|
|
|
|
# == Schema Information
|
|
|
|
#
|
|
|
|
# Table name: uploads
|
|
|
|
#
|
2021-01-28 18:03:44 -05:00
|
|
|
# id :integer not null, primary key
|
|
|
|
# user_id :integer not null
|
|
|
|
# original_filename :string not null
|
|
|
|
# filesize :integer not null
|
|
|
|
# width :integer
|
|
|
|
# height :integer
|
|
|
|
# url :string not null
|
|
|
|
# created_at :datetime not null
|
|
|
|
# updated_at :datetime not null
|
|
|
|
# sha1 :string(40)
|
|
|
|
# origin :string(1000)
|
|
|
|
# retain_hours :integer
|
|
|
|
# extension :string(10)
|
|
|
|
# thumbnail_width :integer
|
|
|
|
# thumbnail_height :integer
|
|
|
|
# etag :string
|
|
|
|
# secure :boolean default(FALSE), not null
|
|
|
|
# access_control_post_id :bigint
|
|
|
|
# original_sha1 :string
|
|
|
|
# verification_status :integer default(1), not null
|
|
|
|
# animated :boolean
|
|
|
|
# security_last_changed_at :datetime
|
|
|
|
# security_last_changed_reason :string
|
2013-05-23 22:48:32 -04:00
|
|
|
#
|
|
|
|
# Indexes
|
|
|
|
#
|
2020-10-16 06:41:27 -04:00
|
|
|
# idx_uploads_on_verification_status (verification_status)
|
2020-01-15 22:50:27 -05:00
|
|
|
# index_uploads_on_access_control_post_id (access_control_post_id)
|
|
|
|
# index_uploads_on_etag (etag)
|
|
|
|
# index_uploads_on_extension (lower((extension)::text))
|
|
|
|
# index_uploads_on_id_and_url (id,url)
|
|
|
|
# index_uploads_on_original_sha1 (original_sha1)
|
|
|
|
# index_uploads_on_sha1 (sha1) UNIQUE
|
|
|
|
# index_uploads_on_url (url)
|
|
|
|
# index_uploads_on_user_id (user_id)
|
|
|
|
#
|