DEV: speed up posts base imports
This commit is contained in:
parent
95e5f8380d
commit
788719d271
|
@ -174,15 +174,14 @@ class PostCreator
|
|||
update_user_counts
|
||||
create_embedded_topic
|
||||
link_post_uploads
|
||||
|
||||
ensure_in_allowed_users if guardian.is_staff?
|
||||
unarchive_message
|
||||
@post.advance_draft_sequence
|
||||
@post.advance_draft_sequence unless @opts[:import_mode]
|
||||
@post.save_reply_relationships
|
||||
end
|
||||
end
|
||||
|
||||
if @post && errors.blank?
|
||||
if @post && errors.blank? && !@opts[:import_mode]
|
||||
# update counters etc.
|
||||
@post.topic.reload
|
||||
|
||||
|
@ -194,12 +193,10 @@ class PostCreator
|
|||
|
||||
trigger_after_events unless opts[:skip_events]
|
||||
|
||||
auto_close unless @opts[:import_mode]
|
||||
auto_close
|
||||
end
|
||||
|
||||
if @post || @spam
|
||||
handle_spam unless @opts[:import_mode]
|
||||
end
|
||||
handle_spam if !opts[:import_mode] && (@post || @spam)
|
||||
|
||||
@post
|
||||
end
|
||||
|
@ -428,6 +425,8 @@ class PostCreator
|
|||
end
|
||||
|
||||
def update_topic_auto_close
|
||||
return if @opts[:import_mode]
|
||||
|
||||
if @topic.closed?
|
||||
@topic.delete_topic_timer(TopicTimer.types[:close])
|
||||
else
|
||||
|
@ -510,9 +509,7 @@ class PostCreator
|
|||
end
|
||||
|
||||
def publish
|
||||
return if @opts[:import_mode]
|
||||
return unless @post.post_number > 1
|
||||
|
||||
return if @opts[:import_mode] || @post.post_number == 1
|
||||
@post.publish_change_to_clients! :created
|
||||
end
|
||||
|
||||
|
@ -522,7 +519,7 @@ class PostCreator
|
|||
end
|
||||
|
||||
def track_topic
|
||||
return if @opts[:auto_track] == false
|
||||
return if @opts[:import_mode] || @opts[:auto_track] == false
|
||||
|
||||
unless @user.user_option.disable_jump_reply?
|
||||
TopicUser.change(@post.user_id,
|
||||
|
@ -540,8 +537,7 @@ class PostCreator
|
|||
|
||||
if @user.staged
|
||||
TopicUser.auto_notification_for_staging(@user.id, @topic.id, TopicUser.notification_reasons[:auto_watch])
|
||||
else
|
||||
return if @topic.private_message?
|
||||
elsif !@topic.private_message?
|
||||
notification_level = @user.user_option.notification_level_when_replying || NotificationLevels.topic_levels[:tracking]
|
||||
TopicUser.auto_notification(@user.id, @topic.id, TopicUser.notification_reasons[:created_post], notification_level)
|
||||
end
|
||||
|
|
|
@ -80,8 +80,8 @@ class UploadCreator
|
|||
end
|
||||
|
||||
fixed_original_filename = nil
|
||||
if is_image
|
||||
|
||||
if is_image
|
||||
current_extension = File.extname(@filename).downcase.sub("jpeg", "jpg")
|
||||
expected_extension = ".#{image_type}".downcase.sub("jpeg", "jpg")
|
||||
|
||||
|
@ -89,11 +89,7 @@ class UploadCreator
|
|||
# otherwise validation will fail and we can not save
|
||||
# TODO decide if we only run the validation on the extension
|
||||
if current_extension != expected_extension
|
||||
basename = File.basename(@filename, current_extension)
|
||||
|
||||
if basename.length == 0
|
||||
basename = "image"
|
||||
end
|
||||
basename = File.basename(@filename, current_extension).presence || "image"
|
||||
fixed_original_filename = "#{basename}#{expected_extension}"
|
||||
end
|
||||
end
|
||||
|
@ -173,10 +169,7 @@ class UploadCreator
|
|||
MIN_CONVERT_TO_JPEG_SAVING_RATIO = 0.70
|
||||
|
||||
def convert_to_jpeg!
|
||||
|
||||
if filesize < MIN_CONVERT_TO_JPEG_BYTES_SAVED
|
||||
return
|
||||
end
|
||||
return if filesize < MIN_CONVERT_TO_JPEG_BYTES_SAVED
|
||||
|
||||
jpeg_tempfile = Tempfile.new(["image", ".jpg"])
|
||||
|
||||
|
@ -290,7 +283,6 @@ class UploadCreator
|
|||
|
||||
def crop!
|
||||
max_pixel_ratio = Discourse::PIXEL_RATIOS.max
|
||||
|
||||
filename_with_correct_ext = "image.#{@image_info.type}"
|
||||
|
||||
case @opts[:type]
|
||||
|
|
|
@ -54,6 +54,8 @@ class ImportScripts::Base
|
|||
update_last_posted_at
|
||||
update_last_seen_at
|
||||
update_user_stats
|
||||
update_topic_users
|
||||
update_post_timings
|
||||
update_feature_topic_users
|
||||
update_category_featured_topics
|
||||
update_topic_count_replies
|
||||
|
@ -313,7 +315,7 @@ class ImportScripts::Base
|
|||
|
||||
unless opts[:email][EmailValidator.email_regex]
|
||||
opts[:email] = fake_email
|
||||
puts "Invalid email #{original_email} for #{opts[:username]}. Using: #{opts[:email]}"
|
||||
puts "Invalid email '#{original_email}' for '#{opts[:username]}'. Using '#{opts[:email]}'"
|
||||
end
|
||||
|
||||
opts[:name] = original_username if original_name.blank? && opts[:username] != original_username
|
||||
|
@ -326,7 +328,7 @@ class ImportScripts::Base
|
|||
u = User.new(opts)
|
||||
(opts[:custom_fields] || {}).each { |k, v| u.custom_fields[k] = v }
|
||||
u.custom_fields["import_id"] = import_id
|
||||
u.custom_fields["import_username"] = opts[:username] if original_username.present?
|
||||
u.custom_fields["import_username"] = original_username if original_username.present? && original_username != opts[:username]
|
||||
u.custom_fields["import_avatar_url"] = avatar_url if avatar_url.present?
|
||||
u.custom_fields["import_pass"] = opts[:password] if opts[:password].present?
|
||||
u.custom_fields["import_email"] = original_email if original_email != opts[:email]
|
||||
|
@ -506,21 +508,19 @@ class ImportScripts::Base
|
|||
import_id = params.delete(:id).to_s
|
||||
|
||||
if post_id_from_imported_post_id(import_id)
|
||||
skipped += 1 # already imported this post
|
||||
skipped += 1
|
||||
else
|
||||
begin
|
||||
new_post = create_post(params, import_id)
|
||||
if new_post.is_a?(Post)
|
||||
add_post(import_id, new_post)
|
||||
add_topic(new_post)
|
||||
|
||||
created_post(new_post)
|
||||
|
||||
created += 1
|
||||
else
|
||||
skipped += 1
|
||||
puts "Error creating post #{import_id}. Skipping."
|
||||
puts new_post.inspect
|
||||
p new_post
|
||||
end
|
||||
rescue Discourse::InvalidAccess => e
|
||||
skipped += 1
|
||||
|
@ -632,7 +632,7 @@ class ImportScripts::Base
|
|||
def update_topic_status
|
||||
puts "", "Updating topic status"
|
||||
|
||||
DB.exec(<<~SQL)
|
||||
DB.exec <<~SQL
|
||||
UPDATE topics AS t
|
||||
SET closed = TRUE
|
||||
WHERE EXISTS(
|
||||
|
@ -642,7 +642,7 @@ class ImportScripts::Base
|
|||
)
|
||||
SQL
|
||||
|
||||
DB.exec(<<~SQL)
|
||||
DB.exec <<~SQL
|
||||
UPDATE topics AS t
|
||||
SET archived = TRUE
|
||||
WHERE EXISTS(
|
||||
|
@ -652,7 +652,7 @@ class ImportScripts::Base
|
|||
)
|
||||
SQL
|
||||
|
||||
DB.exec(<<~SQL)
|
||||
DB.exec <<~SQL
|
||||
DELETE FROM topic_custom_fields
|
||||
WHERE name IN ('import_closed', 'import_archived')
|
||||
SQL
|
||||
|
@ -660,13 +660,16 @@ class ImportScripts::Base
|
|||
|
||||
def update_bumped_at
|
||||
puts "", "Updating bumped_at on topics"
|
||||
DB.exec("update topics t set bumped_at = COALESCE((select max(created_at) from posts where topic_id = t.id and post_type = #{Post.types[:regular]}), bumped_at)")
|
||||
DB.exec <<~SQL
|
||||
UPDATE topics t
|
||||
SET bumped_at = COALESCE((SELECT MAX(created_at) FROM posts WHERE topic_id = t.id AND post_type = #{Post.types[:regular]}), bumped_at)
|
||||
SQL
|
||||
end
|
||||
|
||||
def update_last_posted_at
|
||||
puts "", "Updating last posted at on users"
|
||||
|
||||
sql = <<-SQL
|
||||
DB.exec <<~SQL
|
||||
WITH lpa AS (
|
||||
SELECT user_id, MAX(posts.created_at) AS last_posted_at
|
||||
FROM posts
|
||||
|
@ -679,8 +682,6 @@ class ImportScripts::Base
|
|||
WHERE u1.id = users.id
|
||||
AND users.last_posted_at <> lpa.last_posted_at
|
||||
SQL
|
||||
|
||||
DB.exec(sql)
|
||||
end
|
||||
|
||||
def update_user_stats
|
||||
|
@ -699,7 +700,7 @@ class ImportScripts::Base
|
|||
|
||||
puts "", "Updating first_post_created_at..."
|
||||
|
||||
sql = <<-SQL
|
||||
DB.exec <<~SQL
|
||||
WITH sub AS (
|
||||
SELECT user_id, MIN(posts.created_at) AS first_post_created_at
|
||||
FROM posts
|
||||
|
@ -713,11 +714,9 @@ class ImportScripts::Base
|
|||
AND user_stats.first_post_created_at <> sub.first_post_created_at
|
||||
SQL
|
||||
|
||||
DB.exec(sql)
|
||||
|
||||
puts "", "Updating user post_count..."
|
||||
|
||||
sql = <<-SQL
|
||||
DB.exec <<~SQL
|
||||
WITH sub AS (
|
||||
SELECT user_id, COUNT(*) AS post_count
|
||||
FROM posts
|
||||
|
@ -731,11 +730,9 @@ class ImportScripts::Base
|
|||
AND user_stats.post_count <> sub.post_count
|
||||
SQL
|
||||
|
||||
DB.exec(sql)
|
||||
|
||||
puts "", "Updating user topic_count..."
|
||||
|
||||
sql = <<-SQL
|
||||
DB.exec <<~SQL
|
||||
WITH sub AS (
|
||||
SELECT user_id, COUNT(*) AS topic_count
|
||||
FROM topics
|
||||
|
@ -748,8 +745,6 @@ class ImportScripts::Base
|
|||
WHERE u1.user_id = user_stats.user_id
|
||||
AND user_stats.topic_count <> sub.topic_count
|
||||
SQL
|
||||
|
||||
DB.exec(sql)
|
||||
end
|
||||
|
||||
# scripts that are able to import last_seen_at from the source data should override this method
|
||||
|
@ -760,6 +755,31 @@ class ImportScripts::Base
|
|||
DB.exec("UPDATE users SET last_seen_at = last_posted_at WHERE last_posted_at IS NOT NULL")
|
||||
end
|
||||
|
||||
def update_topic_users
|
||||
puts "", "Updating topic users"
|
||||
|
||||
DB.exec <<~SQL
|
||||
INSERT INTO topic_users (user_id, topic_id, posted, last_read_post_number, highest_seen_post_number, first_visited_at, last_visited_at, total_msecs_viewed)
|
||||
SELECT user_id, topic_id, 't' , MAX(post_number), MAX(post_number), MIN(created_at), MAX(created_at), COUNT(id) * 5000
|
||||
FROM posts
|
||||
WHERE user_id > 0
|
||||
GROUP BY user_id, topic_id
|
||||
ON CONFLICT DO NOTHING
|
||||
SQL
|
||||
end
|
||||
|
||||
def update_post_timings
|
||||
puts "", "Updating post timings"
|
||||
|
||||
DB.exec <<~SQL
|
||||
INSERT INTO post_timings (topic_id, post_number, user_id, msecs)
|
||||
SELECT topic_id, post_number, user_id, 5000
|
||||
FROM posts
|
||||
WHERE user_id > 0
|
||||
ON CONFLICT DO NOTHING
|
||||
SQL
|
||||
end
|
||||
|
||||
def update_feature_topic_users
|
||||
puts "", "Updating featured topic users"
|
||||
|
||||
|
|
Loading…
Reference in New Issue