Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DEV: speed up posts base imports

This commit is contained in:
Régis Hanol 2019-01-04 15:30:17 +01:00
parent 95e5f8380d
commit 788719d271
3 changed files with 54 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
lib/post_creator.rb
View File

@ -174,15 +174,14 @@ class PostCreator
update_user_counts
create_embedded_topic
link_post_uploads
ensure_in_allowed_users if guardian.is_staff?
unarchive_message
@post.advance_draft_sequence
@post.advance_draft_sequence unless @opts[:import_mode]
@post.save_reply_relationships
end
end
if @post && errors.blank?
if @post && errors.blank? && !@opts[:import_mode]
# update counters etc.
@post.topic.reload
@ -194,12 +193,10 @@ class PostCreator
trigger_after_events unless opts[:skip_events]
auto_close unless @opts[:import_mode]
auto_close
end
if @post || @spam
handle_spam unless @opts[:import_mode]
end
handle_spam if !opts[:import_mode] && (@post || @spam)
@post
end
@ -428,6 +425,8 @@ class PostCreator
end
def update_topic_auto_close
return if @opts[:import_mode]
if @topic.closed?
@topic.delete_topic_timer(TopicTimer.types[:close])
else
@ -510,9 +509,7 @@ class PostCreator
end
def publish
return if @opts[:import_mode]
return unless @post.post_number > 1
return if @opts[:import_mode] || @post.post_number == 1
@post.publish_change_to_clients! :created
end
@ -522,7 +519,7 @@ class PostCreator
end
def track_topic
return if @opts[:auto_track] == false
return if @opts[:import_mode] || @opts[:auto_track] == false
unless @user.user_option.disable_jump_reply?
TopicUser.change(@post.user_id,
@ -540,8 +537,7 @@ class PostCreator
if @user.staged
TopicUser.auto_notification_for_staging(@user.id, @topic.id, TopicUser.notification_reasons[:auto_watch])
else
return if @topic.private_message?
elsif !@topic.private_message?
notification_level = @user.user_option.notification_level_when_replying || NotificationLevels.topic_levels[:tracking]
TopicUser.auto_notification(@user.id, @topic.id, TopicUser.notification_reasons[:created_post], notification_level)
end

14
lib/upload_creator.rb
View File

@ -80,8 +80,8 @@ class UploadCreator
end
fixed_original_filename = nil
if is_image
if is_image
current_extension = File.extname(@filename).downcase.sub("jpeg", "jpg")
expected_extension = ".#{image_type}".downcase.sub("jpeg", "jpg")
@ -89,11 +89,7 @@ class UploadCreator
# otherwise validation will fail and we can not save
# TODO decide if we only run the validation on the extension
if current_extension != expected_extension
basename = File.basename(@filename, current_extension)
if basename.length == 0
basename = "image"
end
basename = File.basename(@filename, current_extension).presence || "image"
fixed_original_filename = "#{basename}#{expected_extension}"
end
end
@ -173,10 +169,7 @@ class UploadCreator
MIN_CONVERT_TO_JPEG_SAVING_RATIO = 0.70
def convert_to_jpeg!
if filesize < MIN_CONVERT_TO_JPEG_BYTES_SAVED
return
end
return if filesize < MIN_CONVERT_TO_JPEG_BYTES_SAVED
jpeg_tempfile = Tempfile.new(["image", ".jpg"])
@ -290,7 +283,6 @@ class UploadCreator
def crop!
max_pixel_ratio = Discourse::PIXEL_RATIOS.max
filename_with_correct_ext = "image.#{@image_info.type}"
case @opts[:type]

64
script/import_scripts/base.rb
View File

@ -54,6 +54,8 @@ class ImportScripts::Base
update_last_posted_at
update_last_seen_at
update_user_stats
update_topic_users
update_post_timings
update_feature_topic_users
update_category_featured_topics
update_topic_count_replies
@ -313,7 +315,7 @@ class ImportScripts::Base
unless opts[:email][EmailValidator.email_regex]
opts[:email] = fake_email
puts "Invalid email #{original_email} for #{opts[:username]}. Using: #{opts[:email]}"
puts "Invalid email '#{original_email}' for '#{opts[:username]}'. Using '#{opts[:email]}'"
end
opts[:name] = original_username if original_name.blank? && opts[:username] != original_username
@ -326,7 +328,7 @@ class ImportScripts::Base
u = User.new(opts)
(opts[:custom_fields] || {}).each { |k, v| u.custom_fields[k] = v }
u.custom_fields["import_id"] = import_id
u.custom_fields["import_username"] = opts[:username] if original_username.present?
u.custom_fields["import_username"] = original_username if original_username.present? && original_username != opts[:username]
u.custom_fields["import_avatar_url"] = avatar_url if avatar_url.present?
u.custom_fields["import_pass"] = opts[:password] if opts[:password].present?
u.custom_fields["import_email"] = original_email if original_email != opts[:email]
@ -506,21 +508,19 @@ class ImportScripts::Base
import_id = params.delete(:id).to_s
if post_id_from_imported_post_id(import_id)
skipped += 1 # already imported this post
skipped += 1
else
begin
new_post = create_post(params, import_id)
if new_post.is_a?(Post)
add_post(import_id, new_post)
add_topic(new_post)
created_post(new_post)
created += 1
else
skipped += 1
puts "Error creating post #{import_id}. Skipping."
puts new_post.inspect
p new_post
end
rescue Discourse::InvalidAccess => e
skipped += 1
@ -632,7 +632,7 @@ class ImportScripts::Base
def update_topic_status
puts "", "Updating topic status"
DB.exec(<<~SQL)
DB.exec <<~SQL
UPDATE topics AS t
SET closed = TRUE
WHERE EXISTS(
@ -642,7 +642,7 @@ class ImportScripts::Base
)
SQL
DB.exec(<<~SQL)
DB.exec <<~SQL
UPDATE topics AS t
SET archived = TRUE
WHERE EXISTS(
@ -652,7 +652,7 @@ class ImportScripts::Base
)
SQL
DB.exec(<<~SQL)
DB.exec <<~SQL
DELETE FROM topic_custom_fields
WHERE name IN ('import_closed', 'import_archived')
SQL
@ -660,13 +660,16 @@ class ImportScripts::Base
def update_bumped_at
puts "", "Updating bumped_at on topics"
DB.exec("update topics t set bumped_at = COALESCE((select max(created_at) from posts where topic_id = t.id and post_type = #{Post.types[:regular]}), bumped_at)")
DB.exec <<~SQL
UPDATE topics t
SET bumped_at = COALESCE((SELECT MAX(created_at) FROM posts WHERE topic_id = t.id AND post_type = #{Post.types[:regular]}), bumped_at)
SQL
end
def update_last_posted_at
puts "", "Updating last posted at on users"
sql = <<-SQL
DB.exec <<~SQL
WITH lpa AS (
SELECT user_id, MAX(posts.created_at) AS last_posted_at
FROM posts
@ -679,8 +682,6 @@ class ImportScripts::Base
WHERE u1.id = users.id
AND users.last_posted_at <> lpa.last_posted_at
SQL
DB.exec(sql)
end
def update_user_stats
@ -699,7 +700,7 @@ class ImportScripts::Base
puts "", "Updating first_post_created_at..."
sql = <<-SQL
DB.exec <<~SQL
WITH sub AS (
SELECT user_id, MIN(posts.created_at) AS first_post_created_at
FROM posts
@ -713,11 +714,9 @@ class ImportScripts::Base
AND user_stats.first_post_created_at <> sub.first_post_created_at
SQL
DB.exec(sql)
puts "", "Updating user post_count..."
sql = <<-SQL
DB.exec <<~SQL
WITH sub AS (
SELECT user_id, COUNT(*) AS post_count
FROM posts
@ -731,11 +730,9 @@ class ImportScripts::Base
AND user_stats.post_count <> sub.post_count
SQL
DB.exec(sql)
puts "", "Updating user topic_count..."
sql = <<-SQL
DB.exec <<~SQL
WITH sub AS (
SELECT user_id, COUNT(*) AS topic_count
FROM topics
@ -748,8 +745,6 @@ class ImportScripts::Base
WHERE u1.user_id = user_stats.user_id
AND user_stats.topic_count <> sub.topic_count
SQL
DB.exec(sql)
end
# scripts that are able to import last_seen_at from the source data should override this method
@ -760,6 +755,31 @@ class ImportScripts::Base
DB.exec("UPDATE users SET last_seen_at = last_posted_at WHERE last_posted_at IS NOT NULL")
end
def update_topic_users
puts "", "Updating topic users"
DB.exec <<~SQL
INSERT INTO topic_users (user_id, topic_id, posted, last_read_post_number, highest_seen_post_number, first_visited_at, last_visited_at, total_msecs_viewed)
SELECT user_id, topic_id, 't' , MAX(post_number), MAX(post_number), MIN(created_at), MAX(created_at), COUNT(id) * 5000
FROM posts
WHERE user_id > 0
GROUP BY user_id, topic_id
ON CONFLICT DO NOTHING
SQL
end
def update_post_timings
puts "", "Updating post timings"
DB.exec <<~SQL
INSERT INTO post_timings (topic_id, post_number, user_id, msecs)
SELECT topic_id, post_number, user_id, 5000
FROM posts
WHERE user_id > 0
ON CONFLICT DO NOTHING
SQL
end
def update_feature_topic_users
puts "", "Updating featured topic users"