Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FIX & PERF: vanilla import 

PERF: disabled refresh_avatar callback when importing users
PERF: avoid using UsernameSuggester when not needed
FIX: categories wasn't working
FIX: posts from deleted users are now from the system user
This commit is contained in:
Régis Hanol 2014-08-13 22:17:16 +02:00
parent 6201b82a67
commit 4c4ce05964
3 changed files with 46 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
app/models/user.rb
View File

@ -67,19 +67,20 @@ class User < ActiveRecord::Base
validate :password_validator
validates :ip_address, allowed_ip_address: {on: :create, message: :signup_not_allowed}
before_save :update_username_lower
before_save :ensure_password_is_hashed
after_initialize :add_trust_level
after_initialize :set_default_email_digest
after_initialize :set_default_external_links_in_new_tab
after_save :update_tracked_topics
after_save :clear_global_notice_if_needed
after_create :create_email_token
after_create :create_user_stat
after_create :create_user_profile
after_create :ensure_in_trust_level_group
before_save :update_username_lower
before_save :ensure_password_is_hashed
after_save :update_tracked_topics
after_save :clear_global_notice_if_needed
after_save :refresh_avatar
after_save :badge_grant
@ -95,6 +96,9 @@ class User < ActiveRecord::Base
# This is just used to pass some information into the serializer
attr_accessor :notification_channel_position
# set to true to optimize creation and save for imports
attr_accessor :import_mode
scope :blocked, -> { where(blocked: true) } # no index
scope :not_blocked, -> { where(blocked: false) } # no index
scope :suspended, -> { where('suspended_till IS NOT NULL AND suspended_till > ?', Time.zone.now) } # no index
@ -594,6 +598,8 @@ class User < ActiveRecord::Base
end
def refresh_avatar
return if @import_mode
avatar = user_avatar || create_user_avatar
gravatar_downloaded = false

34
script/import_scripts/base.rb
View File

@ -17,6 +17,7 @@ class ImportScripts::Base
def initialize
require File.expand_path(File.dirname(__FILE__) + "/../../config/environment")
preload_i18n
@bbcode_to_md = true if ARGV.include?('bbcode-to-md')
@existing_groups = {}
@ -48,6 +49,11 @@ class ImportScripts::Base
end
end
def preload_i18n
I18n.t("test")
ActiveSupport::Inflector.transliterate("test")
end
def perform
Rails.logger.level = 3 # :error, so that we don't create log files that are many GB
@ -62,12 +68,14 @@ class ImportScripts::Base
execute
puts ""
update_bumped_at
update_feature_topic_users
update_category_featured_topics
update_topic_count_replies
puts '', 'Done'
puts "", "Done"
ensure
RateLimiter.enable
@ -132,8 +140,6 @@ class ImportScripts::Base
# group in the original datasource. The given id will not be used
# to create the Discourse group record.
def create_groups(results, opts={})
puts "", "creating groups"
groups_created = 0
groups_skipped = 0
total = opts[:total] || results.size
@ -182,8 +188,6 @@ class ImportScripts::Base
# user in the original datasource. The given id will not be used to
# create the Discourse user record.
def create_users(results, opts={})
puts "", "creating users"
num_users_before = User.count
users_created = 0
users_skipped = 0
@ -225,17 +229,21 @@ class ImportScripts::Base
opts.delete(:id)
post_create_action = opts.delete(:post_create_action)
existing = User.where(email: opts[:email].downcase, username: opts[:username]).first
return existing if existing and existing.custom_fields["import_id"].to_i == import_id.to_i
return existing if existing && existing.custom_fields["import_id"].to_i == import_id.to_i
bio_raw = opts.delete(:bio_raw)
opts[:name] = User.suggest_name(opts[:name]) if opts[:name]
opts[:username] = UserNameSuggester.suggest((opts[:username].present? ? opts[:username] : nil) || opts[:name] || opts[:email])
opts[:name] = User.suggest_name(opts[:email]) unless opts[:name]
if opts[:username].blank? || !User.username_available?(opts[:username])
opts[:username] = UserNameSuggester.suggest(opts[:username] || opts[:name] || opts[:email])
end
opts[:email] = opts[:email].downcase
opts[:trust_level] = TrustLevel.levels[:basic] unless opts[:trust_level]
opts[:import_mode] = true
u = User.new(opts)
u.custom_fields["import_id"] = import_id
u.custom_fields["import_username"] = opts[:username] if opts[:username].present?
u.custom_fields["import_avatar_url"] = opts[:avatar_url] if opts[:avatar_url].present?
begin
User.transaction do
@ -266,8 +274,6 @@ class ImportScripts::Base
# create the Discourse category record.
# Optional attributes are position, description, and parent_category_id.
def create_categories(results)
puts "", "creating categories"
results.each do |c|
params = yield(c)
puts " #{params[:name]}"
@ -308,8 +314,6 @@ class ImportScripts::Base
# Topics should give attributes title and category.
# Replies should provide topic_id. Use topic_lookup_from_imported_post_id to find the topic.
def create_posts(results, opts={})
puts "", "creating posts"
skipped = 0
created = 0
total = opts[:total] || results.size
@ -389,8 +393,8 @@ class ImportScripts::Base
end
def close_inactive_topics(opts={})
puts "", "Closing topics that have been inactive for more than #{num_days} days."
num_days = opts[:days] || 30
puts '', "Closing topics that have been inactive for more than #{num_days} days."
query = Topic.where('last_posted_at < ?', num_days.days.ago).where(closed: false)
total_count = query.count
@ -404,7 +408,7 @@ class ImportScripts::Base
end
def update_bumped_at
puts '', "updating bumped_at on topics"
puts "updating bumped_at on topics"
Post.exec_sql("update topics t set bumped_at = (select max(created_at) from posts where topic_id = t.id and post_type != #{Post.types[:moderator_action]})")
end
@ -422,7 +426,7 @@ class ImportScripts::Base
end
def update_category_featured_topics
puts '', "updating featured topics in categories"
puts "updating featured topics in categories"
Category.find_each do |category|
CategoryFeaturedTopic.feature_topics_for(category)
end

24
script/import_scripts/vanilla.rb
View File

@ -79,15 +79,22 @@ class ImportScripts::Vanilla < ImportScripts::Base
create_users(@users) do |user|
next if user[:name] == "[Deleted User]"
{
u = {
id: user[:user_id],
email: user[:email],
name: user[:name],
created_at: parse_date(user[:date_inserted]),
bio_raw: clean_up(user[:discovery_text]),
avatar_url: user[:photo],
moderator: @user_roles.select { |ur| ur[:user_id] == user[:user_id] }.map { |ur| ur[:role_id] }.include?(moderator_role_id),
admin: @user_roles.select { |ur| ur[:user_id] == user[:user_id] }.map { |ur| ur[:role_id] }.include?(admin_role_id),
}
# if @comments.select { |c| c[:insert_user_id] == user[:user_id] }.map { |c| c[:discussion_id] }.uniq.count > 3
# u[:trust_level] = TrustLevel.levels[:regular]
# end
u
end
end
@ -118,14 +125,14 @@ class ImportScripts::Vanilla < ImportScripts::Base
c = {
id: category[:category_id],
name: category[:name],
user_id: user_id_from_imported_user_id(category[:insert_user_id]),
user_id: user_id_from_imported_user_id(category[:insert_user_id]) || Discourse::SYSTEM_USER_ID,
position: category[:sort].to_i,
created_at: parse_category_date(category[:date_inserted]),
description: clean_up(category[:description]),
}
if category[:parent_category_id] != "-1"
parent_category = category_from_imported_category_id(category[:parent_category_id].to_i)
c[:parent_category_id] = parent_category[:id] if parent_category
parent_category = category_from_imported_category_id(category[:parent_category_id])
c[:parent_category_id] = parent_category.id if parent_category
end
c
end
@ -140,9 +147,9 @@ class ImportScripts::Vanilla < ImportScripts::Base
create_posts(@discussions) do |discussion|
{
id: "discussion#" + discussion[:discussion_id],
user_id: user_id_from_imported_user_id(discussion[:insert_user_id]),
user_id: user_id_from_imported_user_id(discussion[:insert_user_id]) || Discourse::SYSTEM_USER_ID,
title: discussion[:name],
category_id: category_from_imported_category_id(discussion[:category_id]).try(:id),
category: category_from_imported_category_id(discussion[:category_id]).try(:name),
raw: clean_up(discussion[:body]),
created_at: parse_date(discussion[:date_inserted]),
}
@ -157,7 +164,7 @@ class ImportScripts::Vanilla < ImportScripts::Base
{
id: "comment#" + comment[:comment_id],
user_id: user_id_from_imported_user_id(comment[:insert_user_id]),
user_id: user_id_from_imported_user_id(comment[:insert_user_id]) || Discourse::SYSTEM_USER_ID,
topic_id: t[:topic_id],
raw: clean_up(comment[:body]),
created_at: parse_date(comment[:date_inserted]),
@ -207,7 +214,7 @@ class ImportScripts::Vanilla < ImportScripts::Base
{
archetype: Archetype.private_message,
id: "message#" + message[:message_id],
user_id: user_id_from_imported_user_id(message[:insert_user_id]),
user_id: user_id_from_imported_user_id(message[:insert_user_id]) || Discourse::SYSTEM_USER_ID,
topic_id: t[:topic_id],
raw: clean_up(message[:body]),
created_at: parse_date(message[:date_inserted]),
@ -225,6 +232,7 @@ class ImportScripts::Vanilla < ImportScripts::Base
.gsub(/<\/?code\s*>/i, "`")
.gsub("&lt;", "<")
.gsub("&gt;", ">")
# .gsub("*", "\\*")
end
end