2014-07-02 16:50:44 -04:00
if ARGV . include? ( 'bbcode-to-md' )
# Replace (most) bbcode with markdown before creating posts.
# This will dramatically clean up the final posts in Discourse.
#
# In a temp dir:
#
2014-07-11 13:36:05 -04:00
# git clone https://github.com/nlalonde/ruby-bbcode-to-md.git
2014-07-02 16:50:44 -04:00
# cd ruby-bbcode-to-md
# gem build ruby-bbcode-to-md.gemspec
2015-05-04 17:09:58 -04:00
# gem install ruby-bbcode-to-md-*.gem
2014-07-02 16:50:44 -04:00
require 'ruby-bbcode-to-md'
end
2015-01-31 09:42:39 -05:00
require_relative '../../config/environment'
2015-05-04 17:09:58 -04:00
require_relative 'base/lookup_container'
require_relative 'base/uploader'
2015-01-23 12:19:46 -05:00
2014-05-30 15:09:58 -04:00
module ImportScripts ; end
class ImportScripts :: Base
2015-01-23 12:19:46 -05:00
include ActionView :: Helpers :: NumberHelper
2014-05-30 15:09:58 -04:00
def initialize
2014-08-13 16:17:16 -04:00
preload_i18n
2014-05-30 15:09:58 -04:00
2015-05-04 17:09:58 -04:00
@lookup = ImportScripts :: LookupContainer . new
@uploader = ImportScripts :: Uploader . new
2014-05-30 15:09:58 -04:00
2015-05-04 17:09:58 -04:00
@bbcode_to_md = true if use_bbcode_to_md?
@site_settings_during_import = { }
@old_site_settings = { }
2017-07-27 21:20:09 -04:00
@start_times = { import : Time . now }
2018-01-17 06:03:57 -05:00
@skip_updates = false
2018-06-27 14:27:11 -04:00
@next_category_color_index = { }
2014-05-30 15:09:58 -04:00
end
2014-08-13 16:17:16 -04:00
def preload_i18n
I18n . t ( " test " )
ActiveSupport :: Inflector . transliterate ( " test " )
end
2014-05-30 15:09:58 -04:00
def perform
Rails . logger . level = 3 # :error, so that we don't create log files that are many GB
2014-06-05 14:40:11 -04:00
2014-08-31 12:09:21 -04:00
change_site_settings
2014-05-30 15:09:58 -04:00
execute
2014-08-13 16:17:16 -04:00
puts " "
2018-01-17 06:03:57 -05:00
unless @skip_updates
2018-03-16 17:31:33 -04:00
update_topic_status
2018-01-17 06:03:57 -05:00
update_bumped_at
update_last_posted_at
update_last_seen_at
update_user_stats
update_feature_topic_users
update_category_featured_topics
update_topic_count_replies
reset_topic_counters
end
2014-06-05 15:30:29 -04:00
2015-05-04 17:09:58 -04:00
elapsed = Time . now - @start_times [ :import ]
2017-07-27 21:20:09 -04:00
puts '' , '' , 'Done (%02dh %02dmin %02dsec)' % [ elapsed / 3600 , elapsed / 60 % 60 , elapsed % 60 ]
2014-05-30 15:09:58 -04:00
ensure
2014-08-31 12:09:21 -04:00
reset_site_settings
end
2015-05-04 17:09:58 -04:00
def get_site_settings_for_import
{
2014-08-31 12:09:21 -04:00
email_domains_blacklist : '' ,
min_topic_title_length : 1 ,
min_post_length : 1 ,
2015-03-19 10:17:55 -04:00
min_first_post_length : 1 ,
2018-01-31 00:56:00 -05:00
min_personal_message_post_length : 1 ,
min_personal_message_title_length : 1 ,
2014-08-31 12:09:21 -04:00
allow_duplicate_topic_titles : true ,
2018-06-07 13:19:38 -04:00
disable_emails : 'yes' ,
2017-10-23 04:18:44 -04:00
max_attachment_size_kb : 102400 ,
max_image_size_kb : 102400 ,
2015-03-13 16:24:11 -04:00
authorized_extensions : '*'
2014-08-31 12:09:21 -04:00
}
2015-05-04 17:09:58 -04:00
end
def change_site_settings
2018-07-25 02:42:26 -04:00
if SiteSetting . bootstrap_mode_enabled
SiteSetting . default_trust_level = TrustLevel [ 0 ] if SiteSetting . default_trust_level == TrustLevel [ 1 ]
SiteSetting . default_email_digest_frequency = 10080 if SiteSetting . default_email_digest_frequency == 1440
SiteSetting . bootstrap_mode_enabled = false
end
2015-05-04 17:09:58 -04:00
@site_settings_during_import = get_site_settings_for_import
2014-08-31 12:09:21 -04:00
2015-03-13 16:24:11 -04:00
@site_settings_during_import . each do | key , value |
2014-08-31 12:09:21 -04:00
@old_site_settings [ key ] = SiteSetting . send ( key )
SiteSetting . set ( key , value )
end
2018-04-09 12:44:59 -04:00
# Some changes that should not be rolled back after the script is done
SiteSetting . purge_unactivated_users_grace_period_days = 60
SiteSetting . purge_deleted_uploads_grace_period_days = 90
2014-08-31 12:09:21 -04:00
RateLimiter . disable
end
def reset_site_settings
@old_site_settings . each do | key , value |
2015-03-13 16:24:11 -04:00
current_value = SiteSetting . send ( key )
SiteSetting . set ( key , value ) unless current_value != @site_settings_during_import [ key ]
2014-08-31 12:09:21 -04:00
end
2014-05-30 15:09:58 -04:00
RateLimiter . enable
end
2015-05-04 17:09:58 -04:00
def use_bbcode_to_md?
ARGV . include? ( " bbcode-to-md " )
end
2014-05-30 15:09:58 -04:00
# Implementation will do most of its work in its execute method.
# It will need to call create_users, create_categories, and create_posts.
def execute
raise NotImplementedError
end
2017-04-26 09:36:35 -04:00
% i { post_id_from_imported_post_id
topic_lookup_from_imported_post_id
group_id_from_imported_group_id
find_group_by_import_id
user_id_from_imported_user_id
find_user_by_import_id
category_id_from_imported_category_id
add_group add_user add_category add_topic add_post
} . each do | method_name |
delegate method_name , to : :@lookup
2014-05-30 15:09:58 -04:00
end
2017-07-27 21:20:09 -04:00
def create_admin ( opts = { } )
2014-05-30 15:09:58 -04:00
admin = User . new
admin . email = opts [ :email ] || " sam.saffron@gmail.com "
admin . username = opts [ :username ] || " sam "
admin . password = SecureRandom . uuid
admin . save!
admin . grant_admin!
2014-09-05 01:20:39 -04:00
admin . change_trust_level! ( TrustLevel [ 4 ] )
2014-05-30 15:09:58 -04:00
admin . email_tokens . update_all ( confirmed : true )
admin
end
2016-10-26 11:51:34 -04:00
def created_group ( group )
# override if needed
end
2014-07-16 13:59:30 -04:00
# Iterate through a list of groups to be imported.
# Takes a collection and yields to the block for each element.
# Block should return a hash with the attributes for each element.
# Required fields are :id and :name, where :id is the id of the
# group in the original datasource. The given id will not be used
# to create the Discourse group record.
2017-07-27 21:20:09 -04:00
def create_groups ( results , opts = { } )
2015-05-04 17:09:58 -04:00
created = 0
skipped = 0
failed = 0
2017-10-23 04:18:44 -04:00
total = opts [ :total ] || results . count
2014-07-16 13:59:30 -04:00
results . each do | result |
g = yield ( result )
2018-05-28 05:02:19 -04:00
if g . nil? || group_id_from_imported_group_id ( g [ :id ] )
2015-05-04 17:09:58 -04:00
skipped += 1
2014-07-16 13:59:30 -04:00
else
new_group = create_group ( g , g [ :id ] )
2016-10-26 11:51:34 -04:00
created_group ( new_group )
2014-07-16 13:59:30 -04:00
if new_group . valid?
2017-04-26 09:36:35 -04:00
add_group ( g [ :id ] . to_s , new_group )
2015-05-04 17:09:58 -04:00
created += 1
2014-07-16 13:59:30 -04:00
else
2015-05-04 17:09:58 -04:00
failed += 1
2014-07-16 13:59:30 -04:00
puts " Failed to create group id #{ g [ :id ] } #{ new_group . name } : #{ new_group . errors . full_messages } "
end
end
2018-05-28 05:02:19 -04:00
print_status ( created + skipped + failed + ( opts [ :offset ] || 0 ) , total , get_start_time ( " groups " ) )
2014-07-16 13:59:30 -04:00
end
2015-05-04 17:09:58 -04:00
[ created , skipped ]
2014-07-16 13:59:30 -04:00
end
def create_group ( opts , import_id )
2017-07-27 21:20:09 -04:00
opts = opts . dup . tap { | o | o . delete ( :id ) }
2018-05-28 05:02:19 -04:00
import_name = opts [ :name ] . presence || opts [ :full_name ]
2014-07-16 13:59:30 -04:00
opts [ :name ] = UserNameSuggester . suggest ( import_name )
2018-05-28 05:02:19 -04:00
existing = Group . find_by ( name : opts [ :name ] )
return existing if existing && existing . custom_fields [ " import_id " ] . to_s == import_id . to_s
2014-07-16 13:59:30 -04:00
g = existing || Group . new ( opts )
g . custom_fields [ " import_id " ] = import_id
g . custom_fields [ " import_name " ] = import_name
g . tap ( & :save )
end
2015-09-21 19:48:42 -04:00
def all_records_exist? ( type , import_ids )
return false if import_ids . empty?
2017-02-01 08:33:09 -05:00
connection = ActiveRecord :: Base . connection . raw_connection
connection . exec ( 'CREATE TEMP TABLE import_ids(val text PRIMARY KEY)' )
2015-10-14 22:25:10 -04:00
2016-03-07 12:21:09 -05:00
import_id_clause = import_ids . map { | id | " (' #{ PG :: Connection . escape_string ( id . to_s ) } ') " } . join ( " , " )
2016-06-14 11:44:35 -04:00
2017-02-01 08:33:09 -05:00
connection . exec ( " INSERT INTO import_ids VALUES #{ import_id_clause } " )
2015-09-21 19:48:42 -04:00
2017-02-01 08:33:09 -05:00
existing = " #{ type . to_s . classify } CustomField " . constantize
existing = existing . where ( name : 'import_id' )
2017-07-27 21:20:09 -04:00
. joins ( 'JOIN import_ids ON val = value' )
. count
2018-05-28 05:02:19 -04:00
2017-02-01 08:33:09 -05:00
if existing == import_ids . length
2015-10-14 22:25:10 -04:00
puts " Skipping #{ import_ids . length } already imported #{ type } "
return true
2015-09-21 19:48:42 -04:00
end
2015-10-14 22:25:10 -04:00
ensure
2017-07-07 11:11:43 -04:00
connection . exec ( 'DROP TABLE import_ids' ) unless connection . nil?
2015-09-21 19:48:42 -04:00
end
2016-10-26 11:51:34 -04:00
def created_user ( user )
# override if needed
end
2014-05-30 15:09:58 -04:00
# Iterate through a list of user records to be imported.
# Takes a collection, and yields to the block for each element.
# Block should return a hash with the attributes for the User model.
# Required fields are :id and :email, where :id is the id of the
# user in the original datasource. The given id will not be used to
# create the Discourse user record.
2017-07-27 21:20:09 -04:00
def create_users ( results , opts = { } )
2015-05-04 17:09:58 -04:00
created = 0
skipped = 0
failed = 0
2017-10-23 04:18:44 -04:00
total = opts [ :total ] || results . count
2014-05-30 15:09:58 -04:00
results . each do | result |
u = yield ( result )
2015-01-19 09:00:55 -05:00
# block returns nil to skip a user
2014-08-11 12:44:17 -04:00
if u . nil?
2015-05-04 17:09:58 -04:00
skipped += 1
2014-09-04 13:18:22 -04:00
else
import_id = u [ :id ]
2014-08-18 07:04:08 -04:00
2017-04-26 09:36:35 -04:00
if user_id_from_imported_user_id ( import_id )
2015-05-04 17:09:58 -04:00
skipped += 1
2014-09-04 13:18:22 -04:00
elsif u [ :email ] . present?
new_user = create_user ( u , import_id )
2016-10-26 11:51:34 -04:00
created_user ( new_user )
2014-05-30 15:09:58 -04:00
2015-12-01 09:38:21 -05:00
if new_user && new_user . valid? && new_user . user_profile && new_user . user_profile . valid?
2017-04-26 09:36:35 -04:00
add_user ( import_id . to_s , new_user )
2015-05-04 17:09:58 -04:00
created += 1
2014-09-04 13:18:22 -04:00
else
2015-05-04 17:09:58 -04:00
failed += 1
2015-12-01 09:38:21 -05:00
puts " Failed to create user id: #{ import_id } , username: #{ new_user . try ( :username ) } , email: #{ new_user . try ( :email ) } "
if new_user . try ( :errors )
puts " user errors: #{ new_user . errors . full_messages } "
if new_user . try ( :user_profile ) . try ( :errors )
puts " user_profile errors: #{ new_user . user_profile . errors . full_messages } "
end
end
2014-09-04 13:18:22 -04:00
end
2014-05-30 15:09:58 -04:00
else
2015-05-04 17:09:58 -04:00
failed += 1
2014-09-04 13:18:22 -04:00
puts " Skipping user id #{ import_id } because email is blank "
2014-05-30 15:09:58 -04:00
end
end
2018-05-28 05:02:19 -04:00
print_status ( created + skipped + failed + ( opts [ :offset ] || 0 ) , total , get_start_time ( " users " ) )
2014-05-30 15:09:58 -04:00
end
2015-05-04 17:09:58 -04:00
[ created , skipped ]
2014-05-30 15:09:58 -04:00
end
def create_user ( opts , import_id )
opts . delete ( :id )
2014-08-25 04:48:29 -04:00
merge = opts . delete ( :merge )
2014-07-16 13:59:30 -04:00
post_create_action = opts . delete ( :post_create_action )
2014-08-25 04:48:29 -04:00
2017-11-16 10:26:18 -05:00
existing = find_existing_user ( opts [ :email ] , opts [ :username ] )
2017-10-23 13:59:45 -04:00
return existing if existing && ( merge || existing . custom_fields [ " import_id " ] . to_s == import_id . to_s )
2014-05-30 15:09:58 -04:00
2014-06-10 02:07:16 -04:00
bio_raw = opts . delete ( :bio_raw )
2014-08-18 07:04:08 -04:00
website = opts . delete ( :website )
2015-01-23 12:19:46 -05:00
location = opts . delete ( :location )
2014-08-14 03:43:32 -04:00
avatar_url = opts . delete ( :avatar_url )
2016-07-27 16:30:15 -04:00
original_username = opts [ :username ]
original_name = opts [ :name ]
2017-10-23 04:18:44 -04:00
original_email = opts [ :email ] = opts [ :email ] . downcase
2016-07-27 16:30:15 -04:00
2015-10-29 16:47:27 -04:00
# Allow the || operations to work with empty strings ''
opts [ :username ] = nil if opts [ :username ] . blank?
2014-08-13 16:17:16 -04:00
opts [ :name ] = User . suggest_name ( opts [ :email ] ) unless opts [ :name ]
2016-07-27 16:30:15 -04:00
2014-08-14 12:11:28 -04:00
if opts [ :username ] . blank? ||
2015-01-23 16:44:00 -05:00
opts [ :username ] . length < User . username_length . begin ||
opts [ :username ] . length > User . username_length . end ||
2015-09-21 19:27:47 -04:00
! User . username_available? ( opts [ :username ] ) ||
! UsernameValidator . new ( opts [ :username ] ) . valid_format?
2016-02-21 15:58:47 -05:00
opts [ :username ] = UserNameSuggester . suggest ( opts [ :username ] || opts [ :name ] . presence || opts [ :email ] )
2014-08-13 16:17:16 -04:00
end
2016-07-27 16:30:15 -04:00
2017-10-23 04:18:44 -04:00
unless opts [ :email ] . match ( EmailValidator . email_regex )
opts [ :email ] = " invalid #{ SecureRandom . hex } @no-email.invalid "
puts " Invalid email #{ original_email } for #{ opts [ :username ] } . Using: #{ opts [ :email ] } "
end
2016-07-27 16:30:15 -04:00
opts [ :name ] = original_username if original_name . blank? && opts [ :username ] != original_username
2014-09-05 01:20:39 -04:00
opts [ :trust_level ] = TrustLevel [ 1 ] unless opts [ :trust_level ]
2015-01-23 16:44:00 -05:00
opts [ :active ] = opts . fetch ( :active , true )
2014-08-13 16:17:16 -04:00
opts [ :import_mode ] = true
2015-03-18 15:48:26 -04:00
opts [ :last_emailed_at ] = opts . fetch ( :last_emailed_at , Time . now )
2014-05-30 15:09:58 -04:00
u = User . new ( opts )
2015-12-01 09:38:21 -05:00
( opts [ :custom_fields ] || { } ) . each { | k , v | u . custom_fields [ k ] = v }
2014-05-30 15:09:58 -04:00
u . custom_fields [ " import_id " ] = import_id
2016-07-27 16:30:15 -04:00
u . custom_fields [ " import_username " ] = opts [ :username ] if original_username . present?
2014-08-14 03:43:32 -04:00
u . custom_fields [ " import_avatar_url " ] = avatar_url if avatar_url . present?
2015-08-20 16:15:57 -04:00
u . custom_fields [ " import_pass " ] = opts [ :password ] if opts [ :password ] . present?
2017-10-23 04:18:44 -04:00
u . custom_fields [ " import_email " ] = original_email if original_email != opts [ :email ]
2014-05-30 15:09:58 -04:00
begin
2014-06-10 02:07:16 -04:00
User . transaction do
u . save!
2015-03-07 09:42:42 -05:00
if bio_raw . present? || website . present? || location . present?
2017-12-28 08:51:43 -05:00
if website . present?
u . user_profile . website = website
u . user_profile . website = nil unless u . user_profile . valid?
end
2016-02-21 11:38:04 -05:00
u . user_profile . bio_raw = bio_raw [ 0 .. 2999 ] if bio_raw . present?
2015-01-23 12:19:46 -05:00
u . user_profile . location = location if location . present?
2014-06-10 02:07:16 -04:00
u . user_profile . save!
end
end
2015-08-20 16:15:57 -04:00
if opts [ :active ] && opts [ :password ] . present?
u . activate
end
2015-09-21 19:27:47 -04:00
rescue = > e
2014-05-30 15:09:58 -04:00
# try based on email
2017-07-24 06:52:56 -04:00
if e . try ( :record ) . try ( :errors ) . try ( :messages ) . try ( :[] , :primary_email ) . present?
2017-04-26 14:47:36 -04:00
if existing = User . find_by_email ( opts [ :email ] . downcase )
2015-09-21 19:27:47 -04:00
existing . custom_fields [ " import_id " ] = import_id
existing . save!
u = existing
end
else
2015-10-29 16:47:27 -04:00
puts " Error on record: #{ opts . inspect } "
2015-09-21 19:27:47 -04:00
raise e
2014-05-30 15:09:58 -04:00
end
end
2015-12-01 09:38:21 -05:00
2017-10-23 04:18:44 -04:00
if u . custom_fields [ 'import_email' ]
u . suspended_at = Time . zone . at ( Time . now )
u . suspended_till = 200 . years . from_now
ban_reason = 'Invalid email address on import'
2017-10-23 04:21:50 -04:00
u . active = false
2017-10-23 04:18:44 -04:00
u . save!
user_option = u . user_option
user_option . email_digests = false
user_option . email_private_messages = false
user_option . email_direct = false
user_option . email_always = false
user_option . save!
if u . save
StaffActionLogger . new ( Discourse . system_user ) . log_user_suspend ( u , ban_reason )
else
Rails . logger . error ( " Failed to suspend user #{ u . username } . #{ u . errors . try ( :full_messages ) . try ( :inspect ) } " )
end
end
2014-07-16 13:59:30 -04:00
post_create_action . try ( :call , u ) if u . persisted?
2014-05-30 15:09:58 -04:00
u # If there was an error creating the user, u.errors has the messages
end
2017-11-16 10:26:18 -05:00
def find_existing_user ( email , username )
User . joins ( :user_emails ) . where ( " user_emails.email = ? OR username = ? " , email . downcase , username ) . first
end
2016-10-26 11:51:34 -04:00
def created_category ( category )
# override if needed
end
2014-05-30 15:09:58 -04:00
# Iterates through a collection to create categories.
# The block should return a hash with attributes for the new category.
# Required fields are :id and :name, where :id is the id of the
# category in the original datasource. The given id will not be used to
# create the Discourse category record.
# Optional attributes are position, description, and parent_category_id.
def create_categories ( results )
2015-05-04 17:09:58 -04:00
created = 0
skipped = 0
2017-10-23 04:18:44 -04:00
total = results . count
2015-05-04 17:09:58 -04:00
2014-05-30 15:09:58 -04:00
results . each do | c |
params = yield ( c )
2014-09-10 14:27:18 -04:00
2015-03-12 16:15:02 -04:00
# block returns nil to skip
2017-04-26 09:36:35 -04:00
if params . nil? || category_id_from_imported_category_id ( params [ :id ] )
2015-05-04 17:09:58 -04:00
skipped += 1
else
# Basic massaging on the category name
params [ :name ] = " Blank " if params [ :name ] . blank?
params [ :name ] . strip!
params [ :name ] = params [ :name ] [ 0 .. 49 ]
# make sure categories don't go more than 2 levels deep
if params [ :parent_category_id ]
top = Category . find_by_id ( params [ :parent_category_id ] )
top = top . parent_category while top && ! top . parent_category . nil?
params [ :parent_category_id ] = top . id if top
end
2014-09-10 14:27:18 -04:00
2016-10-26 11:51:34 -04:00
new_category = create_category ( params , params [ :id ] )
created_category ( new_category )
2014-07-04 16:05:15 -04:00
2015-05-04 17:09:58 -04:00
created += 1
2014-07-04 16:05:15 -04:00
end
2018-05-28 05:02:19 -04:00
print_status ( created + skipped , total , get_start_time ( " categories " ) )
2014-05-30 15:09:58 -04:00
end
2015-05-04 17:09:58 -04:00
[ created , skipped ]
2014-05-30 15:09:58 -04:00
end
def create_category ( opts , import_id )
2015-03-12 16:15:02 -04:00
existing = Category . where ( " LOWER(name) = ? " , opts [ :name ] . downcase ) . first
return existing if existing && existing . parent_category . try ( :id ) == opts [ :parent_category_id ]
2014-05-30 15:09:58 -04:00
2014-07-16 13:59:30 -04:00
post_create_action = opts . delete ( :post_create_action )
2014-08-18 07:04:08 -04:00
2014-05-30 15:09:58 -04:00
new_category = Category . new (
name : opts [ :name ] ,
2016-07-27 12:38:23 -04:00
user_id : opts [ :user_id ] || opts [ :user ] . try ( :id ) || Discourse :: SYSTEM_USER_ID ,
2014-05-30 15:09:58 -04:00
position : opts [ :position ] ,
2015-05-19 06:40:35 -04:00
parent_category_id : opts [ :parent_category_id ] ,
2018-06-27 14:27:11 -04:00
color : opts [ :color ] || category_color ( opts [ :parent_category_id ] ) ,
2015-05-19 06:40:35 -04:00
text_color : opts [ :text_color ] || " FFF " ,
2016-07-27 12:38:23 -04:00
read_restricted : opts [ :read_restricted ] || false ,
2014-05-30 15:09:58 -04:00
)
2014-08-18 07:04:08 -04:00
2014-05-30 15:09:58 -04:00
new_category . custom_fields [ " import_id " ] = import_id if import_id
new_category . save!
2014-08-18 07:04:08 -04:00
2018-03-16 17:31:33 -04:00
if opts [ :description ] . present?
changes = { raw : opts [ :description ] }
opts = { skip_revision : true , skip_validations : true , bypass_bump : true }
new_category . topic . first_post . revise ( Discourse . system_user , changes , opts )
end
2017-04-26 09:36:35 -04:00
add_category ( import_id , new_category )
2015-12-03 10:12:06 -05:00
2014-07-16 13:59:30 -04:00
post_create_action . try ( :call , new_category )
2014-08-18 07:04:08 -04:00
2014-05-30 15:09:58 -04:00
new_category
end
2018-06-27 14:27:11 -04:00
def category_color ( parent_category_id )
2018-05-22 15:27:25 -04:00
@category_colors || = SiteSetting . category_colors . split ( '|' )
2018-06-27 14:27:11 -04:00
index = @next_category_color_index [ parent_category_id ] . presence || 0
@next_category_color_index [ parent_category_id ] = index + 1 > = @category_colors . count ? 0 : index + 1
2018-05-22 15:27:25 -04:00
@category_colors [ index ]
end
2014-10-31 00:16:08 -04:00
def created_post ( post )
# override if needed
end
2014-05-30 15:09:58 -04:00
# Iterates through a collection of posts to be imported.
# It can create topics and replies.
# Attributes will be passed to the PostCreator.
# Topics should give attributes title and category.
# Replies should provide topic_id. Use topic_lookup_from_imported_post_id to find the topic.
2017-07-27 21:20:09 -04:00
def create_posts ( results , opts = { } )
2014-05-30 15:09:58 -04:00
skipped = 0
created = 0
2017-10-23 04:18:44 -04:00
total = opts [ :total ] || results . count
2015-05-04 17:09:58 -04:00
start_time = get_start_time ( " posts- #{ total } " ) # the post count should be unique enough to differentiate between posts and PMs
2014-05-30 15:09:58 -04:00
results . each do | r |
params = yield ( r )
2014-09-04 13:18:22 -04:00
# block returns nil to skip a post
2014-05-30 15:09:58 -04:00
if params . nil?
skipped += 1
else
2014-09-04 13:18:22 -04:00
import_id = params . delete ( :id ) . to_s
2017-04-26 09:36:35 -04:00
if post_id_from_imported_post_id ( import_id )
2014-09-04 13:18:22 -04:00
skipped += 1 # already imported this post
else
begin
new_post = create_post ( params , import_id )
if new_post . is_a? ( Post )
2017-04-26 09:36:35 -04:00
add_post ( import_id , new_post )
add_topic ( new_post )
2014-09-04 13:18:22 -04:00
2014-10-31 00:16:08 -04:00
created_post ( new_post )
2014-09-04 13:18:22 -04:00
created += 1
else
skipped += 1
puts " Error creating post #{ import_id } . Skipping. "
puts new_post . inspect
end
rescue Discourse :: InvalidAccess = > e
skipped += 1
puts " InvalidAccess creating post #{ import_id } . Topic is closed? #{ e . message } "
rescue = > e
2014-07-04 16:05:15 -04:00
skipped += 1
2014-09-04 13:18:22 -04:00
puts " Exception while creating post #{ import_id } . Skipping. "
puts e . message
puts e . backtrace . join ( " \n " )
2014-07-04 16:05:15 -04:00
end
2014-05-30 15:09:58 -04:00
end
end
2015-05-04 17:09:58 -04:00
print_status ( created + skipped + ( opts [ :offset ] || 0 ) , total , start_time )
2014-05-30 15:09:58 -04:00
end
2015-05-04 17:09:58 -04:00
[ created , skipped ]
2014-05-30 15:09:58 -04:00
end
2016-07-27 12:38:23 -04:00
STAFF_GUARDIAN || = Guardian . new ( Discourse . system_user )
2015-10-14 22:25:10 -04:00
2014-06-25 19:11:52 -04:00
def create_post ( opts , import_id )
2014-05-30 15:09:58 -04:00
user = User . find ( opts [ :user_id ] )
2014-07-16 13:59:30 -04:00
post_create_action = opts . delete ( :post_create_action )
2014-05-30 15:09:58 -04:00
opts = opts . merge ( skip_validations : true )
2014-07-03 14:43:24 -04:00
opts [ :import_mode ] = true
2014-06-25 19:11:52 -04:00
opts [ :custom_fields ] || = { }
opts [ :custom_fields ] [ 'import_id' ] = import_id
2014-05-30 15:09:58 -04:00
2018-03-16 17:31:33 -04:00
unless opts [ :topic_id ]
opts [ :meta_data ] = meta_data = { }
meta_data [ " import_closed " ] = true if opts [ :closed ]
meta_data [ " import_archived " ] = true if opts [ :archived ]
meta_data [ " import_topic_id " ] = opts [ :import_topic_id ] if opts [ :import_topic_id ]
end
2015-10-14 22:25:10 -04:00
opts [ :guardian ] = STAFF_GUARDIAN
2014-07-02 16:50:44 -04:00
if @bbcode_to_md
2014-07-23 15:15:51 -04:00
opts [ :raw ] = opts [ :raw ] . bbcode_to_md ( false ) rescue opts [ :raw ]
2014-07-02 16:50:44 -04:00
end
2014-07-04 16:05:15 -04:00
post_creator = PostCreator . new ( user , opts )
post = post_creator . create
2014-07-16 13:59:30 -04:00
post_create_action . try ( :call , post ) if post
2014-07-04 16:05:15 -04:00
post ? post : post_creator . errors . full_messages
2014-05-30 15:09:58 -04:00
end
2014-07-16 13:59:30 -04:00
def create_upload ( user_id , path , source_filename )
2015-05-04 17:09:58 -04:00
@uploader . create_upload ( user_id , path , source_filename )
2014-07-16 13:59:30 -04:00
end
2015-03-13 16:24:11 -04:00
# Iterate through a list of bookmark records to be imported.
# Takes a collection, and yields to the block for each element.
# Block should return a hash with the attributes for the bookmark.
# Required fields are :user_id and :post_id, where both ids are
# the values in the original datasource.
2017-07-27 21:20:09 -04:00
def create_bookmarks ( results , opts = { } )
2015-05-04 17:09:58 -04:00
created = 0
skipped = 0
2017-10-23 04:18:44 -04:00
total = opts [ :total ] || results . count
2015-03-13 16:24:11 -04:00
user = User . new
post = Post . new
results . each do | result |
params = yield ( result )
# only the IDs are needed, so this should be enough
2015-05-04 17:09:58 -04:00
if params . nil?
skipped += 1
2015-03-13 16:24:11 -04:00
else
2017-04-26 09:36:35 -04:00
user . id = user_id_from_imported_user_id ( params [ :user_id ] )
post . id = post_id_from_imported_post_id ( params [ :post_id ] )
2015-03-13 16:24:11 -04:00
2015-05-04 17:09:58 -04:00
if user . id . nil? || post . id . nil?
skipped += 1
puts " Skipping bookmark for user id #{ params [ :user_id ] } and post id #{ params [ :post_id ] } "
else
begin
PostAction . act ( user , post , PostActionType . types [ :bookmark ] )
created += 1
rescue PostAction :: AlreadyActed
skipped += 1
end
end
2015-03-13 16:24:11 -04:00
end
2015-05-04 17:09:58 -04:00
2018-05-28 05:02:19 -04:00
print_status ( created + skipped + ( opts [ :offset ] || 0 ) , total , get_start_time ( " bookmarks " ) )
2015-03-13 16:24:11 -04:00
end
2015-05-04 17:09:58 -04:00
[ created , skipped ]
2015-03-13 16:24:11 -04:00
end
2017-07-27 21:20:09 -04:00
def close_inactive_topics ( opts = { } )
2014-06-04 10:37:43 -04:00
num_days = opts [ :days ] || 30
2015-01-23 16:44:00 -05:00
puts '' , " Closing topics that have been inactive for more than #{ num_days } days. "
2014-06-04 10:37:43 -04:00
query = Topic . where ( 'last_posted_at < ?' , num_days . days . ago ) . where ( closed : false )
total_count = query . count
closed_count = 0
query . find_each do | topic |
topic . update_status ( 'closed' , true , Discourse . system_user )
closed_count += 1
2018-05-28 05:02:19 -04:00
print_status ( closed_count , total_count , get_start_time ( " close_inactive_topics " ) )
2014-06-04 10:37:43 -04:00
end
end
2018-03-16 17:31:33 -04:00
def update_topic_status
2018-05-28 05:02:19 -04:00
puts " " , " Updating topic status "
2018-03-16 17:31:33 -04:00
2018-06-19 02:13:14 -04:00
DB . exec ( << ~ SQL )
2018-03-16 17:31:33 -04:00
UPDATE topics AS t
SET closed = TRUE
WHERE EXISTS (
SELECT 1
FROM topic_custom_fields AS f
WHERE f . topic_id = t . id AND f . name = 'import_closed' AND f . value = 't'
)
SQL
2018-06-19 02:13:14 -04:00
DB . exec ( << ~ SQL )
2018-03-16 17:31:33 -04:00
UPDATE topics AS t
SET archived = TRUE
WHERE EXISTS (
SELECT 1
FROM topic_custom_fields AS f
WHERE f . topic_id = t . id AND f . name = 'import_archived' AND f . value = 't'
)
SQL
2018-06-19 02:13:14 -04:00
DB . exec ( << ~ SQL )
2018-03-16 17:31:33 -04:00
DELETE FROM topic_custom_fields
WHERE name IN ( 'import_closed' , 'import_archived' )
SQL
end
2014-06-04 18:21:45 -04:00
def update_bumped_at
2018-05-28 05:02:19 -04:00
puts " " , " Updating bumped_at on topics "
2018-06-19 02:13:14 -04:00
DB . exec ( " update topics t set bumped_at = COALESCE((select max(created_at) from posts where topic_id = t.id and post_type = #{ Post . types [ :regular ] } ), bumped_at) " )
2014-06-04 18:21:45 -04:00
end
2014-09-08 13:36:55 -04:00
def update_last_posted_at
2018-05-28 05:02:19 -04:00
puts " " , " Updating last posted at on users "
2014-09-08 16:08:41 -04:00
sql = <<-SQL
WITH lpa AS (
SELECT user_id , MAX ( posts . created_at ) AS last_posted_at
FROM posts
GROUP BY user_id
)
UPDATE users
SET last_posted_at = lpa . last_posted_at
FROM users u1
JOIN lpa ON lpa . user_id = u1 . id
WHERE u1 . id = users . id
AND users . last_posted_at < > lpa . last_posted_at
SQL
2018-06-19 02:13:14 -04:00
DB . exec ( sql )
2014-09-08 13:36:55 -04:00
end
2016-05-17 18:38:51 -04:00
def update_user_stats
puts " " , " Updating topic reply counts... "
2016-09-20 14:27:45 -04:00
2018-05-28 05:02:19 -04:00
count = 0
total = User . real . count
2016-09-20 14:27:45 -04:00
2018-05-28 05:02:19 -04:00
User . real . find_each do | u |
2016-05-17 18:38:51 -04:00
u . create_user_stat if u . user_stat . nil?
us = u . user_stat
us . update_topic_reply_count
us . save
2018-05-28 05:02:19 -04:00
print_status ( count += 1 , total , get_start_time ( " user_stats " ) )
2016-05-17 18:38:51 -04:00
end
2018-05-28 05:02:19 -04:00
puts " " , " Updating first_post_created_at... "
2016-05-17 18:38:51 -04:00
sql = <<-SQL
WITH sub AS (
SELECT user_id , MIN ( posts . created_at ) AS first_post_created_at
FROM posts
GROUP BY user_id
)
UPDATE user_stats
SET first_post_created_at = sub . first_post_created_at
FROM user_stats u1
JOIN sub ON sub . user_id = u1 . user_id
WHERE u1 . user_id = user_stats . user_id
AND user_stats . first_post_created_at < > sub . first_post_created_at
SQL
2018-06-19 02:13:14 -04:00
DB . exec ( sql )
2016-05-17 18:38:51 -04:00
2018-05-28 05:02:19 -04:00
puts " " , " Updating user post_count... "
2016-05-17 18:38:51 -04:00
sql = <<-SQL
WITH sub AS (
SELECT user_id , COUNT ( * ) AS post_count
FROM posts
GROUP BY user_id
)
UPDATE user_stats
SET post_count = sub . post_count
FROM user_stats u1
JOIN sub ON sub . user_id = u1 . user_id
WHERE u1 . user_id = user_stats . user_id
AND user_stats . post_count < > sub . post_count
SQL
2018-06-19 02:13:14 -04:00
DB . exec ( sql )
2016-05-17 18:38:51 -04:00
2018-05-28 05:02:19 -04:00
puts " " , " Updating user topic_count... "
2016-05-17 18:38:51 -04:00
sql = <<-SQL
WITH sub AS (
SELECT user_id , COUNT ( * ) AS topic_count
FROM topics
GROUP BY user_id
)
UPDATE user_stats
SET topic_count = sub . topic_count
FROM user_stats u1
JOIN sub ON sub . user_id = u1 . user_id
WHERE u1 . user_id = user_stats . user_id
AND user_stats . topic_count < > sub . topic_count
SQL
2018-06-19 02:13:14 -04:00
DB . exec ( sql )
2016-05-17 18:38:51 -04:00
end
2015-02-12 12:24:53 -05:00
# scripts that are able to import last_seen_at from the source data should override this method
def update_last_seen_at
2018-05-28 05:02:19 -04:00
puts " " , " Updating last seen at on users "
2015-02-12 12:24:53 -05:00
2018-06-19 02:13:14 -04:00
DB . exec ( " UPDATE users SET last_seen_at = created_at WHERE last_seen_at IS NULL " )
DB . exec ( " UPDATE users SET last_seen_at = last_posted_at WHERE last_posted_at IS NOT NULL " )
2015-02-12 12:24:53 -05:00
end
2014-06-05 15:30:29 -04:00
def update_feature_topic_users
2018-05-28 05:02:19 -04:00
puts " " , " Updating featured topic users "
2014-06-05 15:30:29 -04:00
2018-05-28 05:02:19 -04:00
count = 0
total = Topic . count
2014-06-05 15:30:29 -04:00
Topic . find_each do | topic |
topic . feature_topic_users
2018-05-28 05:02:19 -04:00
print_status ( count += 1 , total , get_start_time ( " feature_topic_user " ) )
2014-06-05 15:30:29 -04:00
end
end
2014-09-04 13:08:57 -04:00
def reset_topic_counters
2018-05-28 05:02:19 -04:00
puts " " , " Resetting topic counters "
2014-09-04 13:08:57 -04:00
2018-05-28 05:02:19 -04:00
count = 0
total = Topic . count
2014-09-04 13:08:57 -04:00
Topic . find_each do | topic |
Topic . reset_highest ( topic . id )
2018-05-28 05:02:19 -04:00
print_status ( count += 1 , total , get_start_time ( " topic_counters " ) )
2014-09-04 13:08:57 -04:00
end
end
2014-07-03 14:43:24 -04:00
def update_category_featured_topics
2018-05-28 05:02:19 -04:00
puts " " , " Updating featured topics in categories "
2014-08-22 04:11:12 -04:00
2018-05-28 05:02:19 -04:00
count = 0
total = Category . count
2014-08-22 04:11:12 -04:00
2014-07-03 14:43:24 -04:00
Category . find_each do | category |
CategoryFeaturedTopic . feature_topics_for ( category )
2018-05-28 05:02:19 -04:00
print_status ( count += 1 , total , get_start_time ( " category_featured_topics " ) )
2014-07-03 14:43:24 -04:00
end
end
def update_topic_count_replies
2018-05-28 05:02:19 -04:00
puts " " , " Updating user topic reply counts "
2014-07-03 14:43:24 -04:00
2018-05-28 05:02:19 -04:00
count = 0
total = User . real . count
2014-07-03 14:43:24 -04:00
User . real . find_each do | u |
u . user_stat . update_topic_reply_count
u . user_stat . save!
2018-05-28 05:02:19 -04:00
print_status ( count += 1 , total , get_start_time ( " topic_count_replies " ) )
2014-07-03 14:43:24 -04:00
end
end
2015-02-06 16:03:41 -05:00
def update_tl0
2018-05-28 05:02:19 -04:00
puts " " , " Setting users with no posts to trust level 0 "
2015-03-20 17:05:13 -04:00
2018-05-28 05:02:19 -04:00
count = 0
total = User . count
2015-03-20 17:05:13 -04:00
2016-07-06 10:58:43 -04:00
User . includes ( :user_stat ) . find_each do | user |
2015-04-17 11:34:20 -04:00
begin
2016-07-06 10:58:43 -04:00
user . update_columns ( trust_level : 0 ) if user . trust_level > 0 && user . post_count == 0
2015-04-17 11:34:20 -04:00
rescue Discourse :: InvalidAccess
end
2018-05-28 05:02:19 -04:00
print_status ( count += 1 , total , get_start_time ( " update_tl0 " ) )
2015-02-06 16:03:41 -05:00
end
end
2015-12-03 10:12:06 -05:00
def update_user_signup_date_based_on_first_post
2018-05-28 05:02:19 -04:00
puts " " , " Setting users' signup date based on the date of their first post "
2015-12-03 10:12:06 -05:00
2018-05-28 05:02:19 -04:00
count = 0
total = User . count
2015-12-03 10:12:06 -05:00
User . find_each do | user |
2018-05-28 05:02:19 -04:00
if first = user . posts . order ( 'created_at ASC' ) . first
2015-12-03 10:12:06 -05:00
user . created_at = first . created_at
user . save!
end
2018-05-28 05:02:19 -04:00
print_status ( count += 1 , total , get_start_time ( " user_signup " ) )
2015-12-03 10:12:06 -05:00
end
end
2015-01-23 12:19:46 -05:00
def html_for_upload ( upload , display_filename )
2015-05-04 17:09:58 -04:00
@uploader . html_for_upload ( upload , display_filename )
2015-01-23 12:19:46 -05:00
end
def embedded_image_html ( upload )
2015-05-04 17:09:58 -04:00
@uploader . embedded_image_html ( upload )
2015-01-23 12:19:46 -05:00
end
def attachment_html ( upload , display_filename )
2015-05-04 17:09:58 -04:00
@uploader . attachment_html ( upload , display_filename )
2015-01-23 12:19:46 -05:00
end
2015-05-04 17:09:58 -04:00
def print_status ( current , max , start_time = nil )
if start_time . present?
elapsed_seconds = Time . now - start_time
elements_per_minute = '[%.0f items/min] ' % [ current / elapsed_seconds . to_f * 60 ]
else
elements_per_minute = ''
end
print " \r %9d / %d (%5.1f%%) %s " % [ current , max , current / max . to_f * 100 , elements_per_minute ]
2014-05-30 15:09:58 -04:00
end
2015-03-30 12:29:48 -04:00
def print_spinner
@spinner_chars || = %w{ | / - \\ }
@spinner_chars . push @spinner_chars . shift
print " \b #{ @spinner_chars [ 0 ] } "
end
2015-05-04 17:09:58 -04:00
def get_start_time ( key )
2017-07-27 21:20:09 -04:00
@start_times . fetch ( key ) { | k | @start_times [ k ] = Time . now }
2015-05-04 17:09:58 -04:00
end
2018-08-28 10:21:39 -04:00
def batches ( batch_size = 1000 )
2014-05-30 15:09:58 -04:00
offset = 0
loop do
yield offset
offset += batch_size
end
end
end