discourse/script/import_scripts/tnation.rb

481 lines
14 KiB
Ruby

# custom importer for www.t-nation.com, feel free to borrow ideas
require 'mysql2'
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
class ImportScripts::Tnation < ImportScripts::Base
BATCH_SIZE = 1000
# List of user custom fields that will be imported
USER_CUSTOM_FIELDS = %w{WEIGHT HEIGHT}
# Posts older than this date will *not* be imported
THRESHOLD_DATE = 6.months.ago
# Ordered list of category ids that will be imported
MIGRATED_CATEGORY_IDS = [
# Biotest Forums
255, # Micro-PA Users
236, # Biotest Supplement Advice
23, # Supplements and Nutrition
84, # Velocity Diet Support
219, # Velocity Diet Recipes
207, # Before / After Photos
206, # Diet Logs
# Training (87?, 61?)
83, # Training Logs
208, # Christian Thibaudeau Coaching
250, # Jim Wendler 5/3/1 Coaching
234, # Bigger Stronger Leaner
39, # Bodybuilding
29, # Powerlifting
229, # Figure Athletes
81, # Powerful Women
64, # Over 35 Lifter
62, # Beginners
82, # Combat
212, # Conditioning
210, # Olympic Lifting
211, # Strongman
216, # Injuries and Rehab
# Off Topic
3, # Get a Life
32, # Politics and World Issues
6, # Rate My Physique Photos
# Pharma
217, # T Replacement
40, # Steroids
]
MIGRATED_CATEGORY_IDS_SQL = MIGRATED_CATEGORY_IDS.join(",")
PARENT_CATEGORIES = ["Biotest Forums", "Training", "Off Topic", "Pharma"]
PARENT_CATEGORY_ID = {
# Biotest Forums
255 => "biotest-forums",
236 => "biotest-forums",
23 => "biotest-forums",
84 => "biotest-forums",
219 => "biotest-forums",
207 => "biotest-forums",
206 => "biotest-forums",
# Training
83 => "training",
208 => "training",
250 => "training",
234 => "training",
39 => "training",
29 => "training",
229 => "training",
81 => "training",
64 => "training",
62 => "training",
82 => "training",
212 => "training",
210 => "training",
211 => "training",
216 => "training",
# Off Topic
3 => "off-topic",
32 => "off-topic",
6 => "off-topic",
# Pharma
217 => "pharma",
40 => "pharma",
}
HIGHLIGHTED_CATEGORY_IDS = [255, 236, 23, 83, 208, 39].to_set
def initialize
super
# load existing topics
@topic_to_first_post_id = {}
PostCustomField.where(name: 'import_topic_mapping').uniq.pluck(:value).each do |m|
map = MultiJson.load(m)
@topic_to_first_post_id[map[0]] = map[1]
end
# custom site settings
SiteSetting.title = "T Nation Forums"
SiteSetting.top_menu = "categories|latest|top|unread"
SiteSetting.category_colors = "C03|A03"
SiteSetting.limit_suggested_to_category = true
SiteSetting.fixed_category_positions = true
SiteSetting.show_subcategory_list = true
SiteSetting.allow_uncategorized_topics = false
SiteSetting.uncategorized_description = nil
SiteSetting.enable_badges = false
SiteSetting.authorized_extensions = "jpg|jpeg|png|gif|svg"
SiteSetting.max_image_size_kb = 10_000.kilobytes
SiteSetting.max_attachment_size_kb = 10_000.kilobytes
end
def execute
list_imported_user_ids
import_users
import_categories
import_posts
build_topic_mapping
update_topic_views
close_locked_topics
delete_banned_users
download_avatars
# TODO? handle edits?
# TODO? mute_users (ignore_list)
# TODO? watch_category (category_subscription, notify_category)
# TODO? watch_topic (topic_subscription)
end
def list_imported_user_ids
puts "", "listing imported user_ids..."
author_ids = forum_query <<-SQL
SELECT DISTINCT(author_id)
FROM forum_message
WHERE category_id IN (#{MIGRATED_CATEGORY_IDS_SQL})
AND date >= '#{THRESHOLD_DATE}'
AND topic_id NOT IN (SELECT topicId FROM topicDelete)
SQL
@imported_user_ids_sql = author_ids.to_a.map { |d| d["author_id"] }.join(",")
end
def import_users
puts "", "importing users..."
user_count = users_query("SELECT COUNT(id) AS count FROM user WHERE id IN (#{@imported_user_ids_sql})").first["count"]
batches(BATCH_SIZE) do |offset|
users = users_query <<-SQL
SELECT id, username, email, active
FROM user
WHERE id IN (#{@imported_user_ids_sql})
ORDER BY id
LIMIT #{BATCH_SIZE}
OFFSET #{offset}
SQL
users = users.to_a
break if users.size < 1
user_bios = {}
user_avatars = {}
user_properties = {}
user_ids_sql = users.map { |u| u["id"] }.join(",")
# properties
attributes = users_query <<-SQL
SELECT userid, pkey AS "key", TRIM(COALESCE(pvalue, "")) AS "value"
FROM property
WHERE userid IN (#{user_ids_sql})
AND LENGTH(TRIM(COALESCE(pvalue, ""))) > 0
SQL
attributes.each do |a|
user_properties[a["userid"]] ||= {}
user_properties[a["userid"]][a["key"]] = a["value"]
end
# bios
bios = forum_query <<-SQL
SELECT uid, TRIM(COALESCE(quip, "")) AS "bio"
FROM myt_oneliner
WHERE uid IN (#{user_ids_sql})
AND LENGTH(TRIM(COALESCE(quip, ""))) > 0
SQL
bios.each { |b| user_bios[b["uid"]] = b["bio"] }
# avatars
avatars = forum_query <<-SQL
SELECT userid, filename
FROM forum_avatar
WHERE userid IN (#{user_ids_sql})
AND width > 0
AND height > 0
SQL
avatars.each { |a| user_avatars[a["userid"]] = a["filename"] }
# actually create users
create_users(users, total: user_count, offset: offset) do |user|
properties = user_properties[user["id"]] || {}
name = "#{properties["fname"]} #{properties["lname"]}".strip
avatar_url = forum_avatar_url(user_avatars[user["id"]]) if user_avatars.include?(user["id"])
{
id: user["id"],
name: name.presence || user["username"],
username: user["username"],
email: user["email"],
created_at: properties["join_date"].presence || properties["JOIN_DATE"].presence,
active: user["active"],
website: properties["website"],
location: properties["LOCATION"].presence || properties["city"].presence,
date_of_birth: properties["BIRTHDATE"],
bio_raw: user_bios[user["id"]],
avatar_url: avatar_url,
post_create_action: proc do |new_user|
USER_CUSTOM_FIELDS.each do |field|
new_user.custom_fields[field.downcase] = properties[field] if properties.include?(field)
end
new_user.save
end
}
end
end
end
def import_categories
puts "", "importing categories..."
position = Category.count
# create parent categories
create_categories(PARENT_CATEGORIES) do |c|
{
id: c.parameterize,
name: c,
description: c,
color: "A03",
position: position,
post_create_action: proc do position += 1; end
}
end
# children categories
categories = forum_query <<-SQL
SELECT id, name, description, is_veteran
FROM forum_category
WHERE id IN (#{MIGRATED_CATEGORY_IDS_SQL})
ORDER BY id
SQL
create_categories(categories) do |category|
name = category["name"].strip
{
id: category["id"],
name: name,
description: category["description"].strip.presence || name,
position: MIGRATED_CATEGORY_IDS.index(category["id"]) + position,
parent_category_id: category_id_from_imported_category_id(PARENT_CATEGORY_ID[category["id"]]),
read_restricted: category["is_veteran"] == 1,
color: HIGHLIGHTED_CATEGORY_IDS.include?(category["id"]) ? "C03" : "A03",
}
end
end
def import_posts
puts "", "importing posts..."
post_count = forum_query <<-SQL
SELECT COUNT(id) AS count
FROM forum_message
WHERE author_id IN (#{@imported_user_ids_sql})
AND category_id IN (#{MIGRATED_CATEGORY_IDS_SQL})
AND date >= '#{THRESHOLD_DATE}'
AND topic_id NOT IN (SELECT topicId FROM topicDelete)
AND status = 1
AND (edit_parent IS NULL OR edit_parent = -1)
AND topic_id > 0
SQL
post_count = post_count.first["count"]
batches(BATCH_SIZE) do |offset|
posts = forum_query <<-SQL
SELECT fm.id, fm.category_id, fm.topic_id, fm.date, fm.author_id, fm.subject, fm.message, ft.sticky
FROM forum_message fm
LEFT JOIN forum_topic ft ON fm.topic_id = ft.id
WHERE fm.author_id IN (#{@imported_user_ids_sql})
AND fm.category_id IN (#{MIGRATED_CATEGORY_IDS_SQL})
AND fm.date >= '#{THRESHOLD_DATE}'
AND fm.topic_id NOT IN (SELECT topicId FROM topicDelete)
AND fm.status = 1
AND (fm.edit_parent IS NULL OR fm.edit_parent = -1)
AND fm.topic_id > 0
ORDER BY fm.id
LIMIT #{BATCH_SIZE}
OFFSET #{offset}
SQL
posts = posts.to_a
break if posts.size < 1
# load images
forum_images = {}
message_ids_sql = posts.map { |p| p["id"] }.join(",")
images = forum_query <<-SQL
SELECT message_id, filename
FROM forum_image
WHERE message_id IN (#{message_ids_sql})
AND width > 0
AND height > 0
SQL
images.each do |image|
forum_images[image["message_id"]] ||= []
forum_images[image["message_id"]] << image["filename"]
end
create_posts(posts, total: post_count, offset: offset) do |post|
raw = post["message"]
if forum_images.include?(post["id"])
forum_images[post["id"]].each do |filename|
raw = forum_image_url(filename) + "\n\n" + raw
end
end
p = {
id: post["id"],
user_id: user_id_from_imported_user_id(post["author_id"]) || -1,
created_at: post["date"],
raw: raw,
custom_fields: {}
}
if @topic_to_first_post_id.include?(post["topic_id"]) && t = topic_lookup_from_imported_post_id(@topic_to_first_post_id[post["topic_id"]])
first_post_id = @topic_to_first_post_id[post["topic_id"]]
p[:topic_id] = t[:topic_id]
else
@topic_to_first_post_id[post["topic_id"]] = first_post_id = post["id"]
p[:title] = post["subject"].strip
p[:category] = category_id_from_imported_category_id(post["category_id"])
p[:pinned_at] = post["date"] if post["sticky"] == 1
end
p[:custom_fields][:import_topic_mapping] = MultiJson.dump([post["topic_id"], first_post_id])
p
end
end
end
def build_topic_mapping
puts "", "building topic mapping..."
@existing_topics = {}
PostCustomField.where(name: 'import_topic_mapping').uniq.pluck(:value).each do |m|
map = MultiJson.load(m)
@existing_topics[map[0]] = topic_lookup_from_imported_post_id(map[1])[:topic_id]
end
@topic_ids_sql = @existing_topics.keys.join(",")
end
def update_topic_views
puts "", "updating topic views..."
topic_views = forum_query("SELECT topic_id, views FROM topic_views WHERE topic_id IN (#{@topic_ids_sql}) ORDER BY topic_id").to_a
update_topic_views_sql = topic_views.map { |tv| "UPDATE topics SET views = #{tv['views']} WHERE id = #{@existing_topics[tv['topic_id']]}" }.join(";")
postgresql_query(update_topic_views_sql)
end
def close_locked_topics
puts "", "closing locked topics..."
locked_topic_ids = forum_query("SELECT id FROM forum_topic WHERE id IN (#{@topic_ids_sql}) AND locked = 1 ORDER BY id").to_a.map { |d| d["id"] }
current = 0
max = locked_topic_ids.count
locked_topic_ids.each do |id|
print_status(current += 1, max)
topic = Topic.find_by(id: @existing_topics[id])
next if topic.blank?
topic.update_status("closed", true, Discourse.system_user)
end
end
def delete_banned_users
puts "", "deleting banned users..."
user_destroyer = UserDestroyer.new(Discourse.system_user)
ids_from_banned_users = forum_query("SELECT user_id FROM banned_users WHERE user_id IN (#{@imported_user_ids_sql})").to_a.map { |d| @existing_users[d["user_id"]] }
ids_from_cookie_of_death = forum_query("SELECT userid FROM cookie_of_death WHERE userid IN (#{@imported_user_ids_sql})").to_a.map { |d| @existing_users[d["userid"]] }
banned_user_ids = (ids_from_banned_users.to_set | ids_from_cookie_of_death.to_set).to_a
current = 0
max = User.where(id: banned_user_ids).count
User.where(id: banned_user_ids.to_a).find_each do |user|
print_status(current += 1, max)
user_destroyer.destroy(user, delete_posts: true)
end
end
def download_avatars
puts "", "downloading avatars..."
current = 0
max = UserCustomField.where(name: "import_avatar_url").count
UserCustomField.where(name: "import_avatar_url").pluck(:user_id, :value).each do |user_id, avatar_url|
begin
print_status(current += 1, max)
user = User.find(user_id)
next unless user.uploaded_avatar_id.blank?
avatar = FileHelper.download(avatar_url, SiteSetting.max_image_size_kb.kilobytes, "avatar")
upload = Upload.create_for(user_id, avatar, File.basename(avatar_url), avatar.size)
if upload.persisted?
user.create_user_avatar
user.user_avatar.update(custom_upload_id: upload.id)
user.update(uploaded_avatar_id: upload.id)
end
avatar.try(:close!) rescue nil
rescue OpenURI::HTTPError, Net::ReadTimeout
end
end
end
def forum_avatar_url(filename)
"http://images.t-nation.com/avatar_images/#{filename[0]}/#{filename[1]}/#{filename}"
end
def forum_image_url(filename)
"http://images.t-nation.com/forum_images/#{filename[0]}/#{filename[1]}/#{filename}"
end
# def forum_video_url(filename)
# "http://images.t-nation.com/forum_images/forum_video/fullSize/#{filename}.flv"
# end
def forum_query(sql)
@biotest_forum ||= Mysql2::Client.new(username: "root", database: "biotest_forum")
@biotest_forum.query(sql)
end
def users_query(sql)
@biotest_users ||= Mysql2::Client.new(username: "root", database: "biotest_users")
@biotest_users.query(sql)
end
def postgresql_query(sql)
ActiveRecord::Base.connection.execute(sql)
end
end
ImportScripts::Tnation.new.perform