discourse/script/import_scripts/bbpress.rb

275 lines
7.8 KiB
Ruby
Raw Normal View History

require 'mysql2'
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
2016-09-26 18:03:42 -04:00
# Before running this script, paste these lines into your shell,
# then use arrow keys to edit the values
=begin
export BBPRESS_HOST="localhost"
2016-09-26 18:03:42 -04:00
export BBPRESS_USER="root"
export BBPRESS_DB="bbpress"
export BBPRESS_PW=""
=end
class ImportScripts::Bbpress < ImportScripts::Base
BB_PRESS_HOST ||= ENV['BBPRESS_HOST'] || "localhost"
2016-03-07 12:21:09 -05:00
BB_PRESS_DB ||= ENV['BBPRESS_DB'] || "bbpress"
BATCH_SIZE ||= 1000
2016-09-26 18:03:42 -04:00
BB_PRESS_PW ||= ENV['BBPRESS_PW'] || ""
BB_PRESS_USER ||= ENV['BBPRESS_USER'] || "root"
BB_PRESS_PREFIX ||= ENV['BBPRESS_PREFIX'] || "wp_"
def initialize
super
@client = Mysql2::Client.new(
host: BB_PRESS_HOST,
2016-09-26 18:03:42 -04:00
username: BB_PRESS_USER,
2016-03-07 12:21:09 -05:00
database: BB_PRESS_DB,
2016-09-26 18:03:42 -04:00
password: BB_PRESS_PW,
)
end
2016-03-07 12:21:09 -05:00
def execute
import_users
import_anonymous_users
2016-03-07 12:21:09 -05:00
import_categories
import_topics_and_posts
end
2016-03-07 12:21:09 -05:00
def import_users
puts "", "importing users..."
last_user_id = -1
total_users = bbpress_query("SELECT COUNT(*) count FROM #{BB_PRESS_PREFIX}users WHERE user_email LIKE '%@%'").first["count"]
2016-03-07 12:21:09 -05:00
batches(BATCH_SIZE) do |offset|
users = bbpress_query(<<-SQL
SELECT id, user_nicename, display_name, user_email, user_registered, user_url
FROM #{BB_PRESS_PREFIX}users
2016-03-07 12:21:09 -05:00
WHERE user_email LIKE '%@%'
AND id > #{last_user_id}
ORDER BY id
LIMIT #{BATCH_SIZE}
SQL
).to_a
break if users.empty?
last_user_id = users[-1]["id"]
user_ids = users.map { |u| u["id"].to_i }
next if all_records_exist?(:users, user_ids)
user_ids_sql = user_ids.join(",")
users_description = {}
bbpress_query(<<-SQL
SELECT user_id, meta_value description
FROM #{BB_PRESS_PREFIX}usermeta
2016-03-07 12:21:09 -05:00
WHERE user_id IN (#{user_ids_sql})
AND meta_key = 'description'
SQL
).each { |um| users_description[um["user_id"]] = um["description"] }
users_last_activity = {}
bbpress_query(<<-SQL
SELECT user_id, meta_value last_activity
FROM #{BB_PRESS_PREFIX}usermeta
2016-03-07 12:21:09 -05:00
WHERE user_id IN (#{user_ids_sql})
AND meta_key = 'last_activity'
SQL
).each { |um| users_last_activity[um["user_id"]] = um["last_activity"] }
create_users(users, total: total_users, offset: offset) do |u|
{
id: u["id"].to_i,
username: u["user_nicename"],
email: u["user_email"].downcase,
2017-01-11 15:55:25 -05:00
name: u["display_name"].presence || u['user_nicename'],
2016-03-07 12:21:09 -05:00
created_at: u["user_registered"],
website: u["user_url"],
bio_raw: users_description[u["id"]],
last_seen_at: users_last_activity[u["id"]],
}
end
end
2016-03-07 12:21:09 -05:00
end
def import_anonymous_users
puts "", "importing anonymous users..."
anon_posts = Hash.new
anon_names = Hash.new
emails = Array.new
# gather anonymous users via postmeta table
bbpress_query(<<-SQL
SELECT post_id, meta_key, meta_value
FROM #{BB_PRESS_PREFIX}postmeta
WHERE meta_key LIKE '_bbp_anonymous%'
SQL
).each do |pm|
anon_posts[pm['post_id']] = Hash.new if not anon_posts[pm['post_id']]
if pm['meta_key'] == '_bbp_anonymous_email'
anon_posts[pm['post_id']]['email'] = pm['meta_value']
end
if pm['meta_key'] == '_bbp_anonymous_name'
anon_posts[pm['post_id']]['name'] = pm['meta_value']
end
if pm['meta_key'] == '_bbp_anonymous_website'
anon_posts[pm['post_id']]['website'] = pm['meta_value']
end
end
# gather every existent username
anon_posts.each do |id,post|
anon_names[post['name']] = Hash.new if not anon_names[post['name']]
# overwriting email address, one user can only use one email address
anon_names[post['name']]['email'] = post['email']
anon_names[post['name']]['website'] = post['website'] if post['website'] != ''
end
# make sure every user name has a unique email address
anon_names.each do |k,name|
if not emails.include? name['email']
emails.push ( name['email'])
else
name['email'] = "anonymous_#{SecureRandom.hex}@no-email.invalid"
end
end
create_users(anon_names) do |k, n|
{
id: k,
email: n["email"].downcase,
name: k,
website: n["website"]
}
end
end
2016-03-07 12:21:09 -05:00
def import_categories
puts "", "importing categories..."
2016-03-07 12:21:09 -05:00
categories = bbpress_query(<<-SQL
SELECT id, post_name, post_parent
FROM #{BB_PRESS_PREFIX}posts
2016-03-07 12:21:09 -05:00
WHERE post_type = 'forum'
AND LENGTH(COALESCE(post_name, '')) > 0
ORDER BY post_parent, id
SQL
)
2016-03-07 12:21:09 -05:00
create_categories(categories) do |c|
category = { id: c['id'], name: c['post_name'] }
if (parent_id = c['post_parent'].to_i) > 0
category[:parent_category_id] = category_id_from_imported_category_id(parent_id)
end
2016-03-07 12:21:09 -05:00
category
end
end
2016-03-07 12:21:09 -05:00
def import_topics_and_posts
puts "", "importing topics and posts..."
2016-03-07 12:21:09 -05:00
last_post_id = -1
total_posts = bbpress_query(<<-SQL
SELECT COUNT(*) count
FROM #{BB_PRESS_PREFIX}posts
WHERE post_status <> 'spam'
2016-03-07 12:21:09 -05:00
AND post_type IN ('topic', 'reply')
SQL
).first["count"]
batches(BATCH_SIZE) do |offset|
posts = bbpress_query(<<-SQL
SELECT id,
post_author,
post_date,
post_content,
post_title,
post_type,
post_parent
FROM #{BB_PRESS_PREFIX}posts
2016-03-07 12:21:09 -05:00
WHERE post_status <> 'spam'
AND post_type IN ('topic', 'reply')
AND id > #{last_post_id}
ORDER BY id
LIMIT #{BATCH_SIZE}
SQL
).to_a
break if posts.empty?
last_post_id = posts[-1]["id"].to_i
post_ids = posts.map { |p| p["id"].to_i }
next if all_records_exist?(:posts, post_ids)
post_ids_sql = post_ids.join(",")
posts_likes = {}
bbpress_query(<<-SQL
SELECT post_id, meta_value likes
FROM #{BB_PRESS_PREFIX}postmeta
2016-03-07 12:21:09 -05:00
WHERE post_id IN (#{post_ids_sql})
AND meta_key = 'Likes'
SQL
).each { |pm| posts_likes[pm["post_id"]] = pm["likes"].to_i }
anon_names = {}
bbpress_query(<<-SQL
SELECT post_id, meta_value
FROM #{BB_PRESS_PREFIX}postmeta
WHERE post_id IN (#{post_ids_sql})
AND meta_key = '_bbp_anonymous_name'
SQL
).each { |pm| anon_names[pm["post_id"]] = pm["meta_value"] }
2016-03-07 12:21:09 -05:00
create_posts(posts, total: total_posts, offset: offset) do |p|
skip = false
2016-03-07 12:21:09 -05:00
post = {
id: p["id"],
user_id: user_id_from_imported_user_id(p["post_author"]) || find_user_by_import_id(p["post_author"]).try(:id) ||
user_id_from_imported_user_id(anon_names[p['id']]) || find_user_by_import_id(anon_names[p['id']]).try(:id) || -1,
2016-03-07 12:21:09 -05:00
raw: p["post_content"],
created_at: p["post_date"],
like_count: posts_likes[p["id"]],
}
if post[:raw].present?
post[:raw].gsub!("<pre><code>", "```\n")
post[:raw].gsub!("</code></pre>", "\n```")
end
2016-03-07 12:21:09 -05:00
if p["post_type"] == "topic"
post[:category] = category_id_from_imported_category_id(p["post_parent"])
post[:title] = CGI.unescapeHTML(p["post_title"])
else
2016-03-07 12:21:09 -05:00
if parent = topic_lookup_from_imported_post_id(p["post_parent"])
post[:topic_id] = parent[:topic_id]
post[:reply_to_post_number] = parent[:post_number] if parent[:post_number] > 1
else
2016-03-07 12:21:09 -05:00
puts "Skipping #{p["id"]}: #{p["post_content"][0..40]}"
skip = true
end
end
2016-03-07 12:21:09 -05:00
skip ? nil : post
end
end
end
2016-03-07 12:21:09 -05:00
def bbpress_query(sql)
@client.query(sql, cache_rows: false)
end
end
ImportScripts::Bbpress.new.perform