# frozen_string_literal: true require "mysql2" require File.expand_path(File.dirname(__FILE__) + "/base.rb") class ImportScripts::Bbpress < ImportScripts::Base BB_PRESS_HOST ||= ENV["BBPRESS_HOST"] || "localhost" BB_PRESS_DB ||= ENV["BBPRESS_DB"] || "bbpress" BATCH_SIZE ||= 1000 BB_PRESS_PW ||= ENV["BBPRESS_PW"] || "" BB_PRESS_USER ||= ENV["BBPRESS_USER"] || "root" BB_PRESS_PREFIX ||= ENV["BBPRESS_PREFIX"] || "wp_" BB_PRESS_ATTACHMENTS_DIR ||= ENV["BBPRESS_ATTACHMENTS_DIR"] || "/path/to/attachments" def initialize super @he = HTMLEntities.new @client = Mysql2::Client.new( host: BB_PRESS_HOST, username: BB_PRESS_USER, database: BB_PRESS_DB, password: BB_PRESS_PW, ) end def execute import_users import_anonymous_users import_categories import_topics_and_posts import_private_messages import_attachments create_permalinks end def import_users puts "", "importing users..." last_user_id = -1 total_users = bbpress_query(<<-SQL).first["cnt"] SELECT COUNT(DISTINCT(u.id)) AS cnt FROM #{BB_PRESS_PREFIX}users u LEFT JOIN #{BB_PRESS_PREFIX}posts p ON p.post_author = u.id WHERE p.post_type IN ('forum', 'reply', 'topic') AND user_email LIKE '%@%' SQL batches(BATCH_SIZE) do |offset| users = bbpress_query(<<-SQL).to_a SELECT u.id, user_nicename, display_name, user_email, user_registered, user_url, user_pass FROM #{BB_PRESS_PREFIX}users u LEFT JOIN #{BB_PRESS_PREFIX}posts p ON p.post_author = u.id WHERE user_email LIKE '%@%' AND p.post_type IN ('forum', 'reply', 'topic') AND u.id > #{last_user_id} GROUP BY u.id ORDER BY u.id LIMIT #{BATCH_SIZE} SQL break if users.empty? last_user_id = users[-1]["id"] user_ids = users.map { |u| u["id"].to_i } next if all_records_exist?(:users, user_ids) user_ids_sql = user_ids.join(",") users_description = {} bbpress_query(<<-SQL).each { |um| users_description[um["user_id"]] = um["description"] } SELECT user_id, meta_value description FROM #{BB_PRESS_PREFIX}usermeta WHERE user_id IN (#{user_ids_sql}) AND meta_key = 'description' SQL users_last_activity = {} bbpress_query(<<-SQL).each { |um| users_last_activity[um["user_id"]] = um["last_activity"] } SELECT user_id, meta_value last_activity FROM #{BB_PRESS_PREFIX}usermeta WHERE user_id IN (#{user_ids_sql}) AND meta_key = 'last_activity' SQL create_users(users, total: total_users, offset: offset) do |u| { id: u["id"].to_i, username: u["user_nicename"], password: u["user_pass"], email: u["user_email"].downcase, name: u["display_name"].presence || u["user_nicename"], created_at: u["user_registered"], website: u["user_url"], bio_raw: users_description[u["id"]], last_seen_at: users_last_activity[u["id"]], } end end end def import_anonymous_users puts "", "importing anonymous users..." anon_posts = Hash.new anon_names = Hash.new emails = Array.new # gather anonymous users via postmeta table bbpress_query(<<-SQL).each do |pm| SELECT post_id, meta_key, meta_value FROM #{BB_PRESS_PREFIX}postmeta WHERE meta_key LIKE '_bbp_anonymous%' SQL anon_posts[pm["post_id"]] = Hash.new if not anon_posts[pm["post_id"]] if pm["meta_key"] == "_bbp_anonymous_email" anon_posts[pm["post_id"]]["email"] = pm["meta_value"] end if pm["meta_key"] == "_bbp_anonymous_name" anon_posts[pm["post_id"]]["name"] = pm["meta_value"] end if pm["meta_key"] == "_bbp_anonymous_website" anon_posts[pm["post_id"]]["website"] = pm["meta_value"] end end # gather every existent username anon_posts.each do |id, post| anon_names[post["name"]] = Hash.new if not anon_names[post["name"]] # overwriting email address, one user can only use one email address anon_names[post["name"]]["email"] = post["email"] anon_names[post["name"]]["website"] = post["website"] if post["website"] != "" end # make sure every user name has a unique email address anon_names.each do |k, name| if not emails.include? name["email"] emails.push (name["email"]) else name["email"] = "anonymous_#{SecureRandom.hex}@no-email.invalid" end end create_users(anon_names) do |k, n| { id: k, email: n["email"].downcase, name: k, website: n["website"] } end end def import_categories puts "", "importing categories..." categories = bbpress_query(<<-SQL) SELECT id, post_name, post_parent FROM #{BB_PRESS_PREFIX}posts WHERE post_type = 'forum' AND LENGTH(COALESCE(post_name, '')) > 0 ORDER BY post_parent, id SQL create_categories(categories) do |c| category = { id: c["id"], name: c["post_name"] } if (parent_id = c["post_parent"].to_i) > 0 category[:parent_category_id] = category_id_from_imported_category_id(parent_id) end category end end def import_topics_and_posts puts "", "importing topics and posts..." last_post_id = -1 total_posts = bbpress_query(<<-SQL).first["count"] SELECT COUNT(*) count FROM #{BB_PRESS_PREFIX}posts WHERE post_status <> 'spam' AND post_type IN ('topic', 'reply') SQL batches(BATCH_SIZE) do |offset| posts = bbpress_query(<<-SQL).to_a SELECT id, post_author, post_date, post_content, post_title, post_type, post_parent FROM #{BB_PRESS_PREFIX}posts WHERE post_status <> 'spam' AND post_type IN ('topic', 'reply') AND id > #{last_post_id} ORDER BY id LIMIT #{BATCH_SIZE} SQL break if posts.empty? last_post_id = posts[-1]["id"].to_i post_ids = posts.map { |p| p["id"].to_i } next if all_records_exist?(:posts, post_ids) post_ids_sql = post_ids.join(",") posts_likes = {} bbpress_query(<<-SQL).each { |pm| posts_likes[pm["post_id"]] = pm["likes"].to_i } SELECT post_id, meta_value likes FROM #{BB_PRESS_PREFIX}postmeta WHERE post_id IN (#{post_ids_sql}) AND meta_key = 'Likes' SQL anon_names = {} bbpress_query(<<-SQL).each { |pm| anon_names[pm["post_id"]] = pm["meta_value"] } SELECT post_id, meta_value FROM #{BB_PRESS_PREFIX}postmeta WHERE post_id IN (#{post_ids_sql}) AND meta_key = '_bbp_anonymous_name' SQL create_posts(posts, total: total_posts, offset: offset) do |p| skip = false user_id = user_id_from_imported_user_id(p["post_author"]) || find_user_by_import_id(p["post_author"]).try(:id) || user_id_from_imported_user_id(anon_names[p["id"]]) || find_user_by_import_id(anon_names[p["id"]]).try(:id) || -1 post = { id: p["id"], user_id: user_id, raw: p["post_content"], created_at: p["post_date"], like_count: posts_likes[p["id"]], } if post[:raw].present? post[:raw].gsub!(%r{\\(.*?)\\}im) do "```\n#{@he.decode($2)}\n```" end end if p["post_type"] == "topic" post[:category] = category_id_from_imported_category_id(p["post_parent"]) post[:title] = CGI.unescapeHTML(p["post_title"]) else if parent = topic_lookup_from_imported_post_id(p["post_parent"]) post[:topic_id] = parent[:topic_id] post[:reply_to_post_number] = parent[:post_number] if parent[:post_number] > 1 else puts "Skipping #{p["id"]}: #{p["post_content"][0..40]}" skip = true end end skip ? nil : post end end end def import_attachments import_attachments_from_postmeta import_attachments_from_bb_attachments end def import_attachments_from_postmeta puts "", "Importing attachments from 'postmeta'..." count = 0 last_attachment_id = -1 total_attachments = bbpress_query(<<-SQL).first["count"] SELECT COUNT(*) count FROM #{BB_PRESS_PREFIX}postmeta pm JOIN #{BB_PRESS_PREFIX}posts p ON p.id = pm.post_id WHERE pm.meta_key = '_wp_attached_file' AND p.post_parent > 0 SQL batches(BATCH_SIZE) do |offset| attachments = bbpress_query(<<-SQL).to_a SELECT pm.meta_id id, pm.meta_value, p.post_parent post_id FROM #{BB_PRESS_PREFIX}postmeta pm JOIN #{BB_PRESS_PREFIX}posts p ON p.id = pm.post_id WHERE pm.meta_key = '_wp_attached_file' AND p.post_parent > 0 AND pm.meta_id > #{last_attachment_id} ORDER BY pm.meta_id LIMIT #{BATCH_SIZE} SQL break if attachments.empty? last_attachment_id = attachments[-1]["id"].to_i attachments.each do |a| print_status(count += 1, total_attachments, get_start_time("attachments_from_postmeta")) path = File.join(BB_PRESS_ATTACHMENTS_DIR, a["meta_value"]) if File.exist?(path) if post = Post.find_by(id: post_id_from_imported_post_id(a["post_id"])) filename = File.basename(a["meta_value"]) upload = create_upload(post.user.id, path, filename) if upload&.persisted? html = html_for_upload(upload, filename) if !post.raw[html] post.raw << "\n\n" << html post.save! unless PostUpload.where(post: post, upload: upload).exists? PostUpload.create!(post: post, upload: upload) end end end end end end end end def import_attachments_from_bb_attachments puts "", "Importing attachments from 'bb_attachments'..." count = 0 last_attachment_id = -1 total_attachments = bbpress_query(<<-SQL).first["count"] SELECT COUNT(*) count FROM #{BB_PRESS_PREFIX}bb_attachments WHERE post_id IN (SELECT id FROM #{BB_PRESS_PREFIX}posts WHERE post_status <> 'spam' AND post_type IN ('topic', 'reply')) SQL batches(BATCH_SIZE) do |offset| attachments = bbpress_query(<<-SQL).to_a SELECT id, filename, post_id FROM #{BB_PRESS_PREFIX}bb_attachments WHERE post_id IN (SELECT id FROM #{BB_PRESS_PREFIX}posts WHERE post_status <> 'spam' AND post_type IN ('topic', 'reply')) AND id > #{last_attachment_id} ORDER BY id LIMIT #{BATCH_SIZE} SQL break if attachments.empty? last_attachment_id = attachments[-1]["id"].to_i attachments.each do |a| print_status( count += 1, total_attachments, get_start_time("attachments_from_bb_attachments"), ) if path = find_attachment(a["filename"], a["id"]) if post = Post.find_by(id: post_id_from_imported_post_id(a["post_id"])) upload = create_upload(post.user.id, path, a["filename"]) if upload&.persisted? html = html_for_upload(upload, a["filename"]) if !post.raw[html] post.raw << "\n\n" << html post.save! unless PostUpload.where(post: post, upload: upload).exists? PostUpload.create!(post: post, upload: upload) end end end end end end end end def find_attachment(filename, id) @attachments ||= Dir[File.join(BB_PRESS_ATTACHMENTS_DIR, "vf-attachs", "**", "*.*")] @attachments.find { |p| p.end_with?("/#{id}.#{filename}") } end def create_permalinks puts "", "creating permalinks..." last_topic_id = -1 batches(BATCH_SIZE) do |offset| topics = bbpress_query(<<-SQL).to_a SELECT id, guid FROM #{BB_PRESS_PREFIX}posts WHERE post_status <> 'spam' AND post_type IN ('topic') AND id > #{last_topic_id} ORDER BY id LIMIT #{BATCH_SIZE} SQL break if topics.empty? last_topic_id = topics[-1]["id"].to_i topics.each do |t| topic = topic_lookup_from_imported_post_id(t["id"]) begin Permalink.create(url: URI.parse(t["guid"]).path.chomp("/"), topic_id: topic[:topic_id]) rescue StandardError nil end end end end def import_private_messages puts "", "importing private messages..." last_post_id = -1 total_posts = bbpress_query("SELECT COUNT(*) count FROM #{BB_PRESS_PREFIX}bp_messages_messages").first[ "count" ] threads = {} total_count = bbpress_query("SELECT COUNT(*) count FROM #{BB_PRESS_PREFIX}bp_messages_recipients").first[ "count" ] current_count = 0 batches(BATCH_SIZE) do |offset| rows = bbpress_query(<<-SQL).to_a SELECT thread_id, user_id FROM #{BB_PRESS_PREFIX}bp_messages_recipients ORDER BY id LIMIT #{BATCH_SIZE} OFFSET #{offset} SQL break if rows.empty? rows.each do |row| current_count += 1 print_status(current_count, total_count, get_start_time("private_messages")) threads[row["thread_id"]] ||= { target_user_ids: [], imported_topic_id: nil } user_id = user_id_from_imported_user_id(row["user_id"]) if user_id && !threads[row["thread_id"]][:target_user_ids].include?(user_id) threads[row["thread_id"]][:target_user_ids] << user_id end end end batches(BATCH_SIZE) do |offset| posts = bbpress_query(<<-SQL).to_a SELECT id, thread_id, date_sent, sender_id, subject, message FROM wp_bp_messages_messages WHERE id > #{last_post_id} ORDER BY thread_id, date_sent LIMIT #{BATCH_SIZE} SQL break if posts.empty? last_post_id = posts[-1]["id"].to_i create_posts(posts, total: total_posts, offset: offset) do |post| if tcf = TopicCustomField.where(name: "bb_thread_id", value: post["thread_id"]).first { id: "pm#{post["id"]}", topic_id: threads[post["thread_id"]][:imported_topic_id], user_id: user_id_from_imported_user_id(post["sender_id"]) || find_user_by_import_id(post["sender_id"])&.id || -1, raw: post["message"], created_at: post["date_sent"], } else # First post of the thread { id: "pm#{post["id"]}", archetype: Archetype.private_message, user_id: user_id_from_imported_user_id(post["sender_id"]) || find_user_by_import_id(post["sender_id"])&.id || -1, title: post["subject"], raw: post["message"], created_at: post["date_sent"], target_usernames: User.where(id: threads[post["thread_id"]][:target_user_ids]).pluck(:username), post_create_action: proc do |new_post| if topic = new_post.topic threads[post["thread_id"]][:imported_topic_id] = topic.id TopicCustomField.create( topic_id: topic.id, name: "bb_thread_id", value: post["thread_id"], ) else puts "Error in post_create_action! Can't find topic!" end end, } end end end end def bbpress_query(sql) @client.query(sql, cache_rows: false) end end ImportScripts::Bbpress.new.perform