# frozen_string_literal: true require "pg" require File.expand_path(File.dirname(__FILE__) + "/base.rb") # Call it like this: # RAILS_ENV=production bundle exec ruby script/import_scripts/fusionforge.rb class ImportScripts::FusionForge < ImportScripts::Base FUSIONFORGE = "fusionforge" BATCH_SIZE = 1000 def initialize super @client = PG.connect( host: "localhost", user: "fusionforge", password: "fusionforge", dbname: FUSIONFORGE, ) end def execute import_users import_categories import_posts import_attachments end def import_users puts "", "creating users" total_count = @client.exec( " WITH relevant_posts AS ( SELECT DISTINCT posted_by FROM forum ) SELECT COUNT(DISTINCT user_id) AS count FROM users u JOIN relevant_posts f on u.user_id = f.posted_by ", ).first[ "count" ] batches(BATCH_SIZE) do |offset| results = @client.exec( # Only select users which have some content "WITH relevant_posts AS ( SELECT DISTINCT posted_by FROM forum ) SELECT DISTINCT user_id, email, user_name, add_date, status, unix_pw FROM users u JOIN relevant_posts f on u.user_id = f.posted_by LIMIT #{BATCH_SIZE} OFFSET #{offset};", ) break if results.ntuples < 1 next if all_records_exist? :users, results.map { |u| u["user_id"].to_i } puts "Creating users" create_users(results, total: total_count, offset: offset) do |user| { id: user["user_id"], email: user["email"], username: user["user_name"], name: user["name"], active: user["status"] == "A" && user["unix_pw"] != "deleted", created_at: Time.zone.at(user["add_date"].to_i), last_emailed_at: nil, # default is "now", which is not true approved: true, # for https://github.com/communiteq/discourse-migratepassword/ # this field results in custom_fields['import_pass']. This also activates the accounts, see base.rb on `u.activate`. password: user["unix_pw"] != "deleted" ? user["unix_pw"] : nil, } end end end def import_categories puts "", "importing groups..." categories = @client.exec( " SELECT group_id, group_name FROM groups WHERE use_forum = 1 AND (SELECT COUNT(*) FROM forum_group_list WHERE forum_group_list.group_id = groups.group_id) > 0 ORDER BY group_id ASC ", ).to_a create_categories(categories) do |category| { id: category["group_id"], name: category["group_name"] } end puts "", "importing forums..." children_categories = @client.exec( " SELECT group_forum_id, group_id, forum_name, description FROM forum_group_List ORDER BY group_id, group_forum_id ", ).to_a create_categories(children_categories) do |category| { id: "child##{category["group_forum_id"]}", name: category["forum_name"], description: category["description"], parent_category_id: category_id_from_imported_category_id(category["group_id"]), } end end def import_posts puts "", "creating topics and posts" total_count = @client.exec("SELECT count(*) as count from forum").first["count"] batches(BATCH_SIZE) do |offset| results = @client.exec( " SELECT msg_id, group_forum_id, subject, thread_id, posted_by, body, post_date, is_followup_to, has_followups FROM forum ORDER BY thread_id, post_date LIMIT #{BATCH_SIZE} OFFSET #{offset}; ", ).to_a break if results.length < 1 next if all_records_exist? :posts, results.map { |m| m["msg_id"].to_i } create_posts(results, total: total_count, offset: offset) do |m| skip = false mapped = {} mapped[:id] = m["msg_id"] mapped[:user_id] = user_id_from_imported_user_id(m["posted_by"]) || -1 mapped[:raw] = CGI.unescapeHTML(m["body"]) mapped[:created_at] = Time.zone.at(m["post_date"].to_i) if m["is_followup_to"] == "0" # if is not a follow up, then it's a thread mapped[:category] = category_id_from_imported_category_id(m["group_forum_id"]) mapped[:title] = CGI.unescapeHTML(m["subject"]) else parent = topic_lookup_from_imported_post_id(m["is_followup_to"].to_i) if parent mapped[:topic_id] = parent[:topic_id] else skip = true end end skip ? nil : mapped end end end def import_attachments puts "", "importing attachments..." uploads = @client.exec( " SELECT msg_id, filename, attachmentid FROM forum_attachment order by msg_id ", ).to_a current_count = 0 total_count = uploads.count uploads.each do |upload| post_id = post_id_from_imported_post_id(upload["msg_id"]) if post_id.nil? puts "Post #{upload["msg_id"]} for attachment #{upload["attachmentid"]} not found" next end post = Post.find(post_id) real_filename = upload["filename"] real_filename.prepend SecureRandom.hex if real_filename[0] == "." file_hex = sprintf("%x", upload["attachmentid"]) prefix = file_hex[-2..-1] prefix = file_hex if not prefix postfix = file_hex[0..-3].to_s postfix = "0" if postfix == "" filename = File.join("/tmp/var/lib/fusionforge/forum/", prefix, "/", postfix) upl_obj = create_upload(post.user.id, filename, real_filename) if upl_obj&.persisted? html = html_for_upload(upl_obj, real_filename) if !post.raw[html] post.raw += "\n\n#{html}\n\n" post.save! if PostUpload.where(post: post, upload: upl_obj).exists? puts "skipping creating uploaded for previously uploaded file #{upload["attachmentid"]}" else PostUpload.create!(post: post, upload: upl_obj) end else puts "Skipping attachment #{upload["attachmentid"]}" end else puts "Failed to upload attachment #{upload["attachmentid"]}" exit end current_count += 1 print_status(current_count, total_count) end end end ImportScripts::FusionForge.new.perform