FEATURE: Import Script for Fusionforge (#22281)
This is an import script for the forum/development platform https://www.fusionforge.org/projects/fusionforge imports users, forums and posts including attachments
This commit is contained in:
Sebastian Wagner
GitHub
parent
a3a74ee4a5
commit
050a285f40
|
|
@ -0,0 +1,237 @@
|
|||
# frozen_string_literal: true
|
||||
# (c) 2023 Intevation GmbH
|
||||
|
||||
require "pg"
|
||||
|
||||
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
|
||||
|
||||
# Call it like this:
|
||||
# RAILS_ENV=production bundle exec ruby script/import_scripts/fusionforge.rb
|
||||
class ImportScripts::FusionForge < ImportScripts::Base
|
||||
FUSIONFORGE = "fusionforge"
|
||||
BATCH_SIZE = 1000
|
||||
|
||||
def initialize
|
||||
super
|
||||
|
||||
@client =
|
||||
PG.connect(
|
||||
host: "localhost",
|
||||
user: "fusionforge",
|
||||
password: "fusionforge",
|
||||
dbname: FUSIONFORGE,
|
||||
)
|
||||
end
|
||||
|
||||
def execute
|
||||
import_users
|
||||
import_categories
|
||||
import_posts
|
||||
import_attachments
|
||||
end
|
||||
|
||||
def import_users
|
||||
puts "", "creating users"
|
||||
|
||||
total_count = @client.exec("
|
||||
WITH relevant_posts AS (
|
||||
SELECT DISTINCT posted_by FROM forum
|
||||
)
|
||||
SELECT
|
||||
COUNT(DISTINCT user_id) AS count
|
||||
FROM users u
|
||||
JOIN relevant_posts f on u.user_id = f.posted_by
|
||||
").first["count"]
|
||||
|
||||
batches(BATCH_SIZE) do |offset|
|
||||
results =
|
||||
@client.exec(
|
||||
# Only select users which have some content
|
||||
"WITH relevant_posts AS (
|
||||
SELECT DISTINCT posted_by FROM forum
|
||||
)
|
||||
SELECT
|
||||
DISTINCT user_id, email, user_name, add_date, status, unix_pw
|
||||
FROM users u
|
||||
JOIN relevant_posts f on u.user_id = f.posted_by
|
||||
LIMIT #{BATCH_SIZE}
|
||||
OFFSET #{offset};",
|
||||
)
|
||||
|
||||
break if results.ntuples < 1
|
||||
|
||||
next if all_records_exist? :users, results.map { |u| u["user_id"].to_i }
|
||||
puts "Creating users"
|
||||
|
||||
create_users(results, total: total_count, offset: offset) do |user|
|
||||
{
|
||||
id: user["user_id"],
|
||||
email: user["email"],
|
||||
username: user["user_name"],
|
||||
name: user["name"],
|
||||
active: user["status"] == 'A' && user["unix_pw"] != 'deleted',
|
||||
created_at: Time.zone.at(user["add_date"].to_i),
|
||||
last_emailed_at: nil, # default is "now", which is not true
|
||||
approved: true,
|
||||
# for https://github.com/communiteq/discourse-migratepassword/
|
||||
# this field results in custom_fields['import_pass']. This also activates the accounts, see base.rb on `u.activate`.
|
||||
password: user["unix_pw"] != 'deleted' ? user["unix_pw"] : nil,
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def import_categories
|
||||
puts "", "importing groups..."
|
||||
|
||||
categories =
|
||||
@client.exec(
|
||||
"
|
||||
SELECT group_id, group_name
|
||||
FROM groups
|
||||
WHERE use_forum = 1 AND (SELECT COUNT(*) FROM forum_group_list WHERE forum_group_list.group_id = groups.group_id) > 0
|
||||
ORDER BY group_id ASC
|
||||
",
|
||||
).to_a
|
||||
|
||||
create_categories(categories) { |category| { id: category["group_id"], name: category["group_name"] } }
|
||||
|
||||
puts "", "importing forums..."
|
||||
|
||||
children_categories =
|
||||
@client.exec(
|
||||
"
|
||||
SELECT group_forum_id, group_id, forum_name, description
|
||||
FROM forum_group_List
|
||||
ORDER BY group_id, group_forum_id
|
||||
",
|
||||
).to_a
|
||||
|
||||
create_categories(children_categories) do |category|
|
||||
{
|
||||
id: "child##{category["group_forum_id"]}",
|
||||
name: category["forum_name"],
|
||||
description: category["description"],
|
||||
parent_category_id: category_id_from_imported_category_id(category["group_id"]),
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
def import_posts
|
||||
puts "", "creating topics and posts"
|
||||
|
||||
total_count = @client.exec("SELECT count(*) as count from forum").first["count"]
|
||||
|
||||
batches(BATCH_SIZE) do |offset|
|
||||
results =
|
||||
@client.exec(
|
||||
"
|
||||
SELECT msg_id,
|
||||
group_forum_id,
|
||||
subject,
|
||||
thread_id,
|
||||
posted_by,
|
||||
body,
|
||||
post_date,
|
||||
is_followup_to,
|
||||
has_followups
|
||||
FROM forum
|
||||
ORDER BY thread_id, post_date
|
||||
LIMIT #{BATCH_SIZE}
|
||||
OFFSET #{offset};
|
||||
",
|
||||
).to_a
|
||||
|
||||
break if results.length < 1
|
||||
next if all_records_exist? :posts, results.map { |m| m["msg_id"].to_i }
|
||||
|
||||
create_posts(results, total: total_count, offset: offset) do |m|
|
||||
skip = false
|
||||
mapped = {}
|
||||
|
||||
mapped[:id] = m["msg_id"]
|
||||
mapped[:user_id] = user_id_from_imported_user_id(m["posted_by"]) || -1
|
||||
mapped[:raw] = CGI.unescapeHTML(m["body"])
|
||||
mapped[:created_at] = Time.zone.at(m["post_date"].to_i)
|
||||
|
||||
if m["is_followup_to"] == "0"
|
||||
# if is not a follow up, then it's a thread
|
||||
mapped[:category] = category_id_from_imported_category_id(m["group_forum_id"])
|
||||
mapped[:title] = CGI.unescapeHTML(m["subject"])
|
||||
else
|
||||
parent = topic_lookup_from_imported_post_id(m["is_followup_to"].to_i)
|
||||
if parent
|
||||
mapped[:topic_id] = parent[:topic_id]
|
||||
else
|
||||
skip = true
|
||||
end
|
||||
end
|
||||
skip ? nil : mapped
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def import_attachments
|
||||
puts "", "importing attachments..."
|
||||
|
||||
uploads = @client.exec("
|
||||
SELECT msg_id, filename, attachmentid
|
||||
FROM forum_attachment
|
||||
order by msg_id
|
||||
").to_a
|
||||
|
||||
current_count = 0
|
||||
total_count = uploads.count
|
||||
|
||||
uploads.each do |upload|
|
||||
post_id = post_id_from_imported_post_id(upload["msg_id"])
|
||||
|
||||
if post_id.nil?
|
||||
puts "Post #{upload["msg_id"]} for attachment #{upload["attachmentid"]} not found"
|
||||
next
|
||||
end
|
||||
|
||||
post = Post.find(post_id)
|
||||
|
||||
real_filename = upload["filename"]
|
||||
real_filename.prepend SecureRandom.hex if real_filename[0] == "."
|
||||
|
||||
file_hex = sprintf("%x", upload["attachmentid"])
|
||||
prefix = file_hex[-2..-1]
|
||||
if not prefix
|
||||
prefix = file_hex
|
||||
end
|
||||
postfix = file_hex[0..-3].to_s
|
||||
if postfix == ''
|
||||
postfix = '0'
|
||||
end
|
||||
filename = File.join("/tmp/var/lib/fusionforge/forum/", prefix, "/", postfix)
|
||||
|
||||
upl_obj = create_upload(post.user.id, filename, real_filename)
|
||||
|
||||
if upl_obj&.persisted?
|
||||
html = html_for_upload(upl_obj, real_filename)
|
||||
if !post.raw[html]
|
||||
post.raw += "\n\n#{html}\n\n"
|
||||
post.save!
|
||||
if PostUpload.where(post: post, upload: upl_obj).exists?
|
||||
puts "skipping creating uploaded for previously uploaded file #{upload["attachmentid"]}"
|
||||
else
|
||||
PostUpload.create!(post: post, upload: upl_obj)
|
||||
end
|
||||
else
|
||||
puts "Skipping attachment #{upload["attachmentid"]}"
|
||||
end
|
||||
else
|
||||
puts "Failed to upload attachment #{upload["attachmentid"]}"
|
||||
exit
|
||||
end
|
||||
|
||||
current_count += 1
|
||||
print_status(current_count, total_count)
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
ImportScripts::FusionForge.new.perform
|
||||
Loading…
Reference in New Issue