new importer: my little forum
This commit is contained in:
Jay Pfaffman
parent
e07dfe16d7
commit
10f82a9a8e
|
|
@ -0,0 +1,453 @@
|
||||||
|
require "mysql2"
|
||||||
|
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
|
||||||
|
require 'htmlentities'
|
||||||
|
|
||||||
|
|
||||||
|
# Before running this script, paste these lines into your shell,
|
||||||
|
# then use arrow keys to edit the values
|
||||||
|
=begin
|
||||||
|
export DB_HOST="localhost"
|
||||||
|
export DB_NAME="mylittleforum"
|
||||||
|
export DB_PW=""
|
||||||
|
export DB_USER="root"
|
||||||
|
export TABLE_PREFIX="forum_"
|
||||||
|
export IMPORT_AFTER="1970-01-01"
|
||||||
|
export IMAGE_BASE="http://www.example.com/forum"
|
||||||
|
export BASE="forum"
|
||||||
|
=end
|
||||||
|
|
||||||
|
|
||||||
|
class ImportScripts::MylittleforumSQL < ImportScripts::Base
|
||||||
|
|
||||||
|
DB_HOST ||= ENV['DB_HOST'] || "localhost"
|
||||||
|
DB_NAME ||= ENV['DB_NAME'] || "mylittleforum"
|
||||||
|
DB_PW ||= ENV['DB_PW'] || ""
|
||||||
|
DB_USER ||= ENV['DB_USER'] || "root"
|
||||||
|
TABLE_PREFIX ||= ENV['TABLE_PREFIX'] || "forum_"
|
||||||
|
IMPORT_AFTER ||= ENV['IMPORT_AFTER'] || "1970-01-01"
|
||||||
|
IMAGE_BASE ||= ENV['IMAGE_BASE'] || ""
|
||||||
|
BASE ||= ENV['BASE'] || "forum/"
|
||||||
|
BATCH_SIZE = 1000
|
||||||
|
CONVERT_HTML = true
|
||||||
|
QUIET = nil || ENV['VERBOSE'] == "TRUE"
|
||||||
|
FORCE_HOSTNAME = nil || ENV['FORCE_HOSTNAME']
|
||||||
|
|
||||||
|
QUIET = true
|
||||||
|
|
||||||
|
# Site settings
|
||||||
|
SiteSetting.disable_emails = true
|
||||||
|
if FORCE_HOSTNAME
|
||||||
|
SiteSetting.force_hostname=FORCE_HOSTNAME
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
def initialize
|
||||||
|
|
||||||
|
if IMPORT_AFTER > "1970-01-01"
|
||||||
|
print_warning("Importing data after #{IMPORT_AFTER}")
|
||||||
|
end
|
||||||
|
|
||||||
|
super
|
||||||
|
@htmlentities = HTMLEntities.new
|
||||||
|
begin
|
||||||
|
@client = Mysql2::Client.new(
|
||||||
|
host: DB_HOST,
|
||||||
|
username: DB_USER,
|
||||||
|
password: DB_PW,
|
||||||
|
database: DB_NAME
|
||||||
|
)
|
||||||
|
rescue Exception => e
|
||||||
|
puts '='*50
|
||||||
|
puts e.message
|
||||||
|
puts <<EOM
|
||||||
|
Cannot log in to database.
|
||||||
|
|
||||||
|
Hostname: #{DB_HOST}
|
||||||
|
Username: #{DB_USER}
|
||||||
|
Password: #{DB_PW}
|
||||||
|
database: #{DB_NAME}
|
||||||
|
|
||||||
|
You should set these variables:
|
||||||
|
|
||||||
|
export DB_HOST="localhost"
|
||||||
|
export DB_NAME="mylittleforum"
|
||||||
|
export DB_PW=""
|
||||||
|
export DB_USER="root"
|
||||||
|
export TABLE_PREFIX="forum_"
|
||||||
|
export IMPORT_AFTER="1970-01-01"
|
||||||
|
export IMAGE_BASE="http://www.example.com/forum"
|
||||||
|
export BASE="forum"
|
||||||
|
|
||||||
|
Exiting.
|
||||||
|
EOM
|
||||||
|
exit
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def execute
|
||||||
|
import_users
|
||||||
|
import_categories
|
||||||
|
import_topics
|
||||||
|
import_posts
|
||||||
|
|
||||||
|
update_tl0
|
||||||
|
|
||||||
|
create_permalinks
|
||||||
|
end
|
||||||
|
|
||||||
|
def import_users
|
||||||
|
puts '', "creating users"
|
||||||
|
|
||||||
|
username = nil
|
||||||
|
|
||||||
|
total_count = mysql_query("SELECT count(*) count FROM #{TABLE_PREFIX}userdata WHERE last_login > '#{IMPORT_AFTER}';").first['count']
|
||||||
|
|
||||||
|
batches(BATCH_SIZE) do |offset|
|
||||||
|
results = mysql_query("
|
||||||
|
SELECT user_id as UserID, user_name as username,
|
||||||
|
user_real_name as Name,
|
||||||
|
user_email as Email,
|
||||||
|
user_hp as website,
|
||||||
|
user_place as Location,
|
||||||
|
profile as bio_raw,
|
||||||
|
last_login as DateLastActive,
|
||||||
|
user_ip as InsertIPAddress,
|
||||||
|
user_pw as password,
|
||||||
|
logins as days_visited, # user_stats
|
||||||
|
registered as DateInserted,
|
||||||
|
user_pw as password,
|
||||||
|
user_type
|
||||||
|
FROM #{TABLE_PREFIX}userdata
|
||||||
|
WHERE last_login > '#{IMPORT_AFTER}'
|
||||||
|
order by UserID ASC
|
||||||
|
LIMIT #{BATCH_SIZE}
|
||||||
|
OFFSET #{offset};")
|
||||||
|
|
||||||
|
break if results.size < 1
|
||||||
|
|
||||||
|
next if all_records_exist? :users, results.map {|u| u['UserID'].to_i}
|
||||||
|
|
||||||
|
create_users(results, total: total_count, offset: offset) do |user|
|
||||||
|
next if user['Email'].blank?
|
||||||
|
next if @lookup.user_id_from_imported_user_id(user['UserID'])
|
||||||
|
|
||||||
|
# username = fix_username(user['username'])
|
||||||
|
|
||||||
|
{ id: user['UserID'],
|
||||||
|
email: user['Email'],
|
||||||
|
username: user['username'],
|
||||||
|
name: user['Name'],
|
||||||
|
created_at: user['DateInserted'] == nil ? 0 : Time.zone.at(user['DateInserted']),
|
||||||
|
bio_raw: user['bio_raw'],
|
||||||
|
registration_ip_address: user['InsertIPAddress'],
|
||||||
|
website: user['user_hp'],
|
||||||
|
password: user['password'],
|
||||||
|
last_seen_at: user['DateLastActive'] == nil ? 0 : Time.zone.at(user['DateLastActive']),
|
||||||
|
location: user['Location'],
|
||||||
|
admin: user['user_type'] == "admin",
|
||||||
|
moderator: user['user_type'] == "mod",
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def fix_username(username)
|
||||||
|
olduser = username.dup
|
||||||
|
username.gsub!(/Dr\. /,"Dr") # no &
|
||||||
|
username.gsub!(/[ +!\/,*()?]/,"_") # can't have these
|
||||||
|
username.gsub!(/&/,"_and_") # no &
|
||||||
|
username.gsub!(/@/,"_at_") # no @
|
||||||
|
username.gsub!(/#/,"_hash_") # no &
|
||||||
|
username.gsub!(/\'/,"") # seriously?
|
||||||
|
username.gsub!(/[._]+/,"_") # can't have 2 special in a row
|
||||||
|
username.gsub!(/_+/,"_") # could result in dupes, but wtf?
|
||||||
|
username.gsub!(/_$/,"") # could result in dupes, but wtf?
|
||||||
|
if olduser != username
|
||||||
|
print_warning ("#{olduser} --> #{username}")
|
||||||
|
end
|
||||||
|
username
|
||||||
|
end
|
||||||
|
|
||||||
|
def import_categories
|
||||||
|
puts "", "importing categories..."
|
||||||
|
|
||||||
|
categories = mysql_query("
|
||||||
|
SELECT id as CategoryID,
|
||||||
|
category as Name,
|
||||||
|
description as Description
|
||||||
|
FROM #{TABLE_PREFIX}categories
|
||||||
|
ORDER BY CategoryID ASC
|
||||||
|
").to_a
|
||||||
|
|
||||||
|
create_categories(categories) do |category|
|
||||||
|
{
|
||||||
|
id: category['CategoryID'],
|
||||||
|
name: CGI.unescapeHTML(category['Name']),
|
||||||
|
description: CGI.unescapeHTML(category['Description'])
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def import_topics
|
||||||
|
puts "", "importing topics..."
|
||||||
|
|
||||||
|
total_count = mysql_query("SELECT count(*) count FROM #{TABLE_PREFIX}entries
|
||||||
|
WHERE time > '#{IMPORT_AFTER}'
|
||||||
|
AND pid = 0;").first['count']
|
||||||
|
|
||||||
|
batches(BATCH_SIZE) do |offset|
|
||||||
|
discussions = mysql_query(
|
||||||
|
"SELECT id as DiscussionID,
|
||||||
|
category as CategoryID,
|
||||||
|
subject as Name,
|
||||||
|
text as Body,
|
||||||
|
time as DateInserted,
|
||||||
|
youtube_link as youtube,
|
||||||
|
user_id as InsertUserID
|
||||||
|
FROM #{TABLE_PREFIX}entries
|
||||||
|
WHERE pid = 0
|
||||||
|
AND time > '#{IMPORT_AFTER}'
|
||||||
|
ORDER BY time ASC
|
||||||
|
LIMIT #{BATCH_SIZE}
|
||||||
|
OFFSET #{offset};")
|
||||||
|
|
||||||
|
break if discussions.size < 1
|
||||||
|
next if all_records_exist? :posts, discussions.map {|t| "discussion#" + t['DiscussionID'].to_s}
|
||||||
|
|
||||||
|
create_posts(discussions, total: total_count, offset: offset) do |discussion|
|
||||||
|
|
||||||
|
raw = clean_up(discussion['Body'])
|
||||||
|
|
||||||
|
youtube = nil
|
||||||
|
unless discussion['youtube'].blank?
|
||||||
|
youtube = clean_youtube(discussion['youtube'])
|
||||||
|
raw += "\n#{youtube}\n"
|
||||||
|
print_warning(raw)
|
||||||
|
end
|
||||||
|
|
||||||
|
{
|
||||||
|
id: "discussion#" + discussion['DiscussionID'].to_s,
|
||||||
|
user_id: user_id_from_imported_user_id(discussion['InsertUserID']) || Discourse::SYSTEM_USER_ID,
|
||||||
|
title: discussion['Name'].gsub('\\"','"'),
|
||||||
|
category: category_id_from_imported_category_id(discussion['CategoryID']),
|
||||||
|
raw: raw,
|
||||||
|
created_at: Time.zone.at(discussion['DateInserted']),
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def import_posts
|
||||||
|
puts "", "importing posts..."
|
||||||
|
|
||||||
|
total_count = mysql_query(
|
||||||
|
"SELECT count(*) count
|
||||||
|
FROM #{TABLE_PREFIX}entries
|
||||||
|
WHERE pid > 0
|
||||||
|
AND time > '#{IMPORT_AFTER}';").first['count']
|
||||||
|
|
||||||
|
batches(BATCH_SIZE) do |offset|
|
||||||
|
comments = mysql_query(
|
||||||
|
"SELECT id as CommentID,
|
||||||
|
tid as DiscussionID,
|
||||||
|
text as Body,
|
||||||
|
time as DateInserted,
|
||||||
|
youtube_link as youtube,
|
||||||
|
user_id as InsertUserID
|
||||||
|
FROM #{TABLE_PREFIX}entries
|
||||||
|
WHERE pid > 0
|
||||||
|
AND time > '#{IMPORT_AFTER}'
|
||||||
|
ORDER BY time ASC
|
||||||
|
LIMIT #{BATCH_SIZE}
|
||||||
|
OFFSET #{offset};")
|
||||||
|
|
||||||
|
break if comments.size < 1
|
||||||
|
next if all_records_exist? :posts, comments.map {|comment| "comment#" + comment['CommentID'].to_s}
|
||||||
|
|
||||||
|
create_posts(comments, total: total_count, offset: offset) do |comment|
|
||||||
|
next unless t = topic_lookup_from_imported_post_id("discussion#" + comment['DiscussionID'].to_s)
|
||||||
|
next if comment['Body'].blank?
|
||||||
|
raw = clean_up(comment['Body'])
|
||||||
|
youtube = nil
|
||||||
|
unless comment['youtube'].blank?
|
||||||
|
youtube = clean_youtube(comment['youtube'])
|
||||||
|
raw += "\n#{youtube}\n"
|
||||||
|
end
|
||||||
|
{
|
||||||
|
id: "comment#" + comment['CommentID'].to_s,
|
||||||
|
user_id: user_id_from_imported_user_id(comment['InsertUserID']) || Discourse::SYSTEM_USER_ID,
|
||||||
|
topic_id: t[:topic_id],
|
||||||
|
raw: clean_up(raw),
|
||||||
|
created_at: Time.zone.at(comment['DateInserted'])
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def clean_youtube(youtube_raw)
|
||||||
|
youtube_cooked = clean_up(youtube_raw.dup.to_s)
|
||||||
|
# get just src from <iframe> and put on a line by itself
|
||||||
|
re = /<iframe.+?src="(\S+?)".+?<\/iframe>/mix
|
||||||
|
youtube_cooked.gsub!(re) {"\n#{$1}\n"}
|
||||||
|
re = /<object.+?src="(\S+?)".+?<\/object>/mix
|
||||||
|
youtube_cooked.gsub!(re) {"\n#{$1}\n"}
|
||||||
|
youtube_cooked.gsub!(/^\/\//, "https://") # make sure it has a protocol
|
||||||
|
unless /http/.match(youtube_cooked) # handle case of only youtube object number
|
||||||
|
if youtube_cooked.length < 8 || /[<>=]/.match(youtube_cooked)
|
||||||
|
# probably not a youtube id
|
||||||
|
youtube_cooked = ""
|
||||||
|
else
|
||||||
|
youtube_cooked = 'https://www.youtube.com/watch?v=' + youtube_cooked
|
||||||
|
end
|
||||||
|
end
|
||||||
|
print_warning("#{'-'*40}\nBefore: #{youtube_raw}\nAfter: #{youtube_cooked}") unless QUIET
|
||||||
|
|
||||||
|
youtube_cooked
|
||||||
|
end
|
||||||
|
|
||||||
|
def clean_up(raw)
|
||||||
|
return "" if raw.blank?
|
||||||
|
|
||||||
|
# decode HTML entities
|
||||||
|
raw = @htmlentities.decode(raw)
|
||||||
|
|
||||||
|
# don't \ quotes
|
||||||
|
raw = raw.gsub('\\"','"')
|
||||||
|
raw = raw.gsub("\\'","'")
|
||||||
|
|
||||||
|
raw = raw.gsub(/\[b\]/i, "<strong>")
|
||||||
|
raw = raw.gsub(/\[\/b\]/i, "</strong>")
|
||||||
|
|
||||||
|
raw = raw.gsub(/\[i\]/i, "<em>")
|
||||||
|
raw = raw.gsub(/\[\/i\]/i, "</em>")
|
||||||
|
|
||||||
|
raw = raw.gsub(/\[u\]/i, "<em>")
|
||||||
|
raw = raw.gsub(/\[\/u\]/i, "</em>")
|
||||||
|
|
||||||
|
raw = raw.gsub(/\[url\](\S+)\[\/url\]/im) { "#{$1}"}
|
||||||
|
raw = raw.gsub(/\[link\](\S+)\[\/link\]/im) { "#{$1}"}
|
||||||
|
|
||||||
|
# URL & LINK with text
|
||||||
|
raw = raw.gsub(/\[url=(\S+?)\](.*?)\[\/url\]/im) { "<a href=\"#{$1}\">#{$2}</a>"}
|
||||||
|
raw = raw.gsub(/\[link=(\S+?)\](.*?)\[\/link\]/im) { "<a href=\"#{$1}\">#{$2}</a>"}
|
||||||
|
|
||||||
|
# remote images
|
||||||
|
raw = raw.gsub(/\[img\](https?:.+?)\[\/img\]/im) { "<img src=\"#{$1}\">" }
|
||||||
|
raw = raw.gsub(/\[img=(https?.+?)\](.+?)\[\/img\]/im) { "<img src=\"#{$1}\" alt=\"#{$2}\">" }
|
||||||
|
# local images
|
||||||
|
raw = raw.gsub(/\[img\](.+?)\[\/img\]/i) { "<img src=\"#{IMAGE_BASE}/#{$1}\">" }
|
||||||
|
raw = raw.gsub(/\[img=(.+?)\](https?.+?)\[\/img\]/im) { "<img src=\"#{IMAGE_BASE}/#{$1}\" alt=\"#{$2}\">" }
|
||||||
|
|
||||||
|
# Convert image bbcode
|
||||||
|
raw.gsub!(/\[img=(\d+),(\d+)\]([^\]]*)\[\/img\]/im, '<img width="\1" height="\2" src="\3">')
|
||||||
|
|
||||||
|
# [div]s are really [quote]s
|
||||||
|
raw.gsub!(/\[div\]/mix, "[quote]")
|
||||||
|
raw.gsub!(/\[\/div\]/mix, "[/quote]")
|
||||||
|
|
||||||
|
# [postedby] -> link to @user
|
||||||
|
raw.gsub(/\[postedby\](.+?)\[b\](.+?)\[\/b\]\[\/postedby\]/i) { "#{$1}@#{$2}" }
|
||||||
|
|
||||||
|
# CODE (not tested)
|
||||||
|
raw = raw.gsub(/\[code\](\S+)\[\/code\]/im) { "```\n#{$1}\n```"}
|
||||||
|
raw = raw.gsub(/\[pre\](\S+)\[\/pre\]/im) { "```\n#{$1}\n```"}
|
||||||
|
|
||||||
|
raw = raw.gsub(/(https:\/\/youtu\S+)/i) { "\n#{$1}\n" } #youtube links on line by themselves
|
||||||
|
|
||||||
|
# no center
|
||||||
|
raw = raw.gsub(/\[\/?center\]/i,"")
|
||||||
|
|
||||||
|
# no size
|
||||||
|
raw = raw.gsub(/\[\/?size.*?\]/i,"")
|
||||||
|
|
||||||
|
### FROM VANILLA:
|
||||||
|
|
||||||
|
# fix whitespaces
|
||||||
|
raw = raw.gsub(/(\\r)?\\n/, "\n")
|
||||||
|
.gsub("\\t", "\t")
|
||||||
|
|
||||||
|
unless CONVERT_HTML
|
||||||
|
# replace all chevrons with HTML entities
|
||||||
|
# NOTE: must be done
|
||||||
|
# - AFTER all the "code" processing
|
||||||
|
# - BEFORE the "quote" processing
|
||||||
|
raw = raw.gsub(/`([^`]+)`/im) { "`" + $1.gsub("<", "\u2603") + "`" }
|
||||||
|
.gsub("<", "<")
|
||||||
|
.gsub("\u2603", "<")
|
||||||
|
|
||||||
|
raw = raw.gsub(/`([^`]+)`/im) { "`" + $1.gsub(">", "\u2603") + "`" }
|
||||||
|
.gsub(">", ">")
|
||||||
|
.gsub("\u2603", ">")
|
||||||
|
end
|
||||||
|
|
||||||
|
# Remove the color tag
|
||||||
|
raw.gsub!(/\[color=[#a-z0-9]+\]/i, "")
|
||||||
|
raw.gsub!(/\[\/color\]/i, "")
|
||||||
|
### END VANILLA:
|
||||||
|
|
||||||
|
raw
|
||||||
|
end
|
||||||
|
|
||||||
|
def staff_guardian
|
||||||
|
@_staff_guardian ||= Guardian.new(Discourse.system_user)
|
||||||
|
end
|
||||||
|
|
||||||
|
def mysql_query(sql)
|
||||||
|
@client.query(sql)
|
||||||
|
# @client.query(sql, cache_rows: false) #segfault: cache_rows: false causes segmentation fault
|
||||||
|
end
|
||||||
|
|
||||||
|
def create_permalinks
|
||||||
|
puts '', 'Creating redirects...', ''
|
||||||
|
|
||||||
|
puts '', 'Users...', ''
|
||||||
|
User.find_each do |u|
|
||||||
|
ucf = u.custom_fields
|
||||||
|
if ucf && ucf["import_id"] && ucf["import_username"]
|
||||||
|
Permalink.create( url: "#{BASE}/user-id-#{ucf['import_id']}.html", external_url: "/users/#{u.username}" ) rescue nil
|
||||||
|
print '.'
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
puts '', 'Posts...', ''
|
||||||
|
Post.find_each do |post|
|
||||||
|
pcf = post.custom_fields
|
||||||
|
if pcf && pcf["import_id"]
|
||||||
|
topic = post.topic
|
||||||
|
id = pcf["import_id"].split('#').last
|
||||||
|
if post.post_number == 1
|
||||||
|
Permalink.create( url: "#{BASE}/forum_entry-id-#{id}.html", topic_id: topic.id ) rescue nil
|
||||||
|
unless QUIET
|
||||||
|
print_warning("forum_entry-id-#{id}.html --> http://localhost:3000/t/#{topic.id}")
|
||||||
|
end
|
||||||
|
else
|
||||||
|
Permalink.create( url: "#{BASE}/forum_entry-id-#{id}.html", post_id: post.id ) rescue nil
|
||||||
|
unless QUIET
|
||||||
|
print_warning("forum_entry-id-#{id}.html --> http://localhost:3000/t/#{topic.id}/#{post.id}")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
print '.'
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
puts '', 'Categories...', ''
|
||||||
|
Category.find_each do |cat|
|
||||||
|
ccf = cat.custom_fields
|
||||||
|
next unless id = ccf["import_id"]
|
||||||
|
unless QUIET
|
||||||
|
print_warning("forum-category-#{id}.html --> /t/#{cat.id}")
|
||||||
|
end
|
||||||
|
Permalink.create( url: "#{BASE}/forum-category-#{id}.html", category_id: cat.id ) rescue nil
|
||||||
|
print '.'
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def print_warning(message)
|
||||||
|
$stderr.puts "#{message}"
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
ImportScripts::MylittleforumSQL.new.perform
|
||||||
Loading…
Reference in New Issue