Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DEV: Add vBulletin5 bulk importer (#12904) 

This is a pretty straightforward bulk importer, just tailored to the vBulletin 5 database structure.

Also made a few minor improvements to the base importer -- should be self explanatory in the code.
This commit is contained in:
Justin DiRose 2021-04-30 11:03:33 -05:00 committed by GitHub
parent ffa78b5c26
commit c1517e428e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 790 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
script/bulk_import/base.rb
View File

@ -627,6 +627,10 @@ class BulkImport::Base
raw.gsub!(/\[TD\](.*?)\[\/TD\]/im, "\\1") raw.gsub!(/\[TD\](.*?)\[\/TD\]/im, "\\1")
raw.gsub!(/\[TD="?.*?"?\](.*?)\[\/TD\]/im, "\\1") raw.gsub!(/\[TD="?.*?"?\](.*?)\[\/TD\]/im, "\\1")
# [STRIKE]
raw.gsub!(/\[STRIKE\]/i, "<s>")
raw.gsub!(/\[\/STRIKE\]/i, "</s>")
# [QUOTE]...[/QUOTE] # [QUOTE]...[/QUOTE]
raw.gsub!(/\[QUOTE="([^\]]+)"\]/i) { "[QUOTE=#{$1}]" } raw.gsub!(/\[QUOTE="([^\]]+)"\]/i) { "[QUOTE=#{$1}]" }
@ -644,7 +648,7 @@ class BulkImport::Base
username = @mapped_usernames[imported_username] || imported_username username = @mapped_usernames[imported_username] || imported_username
post_number = post_number_from_imported_id(imported_postid) post_number = post_number_from_imported_id(imported_postid)
topic_id = topic_id_from_imported_post_id(imported_post_id) topic_id = topic_id_from_imported_post_id(imported_postid)
if post_number && topic_id if post_number && topic_id
"\n[quote=\"#{username}, post:#{post_number}, topic:#{topic_id}\"]\n" "\n[quote=\"#{username}, post:#{post_number}, topic:#{topic_id}\"]\n"
@ -668,9 +672,9 @@ class BulkImport::Base
# (basically, we're only missing list=a here...) # (basically, we're only missing list=a here...)
# (https://meta.discourse.org/t/phpbb-3-importer-old/17397) # (https://meta.discourse.org/t/phpbb-3-importer-old/17397)
raw.gsub!(/\[list\](.*?)\[\/list\]/im, '[ul]\1[/ul]') raw.gsub!(/\[list\](.*?)\[\/list\]/im, '[ul]\1[/ul]')
raw.gsub!(/\[list=1\](.*?)\[\/list\]/im, '[ol]\1[/ol]') raw.gsub!(/\[list=1\|?[^\]]*\](.*?)\[\/list\]/im, '[ol]\1[/ol]')
raw.gsub!(/\[list\](.*?)\[\/list:u\]/im, '[ul]\1[/ul]') raw.gsub!(/\[list\](.*?)\[\/list:u\]/im, '[ul]\1[/ul]')
raw.gsub!(/\[list=1\](.*?)\[\/list:o\]/im, '[ol]\1[/ol]') raw.gsub!(/\[list=1\|?[^\]]*\](.*?)\[\/list:o\]/im, '[ol]\1[/ol]')
# convert *-tags to li-tags so bbcode-to-md can do its magic on phpBB's lists: # convert *-tags to li-tags so bbcode-to-md can do its magic on phpBB's lists:
raw.gsub!(/\[\*\]\n/, '') raw.gsub!(/\[\*\]\n/, '')
raw.gsub!(/\[\*\](.*?)\[\/\*:m\]/, '[li]\1[/li]') raw.gsub!(/\[\*\](.*?)\[\/\*:m\]/, '[li]\1[/li]')
@ -749,6 +753,7 @@ class BulkImport::Base
name.gsub!(/[^A-Za-z0-9]+$/, "") name.gsub!(/[^A-Za-z0-9]+$/, "")
name.gsub!(/([-_.]{2,})/) { $1.first } name.gsub!(/([-_.]{2,})/) { $1.first }
name.strip! name.strip!
name.truncate(60)
name name
end end
@ -757,7 +762,7 @@ class BulkImport::Base
end end
def random_email def random_email
"#{SecureRandom.hex}@ema.il" "#{SecureRandom.hex}@email.invalid"
end end
def pre_cook(raw) def pre_cook(raw)

781
script/bulk_import/vbulletin5.rb Normal file
View File

@ -0,0 +1,781 @@
# frozen_string_literal: true
require_relative "base"
require "cgi"
require "set"
require "mysql2"
require "htmlentities"
require 'ruby-bbcode-to-md'
require 'find'
class BulkImport::VBulletin5 < BulkImport::Base
DB_PREFIX = ""
SUSPENDED_TILL ||= Date.new(3000, 1, 1)
ATTACH_DIR ||= ENV['ATTACH_DIR'] || '/shared/import/data/attachments'
AVATAR_DIR ||= ENV['AVATAR_DIR'] || '/shared/import/data/customavatars'
ROOT_NODE = 2
def initialize
super
host = ENV["DB_HOST"] || "localhost"
username = ENV["DB_USERNAME"] || "root"
password = ENV["DB_PASSWORD"]
database = ENV["DB_NAME"] || "vbulletin"
charset = ENV["DB_CHARSET"] || "utf8"
@html_entities = HTMLEntities.new
@encoding = CHARSET_MAP[charset]
@bbcode_to_md = true
@client = Mysql2::Client.new(
host: host,
username: username,
password: password,
database: database,
encoding: charset,
reconnect: true
)
@client.query_options.merge!(as: :array, cache_rows: false)
# TODO: Add `LIMIT 1` to the below queries
# ------
# be aware there may be other contenttypeid's in use, such as poll, link, video, etc.
@forum_typeid = mysql_query("SELECT contenttypeid FROM #{DB_PREFIX}contenttype WHERE class='Forum'").to_a[0][0]
@channel_typeid = mysql_query("SELECT contenttypeid FROM #{DB_PREFIX}contenttype WHERE class='Channel'").to_a[0][0]
@text_typeid = mysql_query("SELECT contenttypeid FROM #{DB_PREFIX}contenttype WHERE class='Text'").to_a[0][0]
end
def execute
# enable as per requirement:
#SiteSetting.automatic_backups_enabled = false
#SiteSetting.disable_emails = "non-staff"
#SiteSetting.authorized_extensions = '*'
#SiteSetting.max_image_size_kb = 102400
#SiteSetting.max_attachment_size_kb = 102400
#SiteSetting.clean_up_uploads = false
#SiteSetting.clean_orphan_uploads_grace_period_hours = 43200
#SiteSetting.max_category_nesting = 3
import_groups
import_users
import_group_users
import_user_emails
import_user_stats
import_user_profiles
import_user_account_id
import_categories
import_topics
import_topic_first_posts
import_replies
import_likes
import_private_topics
import_topic_allowed_users
import_private_first_posts
import_private_replies
create_oauth_records
create_permalinks
import_attachments
end
def import_groups
puts "Importing groups..."
groups = mysql_stream <<-SQL
SELECT usergroupid, title, description, usertitle
FROM #{DB_PREFIX}usergroup
WHERE usergroupid > #{@last_imported_group_id}
ORDER BY usergroupid
SQL
create_groups(groups) do |row|
{
imported_id: row[0],
name: normalize_text(row[1]),
bio_raw: normalize_text(row[2]),
title: normalize_text(row[3]),
}
end
end
def import_users
puts "Importing users..."
users = mysql_stream <<-SQL
SELECT u.userid, u.username, u.joindate, u.birthday,
u.ipaddress, u.usergroupid, ub.bandate, ub.liftdate, u.email
FROM #{DB_PREFIX}user u
LEFT JOIN #{DB_PREFIX}userban ub ON ub.userid = u.userid
WHERE u.userid > #{@last_imported_user_id}
ORDER BY u.userid
SQL
create_users(users) do |row|
u = {
imported_id: row[0],
username: normalize_text(row[1].truncate(60)),
name: normalize_text(row[1]),
email: row[8],
created_at: Time.zone.at(row[2]),
date_of_birth: parse_birthday(row[3]),
primary_group_id: group_id_from_imported_id(row[5]),
admin: row[5] == 6,
moderator: row[5] == 7
}
u[:ip_address] = row[4][/\b(?:\d{1,3}\.){3}\d{1,3}\b/] if row[4].present?
if row[7]
u[:suspended_at] = Time.zone.at(row[6])
u[:suspended_till] = row[7] > 0 ? Time.zone.at(row[7]) : SUSPENDED_TILL
end
u
end
end
def import_user_emails
puts "Importing user emails..."
users = mysql_stream <<-SQL
SELECT u.userid, u.email, u.joindate
FROM #{DB_PREFIX}user u
WHERE u.userid > #{@last_imported_user_id}
ORDER BY u.userid
SQL
create_user_emails(users) do |row|
{
imported_id: row[0],
imported_user_id: row[0],
email: random_email,
created_at: Time.zone.at(row[2])
}
end
end
def import_user_stats
puts "Importing user stats..."
users = mysql_stream <<-SQL
SELECT u.userid, u.joindate, u.posts,
SUM(
CASE
WHEN n.contenttypeid = #{@text_typeid}
AND n.parentid IN ( select nodeid from #{DB_PREFIX}node where contenttypeid=#{@channel_typeid} )
THEN 1
ELSE 0
END
) AS threads
FROM #{DB_PREFIX}user u
LEFT OUTER JOIN #{DB_PREFIX}node n ON u.userid = n.userid
WHERE u.userid > #{@last_imported_user_id}
GROUP BY u.userid
ORDER BY u.userid
SQL
create_user_stats(users) do |row|
user = {
imported_id: row[0],
imported_user_id: row[0],
new_since: Time.zone.at(row[1]),
post_count: row[2],
topic_count: row[3],
}
user
end
end
def import_group_users
puts "Importing group users..."
# import primary groups
group_users = mysql_stream <<-SQL
SELECT usergroupid, userid
FROM #{DB_PREFIX}user
WHERE userid > #{@last_imported_user_id}
SQL
create_group_users(group_users) do |row|
{
group_id: group_id_from_imported_id(row[0]),
user_id: user_id_from_imported_id(row[1]),
}
end
# import secondary group memberships
secondary_group_users = mysql_stream <<-SQL
SELECT membergroupids, userid
FROM #{DB_PREFIX}user
SQL
group_mapping = []
secondary_group_users.each do |user|
next unless user_id = user_id_from_imported_id(user[1])
member_groups = user[0].split(",")
member_groups.each do |group|
next unless group_id = group_id_from_imported_id(group)
group_mapping << [group_id, user_id]
end
end
create_group_users(group_mapping) do |row|
{
group_id: row[0],
user_id: row[1]
}
end
end
def import_user_profiles
puts "Importing user profiles..."
user_profiles = mysql_stream <<-SQL
SELECT userid, homepage, profilevisits
FROM #{DB_PREFIX}user
WHERE userid > #{@last_imported_user_id}
ORDER BY userid
SQL
create_user_profiles(user_profiles) do |row|
{
user_id: user_id_from_imported_id(row[0]),
website: (URI.parse(row[1]).to_s rescue nil),
views: row[2],
}
end
end
def import_categories
puts "Importing categories..."
categories = mysql_query(<<-SQL
SELECT nodeid AS forumid, title, description, displayorder, parentid, urlident
FROM #{DB_PREFIX}node
WHERE parentid = #{ROOT_NODE}
AND nodeid > #{@last_imported_category_id}
UNION
SELECT nodeid, title, description, displayorder, parentid, urlident
FROM #{DB_PREFIX}node
WHERE contenttypeid = #{@channel_typeid}
AND nodeid > #{@last_imported_category_id}
SQL
).to_a
return if categories.empty?
parent_categories = categories.select { |c| c[4] == ROOT_NODE }
children_categories = categories.select { |c| c[4] != ROOT_NODE }
parent_category_ids = Set.new parent_categories.map { |c| c[0] }
puts "Importing parent categories..."
create_categories(parent_categories) do |row|
{
imported_id: row[0],
name: normalize_text(row[1]),
description: normalize_text(row[2]),
position: row[3],
slug: row[5]
}
end
puts "Importing children categories..."
create_categories(children_categories) do |row|
{
imported_id: row[0],
name: normalize_text(row[1]),
description: normalize_text(row[2]),
position: row[3],
parent_category_id: category_id_from_imported_id(row[4]),
slug: row[5]
}
end
end
def import_topics
puts "Importing topics..."
topics = mysql_stream <<-SQL
SELECT t.nodeid AS threadid, t.title, t.parentid AS forumid,
t.open, t.userid AS postuserid, t.publishdate AS dateline,
nv.count views, 1 AS visible, t.sticky
FROM #{DB_PREFIX}node t
LEFT JOIN #{DB_PREFIX}nodeview nv ON nv.nodeid = t.nodeid
WHERE t.parentid IN (SELECT nodeid from #{DB_PREFIX}node WHERE contenttypeid = #{@channel_typeid} )
AND t.contenttypeid = #{@text_typeid}
AND t.parentid != 7
AND (t.unpublishdate = 0 OR t.unpublishdate IS NULL)
AND t.approved = 1 AND t.showapproved = 1
AND t.nodeid > #{@last_imported_topic_id}
ORDER BY t.nodeid
SQL
create_topics(topics) do |row|
created_at = Time.zone.at(row[5])
title = normalize_text(row[1])
t = {
imported_id: row[0],
title: title,
category_id: category_id_from_imported_id(row[2]),
user_id: user_id_from_imported_id(row[4]),
closed: row[3] == 0,
created_at: created_at,
views: row[6],
visible: row[7] == 1,
}
t[:pinned_at] = created_at if row[8] == 1
t
end
end
def import_topic_first_posts
puts "Importing topic first posts..."
topics = mysql_stream <<-SQL
SELECT t.nodeid, t.parentid, t.userid,
t.publishdate, 1 AS visible, rawtext
FROM #{DB_PREFIX}node t
LEFT JOIN #{DB_PREFIX}nodeview nv ON nv.nodeid = t.nodeid
LEFT JOIN #{DB_PREFIX}text txt ON txt.nodeid = t.nodeid
WHERE t.parentid IN (SELECT nodeid from #{DB_PREFIX}node WHERE contenttypeid = #{@channel_typeid} )
AND t.contenttypeid = #{@text_typeid}
AND t.parentid != 7
AND (t.unpublishdate = 0 OR t.unpublishdate IS NULL)
AND t.approved = 1 AND t.showapproved = 1
AND t.nodeid > #{@last_imported_topic_id}
ORDER BY t.nodeid
SQL
create_posts(topics) do |row|
next unless topic_id = topic_id_from_imported_id(row[0])
post = {
imported_id: row[0],
user_id: user_id_from_imported_id(row[2]) || -1,
topic_id: topic_id,
created_at: Time.zone.at(row[3]),
hidden: row[4] != 1,
raw: preprocess_raw(row[5]),
}
post
end
end
def import_replies
puts "Importing replies..."
posts = mysql_stream <<-SQL
SELECT p.nodeid, p.userid, p.parentid,
CONVERT(CAST(rawtext AS BINARY)USING utf8),
p.publishdate, 1 AS visible, p.parentid
FROM #{DB_PREFIX}node p
LEFT JOIN #{DB_PREFIX}nodeview nv ON nv.nodeid = p.nodeid
LEFT JOIN #{DB_PREFIX}text txt ON txt.nodeid = p.nodeid
WHERE p.parentid NOT IN (SELECT nodeid FROM #{DB_PREFIX}node WHERE contenttypeid = #{@channel_typeid} )
AND p.contenttypeid = #{@text_typeid} AND p.nodeid > #{@last_imported_post_id}
ORDER BY p.nodeid
SQL
create_posts(posts) do |row|
next unless topic_id = topic_id_from_imported_id(row[2]) || topic_id_from_imported_id(row[0])
post = {
imported_id: row[0],
user_id: user_id_from_imported_id(row[1]) || -1,
topic_id: topic_id,
created_at: Time.zone.at(row[4]),
hidden: row[5] == 0,
raw: preprocess_raw(row[3]),
}
post
end
end
def import_likes
puts "Importing likes..."
@imported_likes = Set.new
post_likes = mysql_stream <<-SQL
SELECT nodeid, userid, dateline
FROM #{DB_PREFIX}reputation
WHERE nodeid > #{@last_imported_post_id}
ORDER BY nodeid
SQL
create_post_actions(post_likes) do |row|
post_id = post_id_from_imported_id(row[0])
user_id = user_id_from_imported_id(row[1])
next if post_id.nil? || user_id.nil?
next if @imported_likes.add?([post_id, user_id]).nil?
{
post_id: post_id,
user_id: user_id,
post_action_type_id: 2,
created_at: Time.zone.at(row[2])
}
end
end
def import_private_topics
puts "Importing private topics..."
topics = mysql_stream <<-SQL
SELECT t.nodeid, t.title, t.userid, t.publishdate AS dateline
FROM #{DB_PREFIX}node t
LEFT JOIN #{DB_PREFIX}privatemessage pm ON pm.nodeid = t.nodeid
WHERE pm.msgtype = 'message'
AND t.parentid = 8
AND t.nodeid > #{@last_imported_private_topic_id - PRIVATE_OFFSET}
ORDER BY t.nodeid
SQL
create_topics(topics) do |row|
title = row[1] || "No title given"
{
archetype: Archetype.private_message,
imported_id: row[0] + PRIVATE_OFFSET,
title: title,
user_id: user_id_from_imported_id(row[2]),
created_at: Time.zone.at(row[3]),
}
end
end
def import_topic_allowed_users
puts "Importing topic allowed users..."
allowed_users_sql = <<-SQL
SELECT p.nodeid, p.userid, p.parentid
FROM #{DB_PREFIX}node p
LEFT JOIN #{DB_PREFIX}privatemessage pm ON pm.nodeid = p.nodeid
WHERE pm.msgtype = 'message'
ORDER BY p.nodeid
SQL
added = 0
users_added = Set.new
create_topic_allowed_users(mysql_stream(allowed_users_sql)) do |row|
next unless topic_id = topic_id_from_imported_id(row[0] + PRIVATE_OFFSET) || topic_id_from_imported_id(row[2] + PRIVATE_OFFSET)
next unless user_id = user_id_from_imported_id(row[1])
next if users_added.add?([topic_id, user_id]).nil?
added += 1
{
topic_id: topic_id,
user_id: user_id,
}
end
puts '', "Added #{added} topic allowed users records."
end
def import_private_first_posts
puts "Importing private message first posts..."
posts = mysql_stream <<-SQL
SELECT p.nodeid, p.userid,
CONVERT(CAST(rawtext AS BINARY)USING utf8),
p.publishdate
FROM #{DB_PREFIX}node p
LEFT JOIN #{DB_PREFIX}text txt ON txt.nodeid = p.nodeid
LEFT JOIN #{DB_PREFIX}privatemessage pm ON pm.nodeid = p.nodeid
WHERE pm.msgtype = 'message'
AND p.parentid = 8
AND p.nodeid > #{@last_imported_private_post_id - PRIVATE_OFFSET}
ORDER BY p.nodeid
SQL
create_posts(posts) do |row|
{
imported_id: row[0] + PRIVATE_OFFSET,
topic_id: topic_id_from_imported_id(row[0] + PRIVATE_OFFSET),
user_id: user_id_from_imported_id(row[1]),
created_at: Time.zone.at(row[3]),
raw: preprocess_raw(row[2]),
}
end
end
def import_private_replies
puts "Importing private replies..."
posts = mysql_stream <<-SQL
SELECT p.nodeid, p.userid, p.parentid,
CONVERT(CAST(rawtext AS BINARY)USING utf8),
p.publishdate
FROM #{DB_PREFIX}node p
LEFT JOIN #{DB_PREFIX}text txt ON txt.nodeid = p.nodeid
LEFT JOIN #{DB_PREFIX}privatemessage pm ON pm.nodeid = p.nodeid
WHERE pm.msgtype = 'message'
AND p.parentid != 8
AND p.nodeid > #{@last_imported_private_post_id - PRIVATE_OFFSET}
ORDER BY p.nodeid
SQL
create_posts(posts) do |row|
{
imported_id: row[0] + PRIVATE_OFFSET,
topic_id: topic_id_from_imported_id(row[2] + PRIVATE_OFFSET),
user_id: user_id_from_imported_id(row[1]),
created_at: Time.zone.at(row[4]),
raw: preprocess_raw(row[3]),
}
end
end
def create_permalinks
puts '', 'creating permalinks...', ''
# add permalink normalizations to site settings
# EVERYTHING: /.*\/([\w-]+)$/\1 -- selects the last segment of the URL
# and matches in the permalink table
# create permalinks
Topic.listable_topics.find_each do |topic|
pcf = topic&.custom_fields
if pcf && pcf["import_id"]
id = pcf["import_id"]
url = "#{id}-#{topic.slug}"
Permalink.create(url: url, topic_id: topic.id) unless permalink_exists(url)
end
end
Category.find_each do |cat|
ccf = cat&.custom_fields
if ccf && ccf["import_id"]
url = cat.slug
Permalink.create(url: url, category_id: cat.id) unless permalink_exists(url)
end
end
end
def permalink_exists(url)
Permalink.find_by(url: url)
end
def check_database_for_attachment(row)
# check if attachment resides in the database & try to retrieve
if row[4].to_i == 0
puts "Attachment file #{row.inspect} doesn't exist"
return nil
end
tmpfile = 'attach_' + row[6].to_s
filename = File.join('/tmp/', tmpfile)
File.open(filename, 'wb') { |f| f.write(row[5]) }
filename
end
def find_upload(post, opts = {})
if opts[:node_id].present?
sql = "SELECT a.nodeid, n.parentid, a.filename, fd.userid, LENGTH(fd.filedata), filedata, fd.filedataid
FROM #{DB_PREFIX}attach a
LEFT JOIN #{DB_PREFIX}filedata fd ON fd.filedataid = a.filedataid
LEFT JOIN #{DB_PREFIX}node n ON n.nodeid = a.nodeid
WHERE a.nodeid = #{opts[:node_id]}"
elsif opts[:attachment_id].present?
sql = "SELECT a.nodeid, n.parentid, a.filename, fd.userid, LENGTH(fd.filedata), filedata, fd.filedataid
FROM #{DB_PREFIX}attachment a
LEFT JOIN #{DB_PREFIX}filedata fd ON fd.filedataid = a.filedataid
LEFT JOIN #{DB_PREFIX}node n ON n.nodeid = a.nodeid
WHERE a.attachmentid = #{opts[:attachment_id]}"
end
results = mysql_query(sql)
unless row = results.first
puts "Couldn't find attachment record -- nodeid/filedataid = #{opts[:attachment_id] || opts[:filedata_id]} / post.id = #{post.id}"
return nil
end
attachment_id = row[6]
user_id = row[3]
db_filename = row[2]
filename = File.join(ATTACH_DIR, user_id.to_s.split('').join('/'), "#{attachment_id}.attach")
real_filename = db_filename
real_filename.prepend SecureRandom.hex if real_filename[0] == '.'
unless File.exists?(filename)
filename = check_database_for_attachment(row) if filename.blank?
return nil if filename.nil?
end
upload = create_upload(post.user_id, filename, real_filename)
if upload.nil? || !upload.valid?
puts "Upload not valid"
puts upload.errors.inspect if upload
return
end
[upload, real_filename]
rescue Mysql2::Error => e
puts "SQL Error"
puts e.message
puts sql
end
def import_attachments
puts '', 'importing attachments...'
# add extensions to authorized setting
#ext = mysql_query("SELECT GROUP_CONCAT(DISTINCT(extension)) exts FROM #{DB_PREFIX}filedata").first[0].split(',')
#SiteSetting.authorized_extensions = (SiteSetting.authorized_extensions.split("|") + ext).uniq.join("|")
RateLimiter.disable
current_count = 0
total_count = Post.all.count
success_count = 0
fail_count = 0
# we need to match an older and newer style for inline attachment import
# new style matches the nodeid in the attach table
# old style matches the filedataid in attach/filedata tables
# if the site is very old, there may be multiple different attachment syntaxes used in posts
attachment_regex = /\[attach[^\]]*\].*\"data-attachmentid\":"?(\d+)"?,?.*\[\/attach\]/i
attachment_regex_oldstyle = /\[attach[^\]]*\](\d+)\[\/attach\]/i
Post.find_each do |post|
current_count += 1
print_status current_count, total_count
pcf = post.custom_fields
next if pcf && pcf["import_attachments"] && pcf["import_attachments"] == true
new_raw = post.raw.dup
# look for new style attachments
new_raw.gsub!(attachment_regex) do |s|
matches = attachment_regex.match(s)
node_id = matches[1]
upload, filename = find_upload(post, { node_id: node_id })
unless upload
fail_count += 1
next
end
html_for_upload(upload, filename)
end
# look for old style attachments
new_raw.gsub!(attachment_regex_oldstyle) do |s|
matches = attachment_regex_oldstyle.match(s)
attachment_id = matches[1]
upload, filename = find_upload(post, { attachment_id: attachment_id })
unless upload
fail_count += 1
next
end
html_for_upload(upload, filename)
end
if new_raw != post.raw
post.raw = new_raw
post.save(validate: false)
PostCustomField.create(post_id: post.id, name: "import_attachments", value: true)
success_count += 1
end
end
puts "", "imported #{success_count} attachments... failed: #{fail_count}"
RateLimiter.enable
end
def extract_pm_title(title)
normalize_text(title).scrub.gsub(/^Re\s*:\s*/i, "")
end
def parse_birthday(birthday)
return if birthday.blank?
date_of_birth = Date.strptime(birthday.gsub(/[^\d-]+/, ""), "%m-%d-%Y") rescue nil
return if date_of_birth.nil?
date_of_birth.year < 1904 ? Date.new(1904, date_of_birth.month, date_of_birth.day) : date_of_birth
end
def preprocess_raw(raw)
return "" if raw.nil?
raw = normalize_text(raw)
raw = raw.dup
# [PLAINTEXT]...[/PLAINTEXT]
raw.gsub!(/\[\/?PLAINTEXT\]/i, "\n\n```\n\n")
# [FONT=font]...[/FONT]
raw.gsub!(/\[FONT=\w*\]/im, "")
raw.gsub!(/\[\/FONT\]/im, "")
# @[URL=<user_profile>]<username>[/URL]
# [USER=id]username[/USER]
# [MENTION=id]username[/MENTION]
raw.gsub!(/@\[URL=\"\S+\"\]([\w\s]+)\[\/URL\]/i) { "@#{$1.gsub(" ", "_")}" }
raw.gsub!(/\[USER=\"\d+\"\]([\S]+)\[\/USER\]/i) { "@#{$1.gsub(" ", "_")}" }
raw.gsub!(/\[MENTION=\d+\]([\S]+)\[\/MENTION\]/i) { "@#{$1.gsub(" ", "_")}" }
# [IMG2=JSON]{..."src":"<url>"}[/IMG2]
raw.gsub!(/\[img2[^\]]*\].*\"src\":\"?([\w\\\/:\.\-;%]*)\"?}.*\[\/img2\]/i) { "\n#{CGI::unescape($1)}\n" }
# [TABLE]...[/TABLE]
raw.gsub!(/\[TABLE=\\"[\w:\-\s,]+\\"\]/i, "")
raw.gsub!(/\[\/TABLE\]/i, "")
# [HR]...[/HR]
raw.gsub(/\[HR\]\s*\[\/HR\]/im, "---")
# [VIDEO=youtube_share;<id>]...[/VIDEO]
# [VIDEO=vimeo;<id>]...[/VIDEO]
raw.gsub!(/\[VIDEO=YOUTUBE_SHARE;([^\]]+)\].*?\[\/VIDEO\]/i) { "\nhttps://www.youtube.com/watch?v=#{$1}\n" }
raw.gsub!(/\[VIDEO=VIMEO;([^\]]+)\].*?\[\/VIDEO\]/i) { "\nhttps://vimeo.com/#{$1}\n" }
raw
end
def print_status(current, max, start_time = nil)
if start_time.present?
elapsed_seconds = Time.now - start_time
elements_per_minute = '[%.0f items/min] ' % [current / elapsed_seconds.to_f * 60]
else
elements_per_minute = ''
end
print "\r%9d / %d (%5.1f%%) %s" % [current, max, current / max.to_f * 100, elements_per_minute]
end
def mysql_stream(sql)
@client.query(sql, stream: true)
end
def mysql_query(sql)
@client.query(sql)
end
end
BulkImport::VBulletin5.new.run