271 lines
7.3 KiB
Ruby
271 lines
7.3 KiB
Ruby
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
|
|
require 'pg'
|
|
|
|
class ImportScripts::MyAskBot < ImportScripts::Base
|
|
# CHANGE THESE BEFORE RUNNING THE IMPORTER
|
|
|
|
BATCH_SIZE = 1000
|
|
|
|
OLD_SITE = "ask.cvxr.com"
|
|
DB_NAME = "cvxforum"
|
|
DB_USER = "cvxforum"
|
|
DB_PORT = 5432
|
|
DB_HOST = "ask.cvxr.com"
|
|
DB_PASS = 'yeah, right'
|
|
|
|
# A list of categories to create. Any post with one of these tags will be
|
|
# assigned to that category. Ties are broken by list orer.
|
|
CATEGORIES = [ 'Nonconvex', 'TFOCS', 'MIDCP', 'FAQ' ]
|
|
|
|
def initialize
|
|
super
|
|
|
|
@thread_parents = {}
|
|
@tagmap = []
|
|
@td = PG::TextDecoder::TimestampWithTimeZone.new
|
|
@client = PG.connect(
|
|
:dbname => DB_NAME,
|
|
:host => DB_HOST,
|
|
:port => DB_PORT,
|
|
:user => DB_USER,
|
|
:password => DB_PASS
|
|
)
|
|
end
|
|
|
|
def execute
|
|
create_cats
|
|
import_users
|
|
read_tags
|
|
import_posts
|
|
import_replies
|
|
post_process_posts
|
|
end
|
|
|
|
def create_cats
|
|
puts "", "creating categories"
|
|
CATEGORIES.each do |cat|
|
|
unless Category.where("LOWER(name) = ?", cat.downcase).first
|
|
Category.new(name: cat, user_id: -1).save!
|
|
end
|
|
end
|
|
end
|
|
|
|
def read_tags
|
|
puts "", "reading thread tags..."
|
|
|
|
tag_count = @client.exec(<<-SQL
|
|
SELECT COUNT(A.id)
|
|
FROM askbot_thread_tags A
|
|
JOIN tag B
|
|
ON A.tag_id = B.id
|
|
WHERE A.tag_id > 0
|
|
SQL
|
|
)[0]["count"]
|
|
|
|
tags_done = 0
|
|
batches(BATCH_SIZE) do |offset|
|
|
tags = @client.exec(<<-SQL
|
|
SELECT A.thread_id, B.name
|
|
FROM askbot_thread_tags A
|
|
JOIN tag B
|
|
ON A.tag_id = B.id
|
|
WHERE A.tag_id > 0
|
|
LIMIT #{BATCH_SIZE}
|
|
OFFSET #{offset}
|
|
SQL
|
|
)
|
|
break if tags.ntuples() < 1
|
|
tags.each do |tag|
|
|
tid = tag["thread_id"].to_i
|
|
tnm = tag["name"].downcase
|
|
if @tagmap[tid]
|
|
@tagmap[tid].push( tnm )
|
|
else
|
|
@tagmap[tid] = [ tnm ]
|
|
end
|
|
tags_done += 1
|
|
print_status tags_done, tag_count
|
|
end
|
|
end
|
|
end
|
|
|
|
def import_users
|
|
puts "", "importing users"
|
|
|
|
total_count = @client.exec(<<-SQL
|
|
SELECT COUNT(id)
|
|
FROM auth_user
|
|
SQL
|
|
)[0]["count"]
|
|
|
|
batches(BATCH_SIZE) do |offset|
|
|
users = @client.query(<<-SQL
|
|
SELECT id, username, email, is_staff, date_joined, last_seen, real_name, website, location, about
|
|
FROM auth_user
|
|
ORDER BY date_joined
|
|
LIMIT #{BATCH_SIZE}
|
|
OFFSET #{offset}
|
|
SQL
|
|
)
|
|
|
|
break if users.ntuples() < 1
|
|
|
|
create_users(users, total: total_count, offset: offset) do |user|
|
|
{
|
|
id: user["id"],
|
|
username: user["username"],
|
|
email: user["email"] || (SecureRandom.hex << "@domain.com"),
|
|
admin: user["is_staff"],
|
|
created_at: Time.zone.at(@td.decode(user["date_joined"])),
|
|
last_seen_at: Time.zone.at(@td.decode(user["last_seen"])),
|
|
name: user["real_name"],
|
|
website: user["website"],
|
|
location: user["location"],
|
|
}
|
|
end
|
|
end
|
|
end
|
|
|
|
def import_posts
|
|
puts "", "importing questions..."
|
|
|
|
post_count = @client.exec(<<-SQL
|
|
SELECT COUNT(A.id)
|
|
FROM askbot_post A
|
|
JOIN askbot_thread B
|
|
ON A.thread_id = B.id
|
|
WHERE NOT B.closed AND A.post_type='question'
|
|
SQL
|
|
)[0]["count"]
|
|
|
|
batches(BATCH_SIZE) do |offset|
|
|
posts = @client.exec(<<-SQL
|
|
SELECT A.id, A.author_id, A.added_at, A.text, A.thread_id, B.title
|
|
FROM askbot_post A
|
|
JOIN askbot_thread B
|
|
ON A.thread_id = B.id
|
|
WHERE NOT B.closed AND A.post_type = 'question'
|
|
ORDER BY A.added_at
|
|
LIMIT #{BATCH_SIZE}
|
|
OFFSET #{offset}
|
|
SQL
|
|
)
|
|
|
|
break if posts.ntuples() < 1
|
|
|
|
create_posts(posts, total: post_count, offset: offset) do |post|
|
|
pid = post["id"]
|
|
tid = post["thread_id"].to_i
|
|
tags = @tagmap[tid]
|
|
cat = nil
|
|
if tags
|
|
CATEGORIES.each do |cname|
|
|
next unless tags.include?(cname.downcase)
|
|
cat = cname
|
|
break
|
|
end
|
|
end
|
|
@thread_parents[tid] = pid
|
|
{
|
|
id: pid,
|
|
title: post["title"],
|
|
category: cat,
|
|
custom_fields: {import_id: pid, import_thread_id: tid, import_tags: tags},
|
|
user_id: user_id_from_imported_user_id(post["author_id"]) || Discourse::SYSTEM_USER_ID,
|
|
created_at: Time.zone.at(@td.decode(post["added_at"])),
|
|
raw: post["text"],
|
|
}
|
|
end
|
|
end
|
|
end
|
|
|
|
def import_replies
|
|
puts "", "importing answers and comments..."
|
|
|
|
post_count = @client.exec(<<-SQL
|
|
SELECT COUNT(A.id)
|
|
FROM askbot_post A
|
|
JOIN askbot_thread B
|
|
ON A.thread_id = B.id
|
|
WHERE NOT B.closed AND A.post_type<>'question'
|
|
SQL
|
|
)[0]["count"]
|
|
|
|
batches(BATCH_SIZE) do |offset|
|
|
posts = @client.exec(<<-SQL
|
|
SELECT A.id, A.author_id, A.added_at, A.text, A.thread_id, B.title
|
|
FROM askbot_post A
|
|
JOIN askbot_thread B
|
|
ON A.thread_id = B.id
|
|
WHERE NOT B.closed AND A.post_type <> 'question'
|
|
ORDER BY A.added_at
|
|
LIMIT #{BATCH_SIZE}
|
|
OFFSET #{offset}
|
|
SQL
|
|
)
|
|
|
|
break if posts.ntuples() < 1
|
|
|
|
create_posts(posts, total: post_count, offset: offset) do |post|
|
|
tid = post["thread_id"].to_i
|
|
next unless thread = @thread_parents[tid]
|
|
next unless parent = topic_lookup_from_imported_post_id(thread)
|
|
pid = post["id"]
|
|
{
|
|
id: pid,
|
|
topic_id: parent[:topic_id],
|
|
custom_fields: {import_id: pid},
|
|
user_id: user_id_from_imported_user_id(post["author_id"]) || Discourse::SYSTEM_USER_ID,
|
|
created_at: Time.zone.at(@td.decode(post["added_at"])),
|
|
raw: post["text"]
|
|
}
|
|
end
|
|
end
|
|
end
|
|
|
|
def post_process_posts
|
|
puts "", "Postprocessing posts..."
|
|
current = 0
|
|
max = Post.count
|
|
# Rewrite internal links; e.g.
|
|
# ask.cvxr.com/question/(\d+)/[^'"}]*
|
|
# I am sure this is incomplete, but we didn't make heavy use of internal
|
|
# links on our site.
|
|
tmp = Regexp.quote("http://" << OLD_SITE)
|
|
r1 = /"(#{tmp})?\/question\/(\d+)\/[a-zA-Z-]*\/?"/
|
|
r2 = /\((#{tmp})?\/question\/(\d+)\/[a-zA-Z-]*\/?\)/
|
|
r3 = /<?#tmp\/question\/(\d+)\/[a-zA-Z-]*\/?>?/
|
|
Post.find_each do |post|
|
|
raw = post.raw.gsub(r1) do
|
|
if topic = topic_lookup_from_imported_post_id($2)
|
|
"\"#{topic[:url]}\""
|
|
else
|
|
$&
|
|
end
|
|
end
|
|
raw = raw.gsub(r2) do
|
|
if topic = topic_lookup_from_imported_post_id($2)
|
|
"(#{topic[:url]})"
|
|
else
|
|
$&
|
|
end
|
|
end
|
|
raw = raw.gsub(r3) do
|
|
if topic = topic_lookup_from_imported_post_id($1)
|
|
trec = Topic.find_by(id: topic[:topic_id])
|
|
"[#{trec.title}](#{topic[:url]})"
|
|
else
|
|
$&
|
|
end
|
|
end
|
|
if raw != post.raw
|
|
post.raw = raw
|
|
post.save
|
|
end
|
|
print_status(current += 1, max)
|
|
end
|
|
end
|
|
end
|
|
|
|
ImportScripts::MyAskBot.new.perform
|