discourse/script/import_scripts/base/generic_database.rb

225 lines
5.3 KiB
Ruby

require 'sqlite3'
module ImportScripts
class GenericDatabase
def initialize(directory, batch_size:, recreate: false)
filename = "#{directory}/index.db"
File.delete(filename) if recreate && File.exists?(filename)
@db = SQLite3::Database.new(filename, results_as_hash: true)
@batch_size = batch_size
configure_database
create_category_table
create_user_table
create_topic_table
create_post_table
end
def insert_category(category)
@db.execute(<<-SQL, prepare(category))
INSERT OR REPLACE INTO category (id, name, description, position, url)
VALUES (:id, :name, :description, :position, :url)
SQL
end
def insert_user(user)
@db.execute(<<-SQL, prepare(user))
INSERT OR REPLACE INTO user (id, email, username, name, created_at, last_seen_at, active)
VALUES (:id, :email, :username, :name, :created_at, :last_seen_at, :active)
SQL
end
def insert_topic(topic)
@db.execute(<<-SQL, prepare(topic))
INSERT OR REPLACE INTO topic (id, title, raw, category_id, closed, user_id, created_at, url)
VALUES (:id, :title, :raw, :category_id, :closed, :user_id, :created_at, :url)
SQL
end
def insert_post(post)
@db.execute(<<-SQL, prepare(post))
INSERT OR REPLACE INTO post (id, raw, topic_id, user_id, created_at, reply_to_post_id, url)
VALUES (:id, :raw, :topic_id, :user_id, :created_at, :reply_to_post_id, :url)
SQL
end
def sort_posts_by_created_at
@db.execute 'DELETE FROM post_order'
@db.execute <<-SQL
INSERT INTO post_order (id)
SELECT id
FROM post
ORDER BY created_at, topic_id, id
SQL
end
def fetch_categories
@db.execute(<<-SQL)
SELECT *
FROM category
ORDER BY position, name
SQL
end
def count_users
@db.get_first_value(<<-SQL)
SELECT COUNT(*)
FROM user
SQL
end
def fetch_users(last_id)
rows = @db.execute(<<-SQL, last_id)
SELECT *
FROM user
WHERE id > :last_id
ORDER BY id
LIMIT #{@batch_size}
SQL
add_last_column_value(rows, 'id')
end
def count_topics
@db.get_first_value(<<-SQL)
SELECT COUNT(*)
FROM topic
SQL
end
def fetch_topics(last_id)
rows = @db.execute(<<-SQL, last_id)
SELECT *
FROM topic
WHERE id > :last_id
ORDER BY id
LIMIT #{@batch_size}
SQL
add_last_column_value(rows, 'id')
end
def count_posts
@db.get_first_value(<<-SQL)
SELECT COUNT(*)
FROM post
SQL
end
def fetch_posts(last_row_id)
rows = @db.execute(<<-SQL, last_row_id)
SELECT o.ROWID, p.*
FROM post p
JOIN post_order o USING (id)
WHERE o.ROWID > :last_row_id
ORDER BY o.ROWID
LIMIT #{@batch_size}
SQL
add_last_column_value(rows, 'rowid')
end
def execute_sql(sql)
@db.execute(sql)
end
def get_first_value(sql)
@db.get_first_value(sql)
end
private
def configure_database
@db.execute 'PRAGMA journal_mode = OFF'
@db.execute 'PRAGMA locking_mode = EXCLUSIVE'
end
def create_category_table
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS category (
id TEXT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
position INTEGER,
url TEXT
)
SQL
end
def create_user_table
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user (
id TEXT NOT NULL PRIMARY KEY,
email TEXT,
username TEXT,
name TEXT,
created_at DATETIME,
last_seen_at DATETIME,
active BOOLEAN NOT NULL DEFAULT true
)
SQL
end
def create_topic_table
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic (
id TEXT NOT NULL PRIMARY KEY,
title TEXT,
raw TEXT,
category_id TEXT NOT NULL,
closed BOOLEAN NOT NULL DEFAULT false,
user_id TEXT NOT NULL,
created_at DATETIME,
url TEXT
)
SQL
@db.execute 'CREATE INDEX IF NOT EXISTS topic_by_user_id ON topic (user_id)'
end
def create_post_table
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS post (
id TEXT NOT NULL PRIMARY KEY,
raw TEXT,
topic_id TEXT NOT NULL,
user_id TEXT NOT NULL,
created_at DATETIME,
reply_to_post_id TEXT,
url TEXT
)
SQL
@db.execute 'CREATE INDEX IF NOT EXISTS post_by_user_id ON post (user_id)'
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS post_order (
id TEXT NOT NULL PRIMARY KEY
)
SQL
end
def prepare(hash)
hash.each do |key, value|
if value.is_a?(TrueClass) || value.is_a?(FalseClass)
hash[key] = value ? 1 : 0
elsif value.is_a?(Date)
hash[key] = value.to_s
end
end
end
def add_last_column_value(rows, *last_columns)
return rows if last_columns.empty?
result = [rows]
last_row = rows.last
last_columns.each { |column| result.push(last_row ? last_row[column] : nil) }
result
end
end
end