Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Import avatars and likes in the Zendesk AP importer 

Co-authored-by: Justin DiRose <justin@justindirose.com>
This commit is contained in:
Gerhard Schlager 2019-08-13 17:24:22 +02:00
parent 2fbafd077c
commit 888b635cfc
2 changed files with 176 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
script/import_scripts/base/generic_database.rb
View File

@ -14,6 +14,7 @@ module ImportScripts
configure_database
create_category_table
create_like_table
create_user_table
create_topic_table
create_post_table
@ -34,37 +35,64 @@ module ImportScripts
SQL
end
def insert_like(like)
@db.execute(<<-SQL, prepare(like))
INSERT OR REPLACE INTO like (id, user_id, post_id, topic)
VALUES (:id, :user_id, :post_id, :topic)
SQL
end
def insert_topic(topic)
like_user_ids = topic.delete(:like_user_ids)
attachments = topic.delete(:attachments)
topic[:upload_count] = attachments&.size || 0
@db.execute(<<-SQL, prepare(topic))
INSERT OR REPLACE INTO topic (id, title, raw, category_id, closed, user_id, created_at, url, upload_count)
VALUES (:id, :title, :raw, :category_id, :closed, :user_id, :created_at, :url, :upload_count)
SQL
attachments&.each do |attachment|
@db.execute(<<-SQL, topic_id: topic[:id], path: attachment)
INSERT OR REPLACE INTO topic_upload (topic_id, path)
VALUES (:topic_id, :path)
@db.transaction do
@db.execute(<<-SQL, prepare(topic))
INSERT OR REPLACE INTO topic (id, title, raw, category_id, closed, user_id, created_at, url, upload_count)
VALUES (:id, :title, :raw, :category_id, :closed, :user_id, :created_at, :url, :upload_count)
SQL
attachments&.each do |attachment|
@db.execute(<<-SQL, topic_id: topic[:id], path: attachment)
INSERT OR REPLACE INTO topic_upload (topic_id, path)
VALUES (:topic_id, :path)
SQL
end
like_user_ids&.each do |user_id|
@db.execute(<<-SQL, topic_id: topic[:id], user_id: user_id)
INSERT OR REPLACE INTO like (topic_id, user_id)
VALUES (:topic_id, :user_id)
SQL
end
end
end
def insert_post(post)
like_user_ids = post.delete(:like_user_ids)
attachments = post.delete(:attachments)
post[:upload_count] = attachments&.size || 0
@db.execute(<<-SQL, prepare(post))
INSERT OR REPLACE INTO post (id, raw, topic_id, user_id, created_at, reply_to_post_id, url, upload_count)
VALUES (:id, :raw, :topic_id, :user_id, :created_at, :reply_to_post_id, :url, :upload_count)
SQL
attachments&.each do |attachment|
@db.execute(<<-SQL, post_id: post[:id], path: attachment)
INSERT OR REPLACE INTO post_upload (post_id, path)
VALUES (:post_id, :path)
@db.transaction do
@db.execute(<<-SQL, prepare(post))
INSERT OR REPLACE INTO post (id, raw, topic_id, user_id, created_at, reply_to_post_id, url, upload_count)
VALUES (:id, :raw, :topic_id, :user_id, :created_at, :reply_to_post_id, :url, :upload_count)
SQL
attachments&.each do |attachment|
@db.execute(<<-SQL, post_id: post[:id], path: attachment)
INSERT OR REPLACE INTO post_upload (post_id, path)
VALUES (:post_id, :path)
SQL
end
like_user_ids&.each do |user_id|
@db.execute(<<-SQL, post_id: post[:id], user_id: user_id)
INSERT OR REPLACE INTO like (post_id, user_id)
VALUES (:post_id, :user_id)
SQL
end
end
end
@ -196,6 +224,25 @@ module ImportScripts
SQL
end
def count_likes
@db.get_first_value(<<-SQL)
SELECT COUNT(*)
FROM like
SQL
end
def fetch_likes(last_row_id)
rows = @db.execute(<<-SQL, last_row_id)
SELECT ROWID AS rowid, *
FROM like
WHERE ROWID > :last_row_id
ORDER BY ROWID
LIMIT #{@batch_size}
SQL
add_last_column_value(rows, 'rowid')
end
def execute_sql(sql)
@db.execute(sql)
end
@ -227,6 +274,16 @@ module ImportScripts
SQL
end
def create_like_table
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS like (
user_id #{key_data_type} NOT NULL,
topic_id #{key_data_type},
post_id #{key_data_type}
)
SQL
end
def create_user_table
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user (

121
script/import_scripts/zendesk_api.rb
View File

@ -30,12 +30,22 @@ class ImportScripts::ZendeskApi < ImportScripts::Base
import_users
import_topics
import_posts
import_likes
end
def fetch_from_api
fetch_categories
fetch_topics
fetch_posts
fetch_users
@db.sort_posts_by_created_at
end
def fetch_categories
puts '', 'fetching categories...'
get_from_api('/api/v2/community/topics.json', 'topics') do |row|
get_from_api('/api/v2/community/topics.json', 'topics', show_status: true) do |row|
@db.insert_category(
id: row['id'],
name: row['name'],
@ -44,10 +54,16 @@ class ImportScripts::ZendeskApi < ImportScripts::Base
url: row['html_url']
)
end
end
def fetch_topics
puts '', 'fetching topics...'
get_from_api('/api/v2/community/posts.json', 'posts') do |row|
get_from_api('/api/v2/community/posts.json', 'posts', show_status: true) do |row|
if row['vote_count'] > 0
like_user_ids = fetch_likes("/api/v2/community/posts/#{row['id']}/votes.json")
end
@db.insert_topic(
id: row['id'],
title: row['title'],
@ -56,11 +72,15 @@ class ImportScripts::ZendeskApi < ImportScripts::Base
closed: row['closed'],
user_id: row['author_id'],
created_at: row['created_at'],
url: row['html_url']
url: row['html_url'],
like_user_ids: like_user_ids
)
end
end
def fetch_posts
puts '', 'fetching posts...'
current_count = 0
total_count = @db.count_topics
start_time = Time.now
last_id = ''
@ -69,49 +89,72 @@ class ImportScripts::ZendeskApi < ImportScripts::Base
rows, last_id = @db.fetch_topics(last_id)
break if rows.empty?
print_status(offset, total_count, start_time)
rows.each do |topic_row|
get_from_api("/api/v2/community/posts/#{topic_row['id']}/comments.json", 'comments', show_status: false) do |row|
get_from_api("/api/v2/community/posts/#{topic_row['id']}/comments.json", 'comments') do |row|
if row['vote_count'] > 0
like_user_ids = fetch_likes("/api/v2/community/posts/#{topic_row['id']}/comments/#{row['id']}/votes.json")
end
@db.insert_post(
id: row['id'],
raw: row['body'],
topic_id: topic_row['id'],
user_id: row['author_id'],
created_at: row['created_at'],
url: row['html_url']
url: row['html_url'],
like_user_ids: like_user_ids
)
end
current_count += 1
print_status(current_count, total_count, start_time)
end
end
end
def fetch_users
puts '', 'fetching users...'
results = @db.execute_sql("SELECT user_id FROM topic")
user_ids = results.map { |h| h['user_id']&.to_i }
results = @db.execute_sql("SELECT user_id FROM post")
user_ids += results.map { |h| h['user_id']&.to_i }
user_ids.uniq!
user_ids.sort!
user_ids = @db.execute_sql(<<~SQL).map { |row| row['user_id'] }
SELECT user_id FROM topic
UNION
SELECT user_id FROM post
UNION
SELECT user_id FROM like
SQL
total_users = user_ids.size
current_count = 0
total_count = user_ids.size
start_time = Time.now
while !user_ids.empty?
print_status(total_users - user_ids.size, total_users, start_time)
get_from_api("/api/v2/users/show_many.json?ids=#{user_ids.shift(50).join(',')}", 'users', show_status: false) do |row|
get_from_api("/api/v2/users/show_many.json?ids=#{user_ids.shift(50).join(',')}", 'users') do |row|
@db.insert_user(
id: row['id'],
email: row['email'],
name: row['name'],
created_at: row['created_at'],
last_seen_at: row['last_login_at'],
active: row['active']
active: row['active'],
avatar_path: row['photo'].present? ? row['photo']['content_url'] : nil
)
current_count += 1
print_status(current_count, total_count, start_time)
end
end
end
def fetch_likes(url)
user_ids = []
get_from_api(url, 'votes') do |row|
if row['id'].present? && row['value'] == 1
user_ids << row['user_id']
end
end
@db.sort_posts_by_created_at
user_ids
end
def import_categories
@ -150,7 +193,12 @@ class ImportScripts::ZendeskApi < ImportScripts::Base
name: row['name'],
created_at: row['created_at'],
last_seen_at: row['last_seen_at'],
active: row['active'] == 1
active: row['active'] == 1,
post_create_action: proc do |user|
if row['avatar_path'].present?
UserAvatar.import_url_for_user(row['avatar_path'], user) rescue nil
end
end
}
end
end
@ -222,6 +270,38 @@ class ImportScripts::ZendeskApi < ImportScripts::Base
end
end
def import_likes
puts "", "importing likes..."
start_time = Time.now
current_count = 0
total_count = @db.count_likes
last_row_id = 0
batches do |offset|
rows, last_row_id = @db.fetch_likes(last_row_id)
break if rows.empty?
rows.each do |row|
import_id = row['topic_id'] ? import_topic_id(row['topic_id']) : row['post_id']
post = Post.find_by(id: post_id_from_imported_post_id(import_id)) if import_id
user = User.find_by(id: user_id_from_imported_user_id(row['user_id']))
if post && user
begin
PostActionCreator.like(user, post) if user && post
rescue => e
puts "error acting on post #{e}"
end
else
puts "Skipping Like from #{row['user_id']} on topic #{row['topic_id']} / post #{row['post_id']}"
end
current_count += 1
print_status(current_count, total_count, start_time)
end
end
end
def normalize_raw(raw)
raw = raw.gsub('\n', '')
raw = ReverseMarkdown.convert(raw)
@ -256,7 +336,7 @@ class ImportScripts::ZendeskApi < ImportScripts::Base
end
end
def get_from_api(path, array_name, show_status: true)
def get_from_api(path, array_name, show_status: false)
url = "#{@source_url}#{path}"
start_time = Time.now
@ -298,6 +378,7 @@ class ImportScripts::ZendeskApi < ImportScripts::Base
end
end
end
end
unless ARGV.length == 4 && Dir.exist?(ARGV[1])