Merge pull request #3988 from mcwumbly/quandora-importer

Add export/import scripts for Quandora
This commit is contained in:
Neil Lalonde 2016-02-03 15:20:21 -05:00
commit 19cdb38b20
9 changed files with 719 additions and 0 deletions

View File

@ -0,0 +1,21 @@
To get started, copy the config.ex.yml to config.yml, and then update the properties for your Quandora instance.
```
domain: 'my-quandora-domain'
username: 'my-quandora-username'
password: 'my-quandora-password'
```
Create the directory for the json files to export: `mkdir output`
Then run `ruby export.rb /path/to/config.yml`
To import, run `ruby import.rb`
To run tests, include id's for a KB and Question that includes answers and comments
```
kb_id: 'some-kb-id'
question_id: 'some-question-id'
```

View File

@ -0,0 +1,30 @@
require 'yaml'
require_relative 'quandora_api'
def load_config file
config = YAML::load_file(File.join(__dir__, file))
@domain = config['domain']
@username = config['username']
@password = config['password']
end
def export
api = QuandoraApi.new @domain, @username, @password
bases = api.list_bases
bases.each do |base|
question_list = api.list_questions base['objectId'], 1000
question_list.each do |q|
question_id = q['uid']
question = api.get_question question_id
File.open("output/#{question_id}.json", 'w') do |f|
puts question['title']
f.write question.to_json
f.close
end
end
end
end
load_config ARGV.shift
export

View File

@ -0,0 +1,94 @@
require_relative './quandora_question.rb'
require File.expand_path(File.dirname(__FILE__) + "/../base.rb")
class ImportScripts::Quandora < ImportScripts::Base
JSON_FILES_DIR = "output"
def initialize
super
@system_user = Discourse.system_user
@questions = []
Dir.foreach(JSON_FILES_DIR) do |filename|
next if filename == '.' or filename == '..'
question = File.read JSON_FILES_DIR + '/' + filename
@questions << question
end
end
def execute
puts "", "Importing from Quandora..."
import_questions @questions
EmailToken.delete_all
puts "", "Done"
end
def import_questions questions
topics = 0
total = questions.size
questions.each do |question|
q = QuandoraQuestion.new question
import_users q.users
created_topic = import_topic q.topic
if created_topic
import_posts q.replies, created_topic.topic_id
end
topics += 1
print_status topics, total
end
puts "", "Imported #{topics} topics."
end
def import_users users
users.each do |user|
create_user user, user[:id]
end
end
def import_topic topic
post = nil
if post_id = post_id_from_imported_post_id(topic[:id])
post = Post.find(post_id) # already imported this topic
else
topic[:user_id] = user_id_from_imported_user_id(topic[:author_id]) || -1
topic[:category] = 'quandora-import'
post = create_post(topic, topic[:id])
unless created_topic.is_a?(Post)
puts "Error creating topic #{topic[:id]}. Skipping."
puts created_topic.inspect
end
end
post
end
def import_posts posts, topic_id
posts.each do |post|
import_post post, topic_id
end
end
def import_post post, topic_id
if post_id_from_imported_post_id(post[:id])
return # already imported
end
post[:topic_id] = topic_id
post[:user_id] = user_id_from_imported_user_id(post[:author_id])
new_post = create_post post, post[:id]
unless new_post.is_a?(Post)
puts "Error creating post #{post[:id]}. Skipping."
puts new_post.inspect
end
end
def file_full_path(relpath)
File.join JSON_FILES_DIR, relpath.split("?").first
end
end
if __FILE__==$0
ImportScripts::Quandora.new.perform
end

View File

@ -0,0 +1,54 @@
require 'base64'
require 'json'
require 'rest-client'
class QuandoraApi
attr_accessor :domain, :username, :password
def initialize domain, username, password
@domain = domain
@username = username
@password = password
end
def base_url domain
"https://#{domain}.quandora.com/m/json"
end
def auth_header username, password
encoded = Base64.encode64 "#{username}:#{password}"
{:Authorization => "Basic #{encoded.strip!}"}
end
def list_bases_url
"#{base_url @domain}/kb"
end
def list_questions_url kb_id, limit
url = "#{base_url @domain}/kb/#{kb_id}/list"
url = "#{url}?l=#{limit}" if limit
url
end
def request url
JSON.parse(RestClient.get url, auth_header(@username, @password))
end
def list_bases
response = request list_bases_url
response['data']
end
def list_questions kb_id, limit = nil
url = list_questions_url(kb_id, limit)
response = request url
response['data']['result']
end
def get_question question_id
url = "#{base_url @domain}/q/#{question_id}"
response = request url
response['data']
end
end

View File

@ -0,0 +1,109 @@
require 'json'
require 'cgi'
require 'time'
class QuandoraQuestion
def initialize question_json
@question = JSON.parse question_json
end
def topic
topic = {}
topic[:id] = @question['uid']
topic[:author_id] = @question['author']['uid']
topic[:title] = unescape @question['title']
topic[:raw] = unescape @question['content']
topic[:created_at] = Time.parse @question['created']
topic
end
def users
users = {}
user = user_from_author @question['author']
users[user[:id]] = user
replies.each do |reply|
user = user_from_author reply[:author]
users[user[:id]] = user
end
users.values.to_a
end
def user_from_author author
email = author['email']
email = "#{author['uid']}@noemail.com" unless email
user = {}
user[:id] = author['uid']
user[:name] = "#{author['firstName']} #{author['lastName']}"
user[:email] = email
user[:staged] = true
user
end
def replies
posts = []
answers = @question['answersList']
comments = @question['comments']
comments.each_with_index do |comment, i|
posts << post_from_comment(comment, i, @question)
end
answers.each do |answer|
posts << post_from_answer(answer)
comments = answer['comments']
comments.each_with_index do |comment, i|
posts << post_from_comment(comment, i, answer)
end
end
order_replies posts
end
def order_replies posts
posts = posts.sort_by { |p| p[:created_at] }
posts.each_with_index do |p, i|
p[:post_number] = i + 2
end
posts.each do |p|
parent = posts.select { |pp| pp[:id] == p[:parent_id] }
p[:reply_to_post_number] = parent[0][:post_number] if parent.size > 0
end
posts
end
def post_from_answer answer
post = {}
post[:id] = answer['uid']
post[:parent_id] = @question['uid']
post[:author] = answer['author']
post[:author_id] = answer['author']['uid']
post[:raw] = unescape answer['content']
post[:created_at] = Time.parse answer['created']
post
end
def post_from_comment comment, index, parent
if comment['created']
created_at = Time.parse comment['created']
else
created_at = Time.parse parent['created']
end
parent_id = parent['uid']
parent_id = "#{parent['uid']}-#{index-1}" if index > 0
post = {}
id = "#{parent['uid']}-#{index}"
post[:id] = id
post[:parent_id] = parent_id
post[:author] = comment['author']
post[:author_id] = comment['author']['uid']
post[:raw] = unescape comment['text']
post[:created_at] = created_at
post
end
private
def unescape html
return nil unless html
CGI.unescapeHTML html
end
end

View File

@ -0,0 +1,6 @@
domain: 'my-quandora-domain'
username: 'my-quandora-username'
password: 'my-quandora-password'
kb_id: 'some-kb-id'
question_id: 'some-question-id'

View File

@ -0,0 +1,179 @@
BASES = '{
"type" : "kbase",
"data" : [ {
"objectId" : "90b1ccf3-35aa-4d6f-848e-e7c122d92c58",
"objectName" : "hotdogs",
"title" : "Hot Dogs",
"description" : "This knowledge base is for questions about Hot Dogs"
} ]
}'
QUESTIONS = '{
"type": "question-search-result",
"data": {
"totalSize": 445,
"offset": 0,
"limit": 1000,
"result": [ {
"uid": "dd2cf490-f564-4147-9943-57682c7fac73",
"title": "How can we improve the office?",
"summary": "Hi everyone, I&rsquo;d love to hear your suggestions about how we can make our office a more pleasant place to work",
"votes": 2,
"views": 107,
"answers": 5,
"commentsCount": 3,
"created": "2013-01-06T18:24:54.62Z",
"modified": "2015-05-03T00:52:45.63Z",
"authorId": "24599c93-0a83-4099-982f-d0d708ea3178",
"baseId": "90b1ccf3-35aa-4d6f-848e-e7c122d92c58",
"prettyUrl": "https://mydomain.quandora.com/general/q/de20ed0a5fe548a59c14d854f9af99f1/How-can-we-improve-the-office",
"accepted": null,
"author": {
"uid": "24599c93-0a83-4099-982f-d0d708ea3178",
"name": "24599c93-0a83-4099-982f-d0d708ea3178",
"email": "flast@mydomain.com",
"firstName": "First",
"lastName": "Last",
"title": "Member",
"score": 236,
"disabled": false,
"badgeCount": [3,0,0],
"avatarUrl": "//www.gravatar.com/avatar/fdf8bd4205dc7ad908ea0578a111cb89?d=mm&s=%s"
},
"tags": [ {
"uid": "88a08f00-3038-4e96-8c26-4a777b46871c",
"name": "office",
"category": null
}]
}]
}
}'
QUESTION = '{
"type" : "question",
"data" : {
"uid" : "de20ed0a-5fe5-48a5-9c14-d854f9af99f1",
"title" : "How can we improve the office?",
"votes" : 2,
"views" : 107,
"answers" : 5,
"commentsCount" : 3,
"created" : "2013-01-06T18:24:54.62Z",
"modified" : "2015-05-03T00:52:45.63Z",
"authorId" : "043c8d91-26f7-44c7-acfa-179f06a4e998",
"baseId" : "7583b6df-2090-46fd-97b5-cdde072ec34e",
"prettyUrl" : "https://mydomain.quandora.com/general/q/de20ed0a5fe548a59c14d854f9af99f1/How-can-we-improve-the-office",
"accepted" : null,
"content" : "<p>Hi everyone,</p> \n<p>I\'d love to hear your suggestions about how we can make our office a more pleasant place to work.</p> \n<p>What things are we missing from our kitchen or supply closet?</p> \n<p>If you don\'t regularly come to the office, and what do you think would make you more likely to make the commute?</p> \n<p>Thanks!</p>",
"contentType" : "markdown",
"answersList" : [ {
"uid" : "78e7dc82-fe0f-4687-8ed9-6ade23d95164",
"contentType" : "markdown",
"content" : "<p>The most value I get out of coming to the office is hearing about weird techy glitches, or announcements that the company has to make.</p> \n<p>The bulletin board, at least in Ohio, seems to have died off a bit. It would be easy to say that people are intimidated by the large group, but in some meetings I think there\'s another problem: it\'s often the case that people just say \'come and grab me after the meeting\'. I\'m sure that works well, but I like it when a summary of the solution arrives back via email or at the next meeting, so that the whole office can benefit from the knowledge transfer.</p>",
"comments" : [ ],
"votes" : 3,
"created" : "2013-01-07T04:59:56.26Z",
"accepted" : false,
"authorId" : "acfd09c6-8bf8-4342-98de-3d7fc4c60ec0",
"author" : {
"uid" : "acfd09c6-8bf8-4342-98de-3d7fc4c60ec0",
"name" : "acfd09c6-8bf8-4342-98de-3d7fc4c60ec0",
"email" : "hharry@mydomain.com",
"firstName" : "Harry",
"lastName" : "Helpful",
"title" : "Member",
"score" : 1615,
"disabled" : false,
"badgeCount" : null,
"avatarUrl" : "//www.gravatar.com/avatar/e3cbc264af6d2392b7f323cebbbcfea6?d=mm&s=%s"
}
}, {
"uid" : "b6864e72-1a03-4f49-aa7f-d2781b14f69c",
"contentType" : "markdown",
"content" : "<p>For Ohio: i don\'t know if you\'ve already tried this, but I recommend doing the meetings in the beginning of the day. That way people are more likely to come into the office early, rather than after lunch :)</p>",
"comments" : [ {
"author" : {
"uid" : "204973f4-2dfe-494c-b1b2-3cd1cbac34f0",
"name" : "204973f4-2dfe-494c-b1b2-3cd1cbac34f0",
"email" : "eexcited@mydomain.com",
"firstName" : "Eddy",
"lastName" : "Excited",
"title" : "Member",
"score" : 516,
"disabled" : false,
"badgeCount" : null,
"avatarUrl" : "//www.gravatar.com/avatar/baa5f96720477108e685d38f5a7fa21c?d=mm&s=%s"
},
"created" : "2016-01-22T15:38:55.91Z",
"text" : "Great idea! I think more people will overlap here if we start our days at the same time.",
"hash" : "7f45b063f8f52eead80a784ca37e901a"
} ],
"votes" : 2,
"created" : "2013-01-08T16:49:32.80Z",
"accepted" : false,
"authorId" : "da0a6658-fa06-420a-9027-7a8051e4ec29",
"author" : {
"uid" : "da0a6658-fa06-420a-9027-7a8051e4ec29",
"name" : "da0a6658-fa06-420a-9027-7a8051e4ec29",
"email" : "ssmartypants@mydomain.com",
"firstName" : "Sam",
"lastName" : "Smarty-Pants",
"title" : "Member",
"score" : 3485,
"disabled" : false,
"badgeCount" : null,
"avatarUrl" : "//www.gravatar.com/avatar/e0be54fafea799f30abb6eacd2459cf6?d=mm&s=%s"
}
} ],
"comments" : [ {
"author" : {
"uid" : "acfd09c6-8bf8-4342-98de-3d7fc4c60ec0",
"name" : "acfd09c6-8bf8-4342-98de-3d7fc4c60ec0",
"email" : "hhelpful@mydomain.com",
"firstName" : "Harry",
"lastName" : "Helpful",
"title" : "Member",
"score" : 236,
"disabled" : false,
"badgeCount" : [ 3, 0, 0 ],
"avatarUrl" : "//www.gravatar.com/avatar/e3cbc264af6d2392b7f323cebbbcfea6?d=mm&s=%s"
},
"created" : "2016-01-20T15:38:55.91Z",
"text" : "Also, what hopes and expectations do you have of the new meeting space that we will be starting to use this week?",
"hash" : "226dbd023cc4e786bf1e7bc08989bde7"
}, {
"author" : {
"uid" : "7fcdc8ee-ab92-43a9-84a6-665aa4edbb49",
"name" : "7fcdc8ee-ab92-43a9-84a6-665aa4edbb49",
"email" : "ggreatful@mydomain.com",
"firstName" : "Greta",
"lastName" : "Greatful",
"title" : "Member",
"score" : 516,
"disabled" : false,
"badgeCount" : null,
"avatarUrl" : "//www.gravatar.com/avatar/d6027aecba638fc8c402c6138e799007?d=mm&s=%s"
},
"created" : "2016-01-21T15:38:55.91Z",
"text" : "I love coming into the office. The view is great, the food is wonderful, and I get to hang out with some awesome people!",
"hash" : "7f45b063f8f52eead80a784ca37e901a"
} ],
"author" : {
"uid" : "8c07ba39-1e2b-406f-b3cf-3da78431d399",
"name" : "8c07ba39-1e2b-406f-b3cf-3da78431d399",
"email" : "iinquisitive@mydomain.com",
"firstName" : "Ida",
"lastName" : "Inquisitive",
"title" : "Member",
"score" : 236,
"disabled" : false,
"badgeCount" : [ 3, 0, 0 ],
"avatarUrl" : "//www.gravatar.com/avatar/187f4bff7780e4a12b727c3ad81cfbac?d=mm&s=%s"
},
"tags" : [ {
"uid" : "53f65082-f081-4fc9-9bd5-a739599ee2b3",
"name" : "office",
"category" : null
} ]
}
}'

View File

@ -0,0 +1,90 @@
require 'minitest/autorun'
require 'yaml'
require_relative '../quandora_api.rb'
require_relative './test_data.rb'
class TestQuandoraApi < Minitest::Test
DEBUG = false
def initialize args
config = YAML::load_file(File.join(__dir__, 'config.yml'))
@domain = config['domain']
@username = config['username']
@password = config['password']
@kb_id = config['kb_id']
@question_id = config['question_id']
super args
end
def setup
@quandora = QuandoraApi.new @domain, @username, @password
end
def test_intialize
assert_equal @domain, @quandora.domain
assert_equal @username, @quandora.username
assert_equal @password, @quandora.password
end
def test_base_url
assert_equal 'https://mydomain.quandora.com/m/json', @quandora.base_url('mydomain')
end
def test_auth_header
user = 'Aladdin'
password = 'open sesame'
auth_header = @quandora.auth_header user, password
assert_equal 'Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==', auth_header[:Authorization]
end
def test_list_bases_element_has_expected_structure
element = @quandora.list_bases[0]
expected = JSON.parse(BASES)['data'][0]
debug element
check_keys expected, element
end
def test_list_questions_has_expected_structure
response = @quandora.list_questions @kb_id, 1
debug response
check_keys JSON.parse(QUESTIONS)['data']['result'][0], response[0]
end
def test_get_question_has_expected_structure
question = @quandora.get_question @question_id
expected = JSON.parse(QUESTION)['data']
check_keys expected, question
expected_comment = expected['comments'][0]
actual_comment = question['comments'][0]
check_keys expected_comment, actual_comment
expected_answer = expected['answersList'][1]
actual_answer = question['answersList'][0]
check_keys expected_answer, actual_answer
expected_answer_comment = expected_answer['comments'][0]
actual_answer_comment = actual_answer['comments'][0]
check_keys expected_answer_comment, actual_answer_comment
end
private
def check_keys expected, actual
msg = "### caller[0]:\nKey not found in actual keys: #{actual.keys}\n"
expected.keys.each do |k|
assert (actual.keys.include? k), "#{k}"
end
end
def debug message, show=false
if show || DEBUG
puts '### ' + caller[0]
puts ''
puts message
puts ''
puts ''
end
end
end

View File

@ -0,0 +1,136 @@
require 'minitest/autorun'
require 'cgi'
require 'time'
require_relative '../quandora_question.rb'
require_relative './test_data.rb'
class TestQuandoraQuestion < Minitest::Test
def setup
@data = JSON.parse(QUESTION)['data']
@question = QuandoraQuestion.new @data.to_json
end
def test_topic
topic = @question.topic
assert_equal @data['uid'], topic[:id]
assert_equal @data['author']['uid'], topic[:author_id]
assert_equal unescape(@data['title']), topic[:title]
assert_equal unescape(@data['content']), topic[:raw]
assert_equal Time.parse(@data['created']), topic[:created_at]
end
def test_user_from_author
author = {}
author['uid'] = 'uid'
author['firstName'] = 'Joe'
author['lastName'] = 'Schmoe'
author['email'] = 'joe.schmoe@mydomain.com'
user = @question.user_from_author author
assert_equal 'uid', user[:id]
assert_equal 'Joe Schmoe', user[:name]
assert_equal 'joe.schmoe@mydomain.com', user[:email]
assert_equal true, user[:staged]
end
def test_user_from_author_with_no_email
author = {}
author['uid'] = 'foo'
user = @question.user_from_author author
assert_equal 'foo@noemail.com', user[:email]
end
def test_replies
replies = @question.replies
assert_equal 5, replies.size
assert_equal 2, replies[0][:post_number]
assert_equal 3, replies[1][:post_number]
assert_equal 4, replies[2][:post_number]
assert_equal 5, replies[3][:post_number]
assert_equal 6, replies[4][:post_number]
assert_equal nil, replies[0][:reply_to_post_number]
assert_equal nil, replies[1][:reply_to_post_number]
assert_equal nil, replies[2][:reply_to_post_number]
assert_equal 4, replies[3][:reply_to_post_number]
assert_equal 3, replies[4][:reply_to_post_number]
assert_equal '2013-01-07 04:59:56 UTC', replies[0][:created_at].to_s
assert_equal '2013-01-08 16:49:32 UTC', replies[1][:created_at].to_s
assert_equal '2016-01-20 15:38:55 UTC', replies[2][:created_at].to_s
assert_equal '2016-01-21 15:38:55 UTC', replies[3][:created_at].to_s
assert_equal '2016-01-22 15:38:55 UTC', replies[4][:created_at].to_s
end
def test_post_from_answer
answer = {}
answer['uid'] = 'uid'
answer['content'] = 'content'
answer['created'] = '2013-01-06T18:24:54.62Z'
answer['author'] = {'uid' => 'auid'}
post = @question.post_from_answer answer
assert_equal 'uid', post[:id]
assert_equal @question.topic[:id], post[:parent_id]
assert_equal answer['author'], post[:author]
assert_equal 'auid', post[:author_id]
assert_equal 'content', post[:raw]
assert_equal Time.parse('2013-01-06T18:24:54.62Z'), post[:created_at]
end
def test_post_from_comment
comment = {}
comment['text'] = 'text'
comment['created'] = '2013-01-06T18:24:54.62Z'
comment['author'] = {'uid' => 'auid'}
parent = {'uid' => 'parent-uid'}
post = @question.post_from_comment comment, 0, parent
assert_equal 'parent-uid-0', post[:id]
assert_equal 'parent-uid', post[:parent_id]
assert_equal comment['author'], post[:author]
assert_equal 'auid', post[:author_id]
assert_equal 'text', post[:raw]
assert_equal Time.parse('2013-01-06T18:24:54.62Z'), post[:created_at]
end
def test_post_from_comment_uses_parent_created_if_necessary
comment = {}
comment['author'] = {'uid' => 'auid'}
parent = {'created' => '2013-01-06T18:24:54.62Z'}
post = @question.post_from_comment comment, 0, parent
assert_equal Time.parse('2013-01-06T18:24:54.62Z'), post[:created_at]
end
def test_post_from_comment_uses_previous_comment_as_parent
comment = {}
comment['author'] = {'uid' => 'auid'}
parent = {'uid' => 'parent-uid', 'created' => '2013-01-06T18:24:54.62Z'}
post = @question.post_from_comment comment, 1, parent
assert_equal 'parent-uid-1', post[:id]
assert_equal 'parent-uid-0', post[:parent_id]
assert_equal Time.parse('2013-01-06T18:24:54.62Z'), post[:created_at]
end
def test_users
users = @question.users
assert_equal 5, users.size
assert_equal 'Ida Inquisitive', users[0][:name]
assert_equal 'Harry Helpful', users[1][:name]
assert_equal 'Sam Smarty-Pants', users[2][:name]
assert_equal 'Greta Greatful', users[3][:name]
assert_equal 'Eddy Excited', users[4][:name]
end
private
def unescape html
CGI.unescapeHTML html
end
end