From a00af4d85a39cd859d29bcd161a4eea2b00c55b1 Mon Sep 17 00:00:00 2001 From: Vinoth Kannan Date: Fri, 27 Oct 2017 01:59:36 +0530 Subject: [PATCH] FEATURE: Rake task to export and import category structure --- lib/import_export/base_exporter.rb | 162 ++++++++++++++++++ lib/import_export/category_exporter.rb | 75 ++------ lib/import_export/category_importer.rb | 89 ---------- .../category_structure_exporter.rb | 30 ++++ lib/import_export/import_export.rb | 26 +-- lib/import_export/importer.rb | 160 +++++++++++++++++ lib/import_export/topic_exporter.rb | 83 ++------- lib/import_export/topic_importer.rb | 90 ---------- lib/tasks/export.rake | 7 + lib/tasks/import.rake | 8 + script/discourse | 4 +- spec/fabricators/group_user_fabricator.rb | 4 + spec/fixtures/json/import-export.json | 31 ++++ spec/import_export/category_exporter_spec.rb | 42 +++++ .../category_structure_exporter_spec.rb | 39 +++++ spec/import_export/importer_spec.rb | 68 ++++++++ spec/import_export/topic_exporter_spec.rb | 30 ++++ 17 files changed, 621 insertions(+), 327 deletions(-) create mode 100644 lib/import_export/base_exporter.rb delete mode 100644 lib/import_export/category_importer.rb create mode 100644 lib/import_export/category_structure_exporter.rb create mode 100644 lib/import_export/importer.rb delete mode 100644 lib/import_export/topic_importer.rb create mode 100644 lib/tasks/export.rake create mode 100644 spec/fabricators/group_user_fabricator.rb create mode 100644 spec/fixtures/json/import-export.json create mode 100644 spec/import_export/category_exporter_spec.rb create mode 100644 spec/import_export/category_structure_exporter_spec.rb create mode 100644 spec/import_export/importer_spec.rb create mode 100644 spec/import_export/topic_exporter_spec.rb diff --git a/lib/import_export/base_exporter.rb b/lib/import_export/base_exporter.rb new file mode 100644 index 00000000000..ff6cd2ed076 --- /dev/null +++ b/lib/import_export/base_exporter.rb @@ -0,0 +1,162 @@ +module ImportExport + class BaseExporter + + attr_reader :export_data, :categories + + CATEGORY_ATTRS = [:id, :name, :color, :created_at, :user_id, :slug, :description, :text_color, + :auto_close_hours, :parent_category_id, :auto_close_based_on_last_post, + :topic_template, :suppress_from_homepage, :all_topics_wiki, :permissions_params] + + GROUP_ATTRS = [ :id, :name, :created_at, :mentionable_level, :messageable_level, :visibility_level, + :automatic_membership_email_domains, :automatic_membership_retroactive, + :primary_group, :title, :grant_trust_level, :incoming_email] + + USER_ATTRS = [:id, :email, :username, :name, :created_at, :trust_level, :active, :last_emailed_at] + + TOPIC_ATTRS = [:id, :title, :created_at, :views, :category_id, :closed, :archived, :archetype] + + POST_ATTRS = [:id, :user_id, :post_number, :raw, :created_at, :reply_to_post_number, :hidden, + :hidden_reason_id, :wiki] + + def categories + @categories ||= Category.all.to_a + end + + def export_categories + data = [] + + categories.each do |cat| + data << CATEGORY_ATTRS.inject({}) { |h, a| h[a] = cat.send(a); h } + end + + data + end + + def export_categories! + @export_data[:categories] = export_categories + + self + end + + def export_category_groups + groups = [] + group_names = [] + auto_group_names = Group::AUTO_GROUPS.keys.map(&:to_s) + + @export_data[:categories].each do |c| + c[:permissions_params].each do |group_name, _| + group_names << group_name unless auto_group_names.include?(group_name.to_s) + end + end + + group_names.uniq! + return [] if group_names.empty? + + Group.where(name: group_names).find_each do |group| + attrs = GROUP_ATTRS.inject({}) { |h, a| h[a] = group.send(a); h } + attrs[:user_ids] = group.users.pluck(:id) + groups << attrs + end + + groups + end + + def export_category_groups! + @export_data[:groups] = export_category_groups + + self + end + + def export_group_users + user_ids = [] + + @export_data[:groups].each do |g| + user_ids += g[:user_ids] + end + + user_ids.uniq! + return [] if user_ids.empty? + + users = User.where(id: user_ids) + export_users(users.to_a) + end + + def export_group_users! + @export_data[:users] = export_group_users + + self + end + + def export_topics + data = [] + + @topics.each do |topic| + puts topic.title + + topic_data = TOPIC_ATTRS.inject({}) { |h, a| h[a] = topic.send(a); h; } + topic_data[:posts] = [] + + topic.ordered_posts.find_each do |post| + h = POST_ATTRS.inject({}) { |h, a| h[a] = post.send(a); h; } + h[:raw] = h[:raw].gsub('src="/uploads', "src=\"#{Discourse.base_url_no_prefix}/uploads") + topic_data[:posts] << h + end + + data << topic_data + end + + data + end + + def export_topics! + @export_data[:topics] = export_topics + + self + end + + def export_topic_users + return if @export_data[:topics].blank? + topic_ids = @export_data[:topics].pluck(:id) + + users = User.joins(:topics).where('topics.id IN (?)', topic_ids).to_a + users.uniq! + + export_users(users.to_a) + end + + def export_topic_users! + @export_data[:users] = export_topic_users + + self + end + + def export_users(users) + data = [] + users.reject! { |u| u.id == Discourse::SYSTEM_USER_ID } + + users.each do |u| + x = USER_ATTRS.inject({}) { |h, a| h[a] = u.send(a); h; } + x.merge(bio_raw: u.user_profile.bio_raw, + website: u.user_profile.website, + location: u.user_profile.location) + data << x + end + + data + end + + def default_filename_prefix + raise "Overwrite me!" + end + + def save_to_file(filename = nil) + output_basename = filename || File.join("#{default_filename_prefix}-#{Time.now.strftime("%Y-%m-%d-%H%M%S")}.json") + File.open(output_basename, "w:UTF-8") do |f| + f.write(@export_data.to_json) + end + puts "Export saved to #{output_basename}" + output_basename + end + + end +end diff --git a/lib/import_export/category_exporter.rb b/lib/import_export/category_exporter.rb index d40bd327c49..4ea7209c19f 100644 --- a/lib/import_export/category_exporter.rb +++ b/lib/import_export/category_exporter.rb @@ -1,71 +1,32 @@ -module ImportExport - class CategoryExporter +require "import_export/base_exporter" +require "import_export/topic_exporter" - attr_reader :export_data +module ImportExport + class CategoryExporter < BaseExporter def initialize(category_id) @category = Category.find(category_id) - @subcategories = Category.where(parent_category_id: category_id) + @categories = Category.where(parent_category_id: category_id).to_a + @categories << @category @export_data = { - users: [], + categories: [], groups: [], - category: nil, - subcategories: [], - topics: [] + topics: [], + users: [] } end def perform puts "Exporting category #{@category.name}...", "" - export_categories + export_categories! + export_category_groups! export_topics_and_users self end - CATEGORY_ATTRS = [:id, :name, :color, :created_at, :user_id, :slug, :description, :text_color, - :auto_close_hours, :auto_close_based_on_last_post, - :topic_template, :suppress_from_homepage, :all_topics_wiki, :permissions_params] - - def export_categories - @export_data[:category] = CATEGORY_ATTRS.inject({}) { |h, a| h[a] = @category.send(a); h } - @subcategories.find_each do |subcat| - @export_data[:subcategories] << CATEGORY_ATTRS.inject({}) { |h, a| h[a] = subcat.send(a); h } - end - - # export groups that are mentioned in category permissions - group_names = [] - auto_group_names = Group::AUTO_GROUPS.keys.map(&:to_s) - - ([@export_data[:category]] + @export_data[:subcategories]).each do |c| - c[:permissions_params].each do |group_name, _| - group_names << group_name unless auto_group_names.include?(group_name.to_s) - end - end - - group_names.uniq! - export_groups(group_names) unless group_names.empty? - - self - end - - GROUP_ATTRS = [ :id, :name, :created_at, :mentionable_level, :messageable_level, :visible, - :automatic_membership_email_domains, :automatic_membership_retroactive, - :primary_group, :title, :grant_trust_level, :incoming_email] - - def export_groups(group_names) - group_names.each do |name| - group = Group.find_by_name(name) - group_attrs = GROUP_ATTRS.inject({}) { |h, a| h[a] = group.send(a); h } - group_attrs[:user_ids] = group.users.pluck(:id) - @export_data[:groups] << group_attrs - end - - self - end - def export_topics_and_users - all_category_ids = [@category.id] + @subcategories.pluck(:id) - description_topic_ids = Category.where(id: all_category_ids).pluck(:topic_id) + all_category_ids = @categories.pluck(:id) + description_topic_ids = @categories.pluck(:topic_id) topic_exporter = ImportExport::TopicExporter.new(Topic.where(category_id: all_category_ids).pluck(:id) - description_topic_ids) topic_exporter.perform @export_data[:users] = topic_exporter.export_data[:users] @@ -73,14 +34,8 @@ module ImportExport self end - def save_to_file(filename = nil) - require 'json' - output_basename = filename || File.join("category-export-#{Time.now.strftime("%Y-%m-%d-%H%M%S")}.json") - File.open(output_basename, "w:UTF-8") do |f| - f.write(@export_data.to_json) - end - puts "Export saved to #{output_basename}" - output_basename + def default_filename_prefix + "category-export" end end diff --git a/lib/import_export/category_importer.rb b/lib/import_export/category_importer.rb deleted file mode 100644 index b432bd078bf..00000000000 --- a/lib/import_export/category_importer.rb +++ /dev/null @@ -1,89 +0,0 @@ -require File.join(Rails.root, 'script', 'import_scripts', 'base.rb') - -module ImportExport - class CategoryImporter < ImportScripts::Base - def initialize(export_data) - @export_data = export_data - @topic_importer = TopicImporter.new(@export_data) - end - - def perform - RateLimiter.disable - - import_users - import_groups - import_categories - import_topics - self - ensure - RateLimiter.enable - end - - def import_groups - return if @export_data[:groups].empty? - - @export_data[:groups].each do |group_data| - g = group_data.dup - user_ids = g.delete(:user_ids) - external_id = g.delete(:id) - new_group = Group.find_by_name(g[:name]) || Group.create!(g) - user_ids.each do |external_user_id| - new_group.add(User.find(@topic_importer.new_user_id(external_user_id))) rescue ActiveRecord::RecordNotUnique - end - end - end - - def import_users - @topic_importer.import_users - end - - def import_categories - id = @export_data[:category].delete(:id) - import_id = "#{id}#{import_source}" - - parent = CategoryCustomField.where(name: 'import_id', value: import_id).first.try(:category) - - unless parent - permissions = @export_data[:category].delete(:permissions_params) - parent = Category.new(@export_data[:category]) - parent.user_id = @topic_importer.new_user_id(@export_data[:category][:user_id]) # imported user's new id - parent.custom_fields["import_id"] = import_id - parent.permissions = permissions.present? ? permissions : { "everyone" => CategoryGroup.permission_types[:full] } - parent.save! - set_category_description(parent, @export_data[:category][:description]) - end - - @export_data[:subcategories].each do |cat_attrs| - id = cat_attrs.delete(:id) - import_id = "#{id}#{import_source}" - existing = CategoryCustomField.where(name: 'import_id', value: import_id).first.try(:category) - - unless existing - permissions = cat_attrs.delete(:permissions_params) - subcategory = Category.new(cat_attrs) - subcategory.parent_category_id = parent.id - subcategory.user_id = @topic_importer.new_user_id(cat_attrs[:user_id]) - subcategory.custom_fields["import_id"] = import_id - subcategory.permissions = permissions.present? ? permissions : { "everyone" => CategoryGroup.permission_types[:full] } - subcategory.save! - set_category_description(subcategory, cat_attrs[:description]) - end - end - end - - def set_category_description(c, description) - post = c.topic.ordered_posts.first - post.raw = description - post.save! - post.rebake! - end - - def import_topics - @topic_importer.import_topics - end - - def import_source - @_import_source ||= "#{ENV['IMPORT_SOURCE'] || ''}" - end - end -end diff --git a/lib/import_export/category_structure_exporter.rb b/lib/import_export/category_structure_exporter.rb new file mode 100644 index 00000000000..7c6f3c61db1 --- /dev/null +++ b/lib/import_export/category_structure_exporter.rb @@ -0,0 +1,30 @@ +require "import_export/base_exporter" + +module ImportExport + class CategoryStructureExporter < ImportExport::BaseExporter + + def initialize(include_group_users = false) + @include_group_users = include_group_users + + @export_data = { + groups: [], + categories: [] + } + @export_data[:users] = [] if @include_group_users + end + + def perform + puts "Exporting all the categories...", "" + export_categories! + export_category_groups! + export_group_users! if @include_group_users + + self + end + + def default_filename_prefix + "category-structure-export" + end + + end +end diff --git a/lib/import_export/import_export.rb b/lib/import_export/import_export.rb index fad560768f2..53ed9f591ca 100644 --- a/lib/import_export/import_export.rb +++ b/lib/import_export/import_export.rb @@ -1,26 +1,26 @@ +require "import_export/importer" +require "import_export/category_structure_exporter" require "import_export/category_exporter" -require "import_export/category_importer" require "import_export/topic_exporter" -require "import_export/topic_importer" require "json" module ImportExport + def self.import(filename) + data = ActiveSupport::HashWithIndifferentAccess.new(File.open(filename, "r:UTF-8") { |f| JSON.parse(f.read) }) + ImportExport::Importer.new(data).perform + end + + def self.export_categories(include_users, filename = nil) + ImportExport::CategoryStructureExporter.new(include_users).perform.save_to_file(filename) + end + def self.export_category(category_id, filename = nil) ImportExport::CategoryExporter.new(category_id).perform.save_to_file(filename) end - def self.import_category(filename) - export_data = ActiveSupport::HashWithIndifferentAccess.new(File.open(filename, "r:UTF-8") { |f| JSON.parse(f.read) }) - ImportExport::CategoryImporter.new(export_data).perform + def self.export_topics(topic_ids, filename = nil) + ImportExport::TopicExporter.new(topic_ids).perform.save_to_file(filename) end - def self.export_topics(topic_ids) - ImportExport::TopicExporter.new(topic_ids).perform.save_to_file - end - - def self.import_topics(filename) - export_data = ActiveSupport::HashWithIndifferentAccess.new(File.open(filename, "r:UTF-8") { |f| JSON.parse(f.read) }) - ImportExport::TopicImporter.new(export_data).perform - end end diff --git a/lib/import_export/importer.rb b/lib/import_export/importer.rb new file mode 100644 index 00000000000..9bc43ae5069 --- /dev/null +++ b/lib/import_export/importer.rb @@ -0,0 +1,160 @@ +require File.join(Rails.root, 'script', 'import_scripts', 'base.rb') + +module ImportExport + class Importer < ImportScripts::Base + + def initialize(data) + @users = data[:users] + @groups = data[:groups] + @categories = data[:categories] + @topics = data[:topics] + + # To support legacy `category_export` script + if data[:category].present? + @categories = [] if @categories.blank? + @categories << data[:category] + end + end + + def perform + RateLimiter.disable + + import_users + import_groups + import_categories + import_topics + + self + ensure + RateLimiter.enable + end + + def import_users + return if @users.blank? + + @users.each do |u| + import_id = "#{u[:id]}#{import_source}" + existing = User.with_email(u[:email]).first + + if existing + if existing.custom_fields["import_id"] != import_id + existing.custom_fields["import_id"] = import_id + existing.save! + end + else + u = create_user(u, import_id) # see ImportScripts::Base + end + end + + self + end + + def import_groups + return if @groups.blank? + + @groups.each do |group_data| + g = group_data.dup + user_ids = g.delete(:user_ids) + external_id = g.delete(:id) + new_group = Group.find_by_name(g[:name]) || Group.create!(g) + user_ids.each do |external_user_id| + new_group.add(User.find(new_user_id(external_user_id))) rescue ActiveRecord::RecordNotUnique + end + end + + self + end + + def import_categories + return if @categories.blank? + + import_ids = @categories.collect { |c| "#{c[:id]}#{import_source}" } + existing_categories = CategoryCustomField.where("name = 'import_id' AND value IN (?)", import_ids).select(:category_id, :value).to_a + existing_category_ids = existing_categories.pluck(:value) + + @categories.reject! { |c| existing_category_ids.include? c[:id].to_s } + @categories.sort_by! { |c| c[:parent_category_id].presence || 0 } + + @categories.each do |cat_attrs| + id = cat_attrs.delete(:id) + permissions = cat_attrs.delete(:permissions_params) + + category = Category.new(cat_attrs) + category.parent_category_id = new_category_id(cat_attrs[:parent_category_id]) if cat_attrs[:parent_category_id].present? + category.user_id = new_user_id(cat_attrs[:user_id]) + import_id = "#{id}#{import_source}" + category.custom_fields["import_id"] = import_id + category.permissions = permissions.present? ? permissions : { "everyone" => CategoryGroup.permission_types[:full] } + category.save! + existing_categories << { category_id: category.id, value: import_id } + + if cat_attrs[:description].present? + post = category.topic.ordered_posts.first + post.raw = cat_attrs[:description] + post.save! + post.rebake! + end + end + + self + end + + def import_topics + return if @topics.blank? + + @topics.each do |t| + puts "" + print t[:title] + + first_post_attrs = t[:posts].first.merge(t.slice(*(TopicExporter::TOPIC_ATTRS - [:id, :category_id]))) + + first_post_attrs[:user_id] = new_user_id(first_post_attrs[:user_id]) + first_post_attrs[:category] = new_category_id(t[:category_id]) + + import_id = "#{first_post_attrs[:id]}#{import_source}" + first_post = PostCustomField.where(name: "import_id", value: import_id).first&.post + + unless first_post + first_post = create_post(first_post_attrs, import_id) + end + + topic_id = first_post.topic_id + + t[:posts].each_with_index do |post_data, i| + next if i == 0 + print "." + post_import_id = "#{post_data[:id]}#{import_source}" + existing = PostCustomField.where(name: "import_id", value: post_import_id).first&.post + unless existing + # see ImportScripts::Base + create_post( + post_data.merge( + topic_id: topic_id, + user_id: new_user_id(post_data[:user_id]) + ), + post_import_id + ) + end + end + end + + puts "" + + self + end + + def new_user_id(external_user_id) + ucf = UserCustomField.where(name: "import_id", value: "#{external_user_id}#{import_source}").first + ucf ? ucf.user_id : Discourse::SYSTEM_USER_ID + end + + def new_category_id(external_category_id) + CategoryCustomField.where(name: "import_id", value: "#{external_category_id}#{import_source}").first.category_id rescue nil + end + + def import_source + @_import_source ||= "#{ENV['IMPORT_SOURCE'] || ''}" + end + + end +end diff --git a/lib/import_export/topic_exporter.rb b/lib/import_export/topic_exporter.rb index 4c494003edd..4603e3d1b66 100644 --- a/lib/import_export/topic_exporter.rb +++ b/lib/import_export/topic_exporter.rb @@ -1,89 +1,26 @@ -module ImportExport - class TopicExporter +require "import_export/base_exporter" - attr_reader :exported_user_ids, :export_data +module ImportExport + class TopicExporter < ImportExport::BaseExporter def initialize(topic_ids) - @topic_ids = topic_ids - @exported_user_ids = [] + @topics = Topic.where(id: topic_ids).to_a @export_data = { - users: [], - topics: [] + topics: [], + users: [] } end def perform - export_users - export_topics + export_topics! + export_topic_users! # TODO: user actions self end - USER_ATTRS = [:id, :email, :username, :name, :created_at, :trust_level, :active, :last_emailed_at] - - def export_users - # TODO: avatar - - @exported_user_ids = [] - @topic_ids.each do |topic_id| - t = Topic.find(topic_id) - t.posts.includes(user: [:user_profile]).find_each do |post| - u = post.user - unless @exported_user_ids.include?(u.id) - x = USER_ATTRS.inject({}) { |h, a| h[a] = u.send(a); h; } - @export_data[:users] << x.merge(bio_raw: u.user_profile.bio_raw, - website: u.user_profile.website, - location: u.user_profile.location) - @exported_user_ids << u.id - end - end - end - - self - end - - def export_topics - @topic_ids.each do |topic_id| - t = Topic.find(topic_id) - puts t.title - export_topic(t) - end - puts "" - end - - TOPIC_ATTRS = [:id, :title, :created_at, :views, :category_id, :closed, :archived, :archetype] - POST_ATTRS = [:id, :user_id, :post_number, :raw, :created_at, :reply_to_post_number, - :hidden, :hidden_reason_id, :wiki] - - def export_topic(topic) - topic_data = {} - - TOPIC_ATTRS.each do |a| - topic_data[a] = topic.send(a) - end - - topic_data[:posts] = [] - - topic.ordered_posts.find_each do |post| - h = POST_ATTRS.inject({}) { |h, a| h[a] = post.send(a); h; } - h[:raw] = h[:raw].gsub('src="/uploads', "src=\"#{Discourse.base_url_no_prefix}/uploads") - topic_data[:posts] << h - end - - @export_data[:topics] << topic_data - - self - end - - def save_to_file(filename = nil) - require 'json' - output_basename = filename || File.join("topic-export-#{Time.now.strftime("%Y-%m-%d-%H%M%S")}.json") - File.open(output_basename, "w:UTF-8") do |f| - f.write(@export_data.to_json) - end - puts "Export saved to #{output_basename}" - output_basename + def default_filename_prefix + "topic-export" end end diff --git a/lib/import_export/topic_importer.rb b/lib/import_export/topic_importer.rb deleted file mode 100644 index 721536d7227..00000000000 --- a/lib/import_export/topic_importer.rb +++ /dev/null @@ -1,90 +0,0 @@ -require File.join(Rails.root, 'script', 'import_scripts', 'base.rb') - -module ImportExport - class TopicImporter < ImportScripts::Base - def initialize(export_data) - @export_data = export_data - end - - def perform - RateLimiter.disable - - import_users - import_topics - self - ensure - RateLimiter.enable - end - - def import_users - @export_data[:users].each do |u| - import_id = "#{u[:id]}#{import_source}" - existing = User.with_email(u[:email]).first - if existing - if existing.custom_fields["import_id"] != import_id - existing.custom_fields["import_id"] = import_id - existing.save! - end - else - u = create_user(u, import_id) # see ImportScripts::Base - end - end - self - end - - def import_topics - @export_data[:topics].each do |t| - puts "" - print t[:title] - - first_post_attrs = t[:posts].first.merge(t.slice(*(TopicExporter::TOPIC_ATTRS - [:id, :category_id]))) - - first_post_attrs[:user_id] = new_user_id(first_post_attrs[:user_id]) - first_post_attrs[:category] = new_category_id(t[:category_id]) - - import_id = "#{first_post_attrs[:id]}#{import_source}" - first_post = PostCustomField.where(name: "import_id", value: import_id).first&.post - - unless first_post - first_post = create_post(first_post_attrs, import_id) - end - - topic_id = first_post.topic_id - - t[:posts].each_with_index do |post_data, i| - next if i == 0 - print "." - post_import_id = "#{post_data[:id]}#{import_source}" - existing = PostCustomField.where(name: "import_id", value: post_import_id).first&.post - unless existing - # see ImportScripts::Base - create_post( - post_data.merge( - topic_id: topic_id, - user_id: new_user_id(post_data[:user_id]) - ), - post_import_id - ) - end - end - end - - puts "" - - self - end - - def new_user_id(external_user_id) - ucf = UserCustomField.where(name: "import_id", value: "#{external_user_id}#{import_source}").first - ucf ? ucf.user_id : Discourse::SYSTEM_USER_ID - end - - def new_category_id(external_category_id) - CategoryCustomField.where(name: "import_id", value: "#{external_category_id}#{import_source}").first.category_id rescue nil - end - - def import_source - @_import_source ||= "#{ENV['IMPORT_SOURCE'] || ''}" - end - end -end diff --git a/lib/tasks/export.rake b/lib/tasks/export.rake new file mode 100644 index 00000000000..cfa59540277 --- /dev/null +++ b/lib/tasks/export.rake @@ -0,0 +1,7 @@ +desc 'Export all the categories' +task 'export:categories', [:include_group_users, :file_name] => [:environment] do |_, args| + require "import_export/import_export" + + ImportExport.export_categories(args[:include_group_users], args[:file_name]) + puts "", "Done", "" +end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 129aeca2348..f26e7b30257 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -500,3 +500,11 @@ task "import:create_vbulletin_permalinks" => :environment do log "Done!" end + +desc 'Import existing exported file' +task 'import:file', [:file_name] => [:environment] do |_, args| + require "import_export/import_export" + + ImportExport.import(args[:file_name]) + puts "", "Done", "" +end diff --git a/script/discourse b/script/discourse index 85b7fa491af..1f8fe1275f7 100755 --- a/script/discourse +++ b/script/discourse @@ -198,7 +198,7 @@ class DiscourseCLI < Thor puts "Starting import from #{filename}..." load_rails load_import_export - ImportExport.import_category(filename) + ImportExport.import(filename) puts "", "Done", "" end @@ -218,7 +218,7 @@ class DiscourseCLI < Thor puts "Starting import from #{filename}..." load_rails load_import_export - ImportExport.import_topics(filename) + ImportExport.import(filename) puts "", "Done", "" end diff --git a/spec/fabricators/group_user_fabricator.rb b/spec/fabricators/group_user_fabricator.rb new file mode 100644 index 00000000000..9ed23e47eb3 --- /dev/null +++ b/spec/fabricators/group_user_fabricator.rb @@ -0,0 +1,4 @@ +Fabricator(:group_user) do + user + group +end diff --git a/spec/fixtures/json/import-export.json b/spec/fixtures/json/import-export.json new file mode 100644 index 00000000000..9af32b8e1e9 --- /dev/null +++ b/spec/fixtures/json/import-export.json @@ -0,0 +1,31 @@ +{ + "groups":[ + {"id":41,"name":"custom_group","created_at":"2017-10-26T15:33:46.328Z","mentionable_level":0,"messageable_level":0,"visibility_level":0,"automatic_membership_email_domains":"","automatic_membership_retroactive":false,"primary_group":false,"title":null,"grant_trust_level":null,"incoming_email":null,"user_ids":[1]}, + {"id":42,"name":"custom_group_import","created_at":"2017-10-26T15:33:46.328Z","mentionable_level":0,"messageable_level":0,"visibility_level":0,"automatic_membership_email_domains":"","automatic_membership_retroactive":false,"primary_group":false,"title":null,"grant_trust_level":null,"incoming_email":null,"user_ids":[2]} + ], + "categories":[ + {"id":8,"name":"Custom Category","color":"AB9364","created_at":"2017-10-26T15:32:44.083Z","user_id":1,"slug":"custom-category","description":null,"text_color":"FFFFFF","auto_close_hours":null,"parent_category_id":3,"auto_close_based_on_last_post":false,"topic_template":"","suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{"custom_group":1,"everyone":2}}, + {"id":10,"name":"Site Feedback Import","color":"808281","created_at":"2017-10-26T17:12:39.995Z","user_id":-1,"slug":"site-feedback-import","description":"Discussion about this site, its organization, how it works, and how we can improve it.","text_color":"FFFFFF","auto_close_hours":null,"parent_category_id":null,"auto_close_based_on_last_post":false,"topic_template":null,"suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{}}, + {"id":11,"name":"Uncategorized Import","color":"AB9364","created_at":"2017-10-26T17:12:32.359Z","user_id":-1,"slug":"uncategorized-import","description":"","text_color":"FFFFFF","auto_close_hours":null,"parent_category_id":null,"auto_close_based_on_last_post":false,"topic_template":null,"suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{}}, + {"id":12,"name":"Lounge Import","color":"EEEEEE","created_at":"2017-10-26T17:12:39.490Z","user_id":-1,"slug":"lounge-import","description":"A category exclusive to members with trust level 3 and higher.","text_color":"652D90","auto_close_hours":null,"parent_category_id":null,"auto_close_based_on_last_post":false,"topic_template":null,"suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{"trust_level_3":1}}, + {"id":13,"name":"Staff Import","color":"283890","created_at":"2017-10-26T17:12:42.806Z","user_id":2,"slug":"staff-import","description":"Private category for staff discussions. Topics are only visible to admins and moderators.","text_color":"FFFFFF","auto_close_hours":null,"parent_category_id":null,"auto_close_based_on_last_post":false,"topic_template":null,"suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{"custom_group_import":1,"staff":1}}, + {"id":15,"name":"Custom Category Import","color":"AB9364","created_at":"2017-10-26T15:32:44.083Z","user_id":2,"slug":"custom-category-import","description":null,"text_color":"FFFFFF","auto_close_hours":null,"parent_category_id":10,"auto_close_based_on_last_post":false,"topic_template":"","suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{"custom_group_import":1,"everyone":2}} + ], + "users":[ + {"id":1,"email":"vinothkannan@example.com","username":"example","name":"Example","created_at":"2017-10-07T15:01:24.597Z","trust_level":4,"active":true,"last_emailed_at":null}, + {"id":2,"email":"vinoth.kannan@discourse.org","username":"vinothkannans","name":"Vinoth Kannan","created_at":"2017-10-07T15:01:24.597Z","trust_level":4,"active":true,"last_emailed_at":null} + ], + "topics":[ + {"id":7,"title":"Assets for the site design","created_at":"2017-10-26T17:15:04.590Z","views":0,"category_id":8,"closed":false,"archived":false,"archetype":"regular", + "posts":[ + {"id":10,"user_id":-1,"post_number":1,"raw":"This topic, visible only to staff, is for storing images and files used in the site design.","created_at":"2017-10-26T17:15:04.720Z","reply_to_post_number":null,"hidden":false,"hidden_reason_id":null,"wiki":false} + ] + }, + {"id":6,"title":"Privacy Policy","created_at":"2017-10-26T17:15:04.009Z","views":0,"category_id":15,"closed":false,"archived":false,"archetype":"regular", + "posts":[ + {"id":8,"user_id":-1,"post_number":1,"raw":"[Third party links](#third-party)\n\nOccasionally, at our discretion, we may include or offer third party products or services on our site.","created_at":"2017-10-26T17:15:03.535Z","reply_to_post_number":null,"hidden":false,"hidden_reason_id":null,"wiki":false}, + {"id":7,"user_id":-1,"post_number":2,"raw":"Edit the first post in this topic to change the contents of the FAQ/Guidelines page.","created_at":"2017-10-26T17:15:03.822Z","reply_to_post_number":null,"hidden":false,"hidden_reason_id":null,"wiki":false} + ] + } + ] +} diff --git a/spec/import_export/category_exporter_spec.rb b/spec/import_export/category_exporter_spec.rb new file mode 100644 index 00000000000..5a6420712e5 --- /dev/null +++ b/spec/import_export/category_exporter_spec.rb @@ -0,0 +1,42 @@ +require "rails_helper" +require "import_export/category_exporter" + +describe ImportExport::CategoryExporter do + + let(:category) { Fabricate(:category) } + let(:group) { Fabricate(:group) } + let(:user) { Fabricate(:user) } + + context '.perform' do + it 'raises an error when the category is not found' do + expect { ImportExport::CategoryExporter.new(100).perform }.to raise_error(ActiveRecord::RecordNotFound) + end + + it 'export the category when it is found' do + data = ImportExport::CategoryExporter.new(category.id).perform.export_data + + expect(data[:categories].count).to eq(1) + expect(data[:groups].count).to eq(0) + end + + it 'export the category with permission groups' do + category_group = Fabricate(:category_group, category: category, group: group) + data = ImportExport::CategoryExporter.new(category.id).perform.export_data + + expect(data[:categories].count).to eq(1) + expect(data[:groups].count).to eq(1) + end + + it 'export the category with topics and users' do + topic1 = Fabricate(:topic, category: category, user_id: -1) + topic2 = Fabricate(:topic, category: category, user: user) + data = ImportExport::CategoryExporter.new(category.id).perform.export_data + + expect(data[:categories].count).to eq(1) + expect(data[:groups].count).to eq(0) + expect(data[:topics].count).to eq(2) + expect(data[:users].count).to eq(1) + end + end + +end diff --git a/spec/import_export/category_structure_exporter_spec.rb b/spec/import_export/category_structure_exporter_spec.rb new file mode 100644 index 00000000000..a840bf940cc --- /dev/null +++ b/spec/import_export/category_structure_exporter_spec.rb @@ -0,0 +1,39 @@ +require "rails_helper" +require "import_export/category_structure_exporter" + +describe ImportExport::CategoryStructureExporter do + + it 'export all the categories' do + category = Fabricate(:category) + data = ImportExport::CategoryStructureExporter.new.perform.export_data + + expect(data[:categories].count).to eq(2) + expect(data[:groups].count).to eq(0) + expect(data[:users].blank?).to eq(true) + end + + it 'export all the categories with permission groups' do + category = Fabricate(:category) + group = Fabricate(:group) + category_group = Fabricate(:category_group, category: category, group: group) + data = ImportExport::CategoryStructureExporter.new.perform.export_data + + expect(data[:categories].count).to eq(2) + expect(data[:groups].count).to eq(1) + expect(data[:users].blank?).to eq(true) + end + + it 'export all the categories with permission groups and users' do + category = Fabricate(:category) + group = Fabricate(:group) + user = Fabricate(:user) + category_group = Fabricate(:category_group, category: category, group: group) + group_user = Fabricate(:group_user, group: group, user: user) + data = ImportExport::CategoryStructureExporter.new(true).perform.export_data + + expect(data[:categories].count).to eq(2) + expect(data[:groups].count).to eq(1) + expect(data[:users].count).to eq(1) + end + +end diff --git a/spec/import_export/importer_spec.rb b/spec/import_export/importer_spec.rb new file mode 100644 index 00000000000..3d5146b3ed9 --- /dev/null +++ b/spec/import_export/importer_spec.rb @@ -0,0 +1,68 @@ +require "rails_helper" +require "import_export/category_exporter" +require "import_export/category_structure_exporter" +require "import_export/importer" + +describe ImportExport::Importer do + + let(:import_data) do + import_file = Rack::Test::UploadedFile.new(file_from_fixtures("import-export.json", "json")) + data = ActiveSupport::HashWithIndifferentAccess.new(JSON.parse(import_file.read)) + end + + def import(data) + ImportExport::Importer.new(data).perform + end + + context '.perform' do + + it 'topics and users' do + data = import_data.dup + data[:categories] = nil + data[:groups] = nil + + expect { + import(data) + }.to change { Category.count }.by(0) + .and change { Group.count }.by(0) + .and change { Topic.count }.by(2) + .and change { User.count }.by(2) + end + + it 'categories and groups' do + data = import_data.dup + data[:topics] = nil + data[:users] = nil + + expect { + import(data) + }.to change { Category.count }.by(6) + .and change { Group.count }.by(2) + .and change { Topic.count }.by(6) + .and change { User.count }.by(0) + end + + it 'categories, groups and users' do + data = import_data.dup + data[:topics] = nil + + expect { + import(data) + }.to change { Category.count }.by(6) + .and change { Group.count }.by(2) + .and change { Topic.count }.by(6) + .and change { User.count }.by(2) + end + + it 'all' do + expect { + import(import_data) + }.to change { Category.count }.by(6) + .and change { Group.count }.by(2) + .and change { Topic.count }.by(8) + .and change { User.count }.by(2) + end + + end + +end diff --git a/spec/import_export/topic_exporter_spec.rb b/spec/import_export/topic_exporter_spec.rb new file mode 100644 index 00000000000..47a76f02bd6 --- /dev/null +++ b/spec/import_export/topic_exporter_spec.rb @@ -0,0 +1,30 @@ +require "rails_helper" +require "import_export/topic_exporter" + +describe ImportExport::TopicExporter do + + let(:user) { Fabricate(:user) } + let(:topic) { Fabricate(:topic, user: user) } + + context '.perform' do + it 'export a single topic' do + data = ImportExport::TopicExporter.new([topic.id]).perform.export_data + + expect(data[:categories].blank?).to eq(true) + expect(data[:groups].blank?).to eq(true) + expect(data[:topics].count).to eq(1) + expect(data[:users].count).to eq(1) + end + + it 'export multiple topics' do + topic2 = Fabricate(:topic, user: user) + data = ImportExport::TopicExporter.new([topic.id, topic2.id]).perform.export_data + + expect(data[:categories].blank?).to eq(true) + expect(data[:groups].blank?).to eq(true) + expect(data[:topics].count).to eq(2) + expect(data[:users].count).to eq(1) + end + end + +end