Merge pull request #5270 from vinothkannans/rake_categories

FEATURE: Rake task to export and import category structure

lib/import_export/base_exporter.rb

@@ -0,0 +1,162 @@
+module ImportExport
+  class BaseExporter
+
+    attr_reader :export_data, :categories
+
+    CATEGORY_ATTRS = [:id, :name, :color, :created_at, :user_id, :slug, :description, :text_color,
+                      :auto_close_hours, :parent_category_id, :auto_close_based_on_last_post,
+                      :topic_template, :suppress_from_homepage, :all_topics_wiki, :permissions_params]
+
+    GROUP_ATTRS = [ :id, :name, :created_at, :mentionable_level, :messageable_level, :visibility_level,
+                    :automatic_membership_email_domains, :automatic_membership_retroactive,
+                    :primary_group, :title, :grant_trust_level, :incoming_email]
+
+    USER_ATTRS = [:id, :email, :username, :name, :created_at, :trust_level, :active, :last_emailed_at]
+
+    TOPIC_ATTRS = [:id, :title, :created_at, :views, :category_id, :closed, :archived, :archetype]
+
+    POST_ATTRS = [:id, :user_id, :post_number, :raw, :created_at, :reply_to_post_number, :hidden,
+                  :hidden_reason_id, :wiki]
+
+    def categories
+      @categories ||= Category.all.to_a
+    end
+
+    def export_categories
+      data = []
+
+      categories.each do |cat|
+        data << CATEGORY_ATTRS.inject({}) { |h, a| h[a] = cat.send(a); h }
+      end
+
+      data
+    end
+
+    def export_categories!
+      @export_data[:categories] = export_categories
+
+      self
+    end
+
+    def export_category_groups
+      groups = []
+      group_names = []
+      auto_group_names = Group::AUTO_GROUPS.keys.map(&:to_s)
+
+      @export_data[:categories].each do |c|
+        c[:permissions_params].each do |group_name, _|
+          group_names << group_name unless auto_group_names.include?(group_name.to_s)
+        end
+      end
+
+      group_names.uniq!
+      return [] if group_names.empty?
+
+      Group.where(name: group_names).find_each do |group|
+        attrs = GROUP_ATTRS.inject({}) { |h, a| h[a] = group.send(a); h }
+        attrs[:user_ids] = group.users.pluck(:id)
+        groups << attrs
+      end
+
+      groups
+    end
+
+    def export_category_groups!
+      @export_data[:groups] = export_category_groups
+
+      self
+    end
+
+    def export_group_users
+      user_ids = []
+
+      @export_data[:groups].each do |g|
+        user_ids += g[:user_ids]
+      end
+
+      user_ids.uniq!
+      return [] if user_ids.empty?
+
+      users = User.where(id: user_ids)
+      export_users(users.to_a)
+    end
+
+    def export_group_users!
+      @export_data[:users] = export_group_users
+
+      self
+    end
+
+    def export_topics
+      data = []
+
+      @topics.each do |topic|
+        puts topic.title
+
+        topic_data = TOPIC_ATTRS.inject({}) { |h, a| h[a] = topic.send(a); h; }
+        topic_data[:posts] = []
+
+        topic.ordered_posts.find_each do |post|
+          h = POST_ATTRS.inject({}) { |h, a| h[a] = post.send(a); h; }
+          h[:raw] = h[:raw].gsub('src="/uploads', "src=\"#{Discourse.base_url_no_prefix}/uploads")
+          topic_data[:posts] << h
+        end
+
+        data << topic_data
+      end
+
+      data
+    end
+
+    def export_topics!
+      @export_data[:topics] = export_topics
+
+      self
+    end
+
+    def export_topic_users
+      return if @export_data[:topics].blank?
+      topic_ids = @export_data[:topics].pluck(:id)
+
+      users = User.joins(:topics).where('topics.id IN (?)', topic_ids).to_a
+      users.uniq!
+
+      export_users(users.to_a)
+    end
+
+    def export_topic_users!
+      @export_data[:users] = export_topic_users
+
+      self
+    end
+
+    def export_users(users)
+      data = []
+      users.reject! { |u| u.id == Discourse::SYSTEM_USER_ID }
+
+      users.each do |u|
+        x = USER_ATTRS.inject({}) { |h, a| h[a] = u.send(a); h; }
+        x.merge(bio_raw: u.user_profile.bio_raw,
+                website: u.user_profile.website,
+                location: u.user_profile.location)
+        data << x
+      end
+
+      data
+    end
+
+    def default_filename_prefix
+      raise "Overwrite me!"
+    end
+
+    def save_to_file(filename = nil)
+      output_basename = filename || File.join("#{default_filename_prefix}-#{Time.now.strftime("%Y-%m-%d-%H%M%S")}.json")
+      File.open(output_basename, "w:UTF-8") do |f|
+        f.write(@export_data.to_json)
+      end
+      puts "Export saved to #{output_basename}"
+      output_basename
+    end
+
+  end
+end

lib/import_export/category_exporter.rb

@@ -1,71 +1,32 @@
-module ImportExport
+require "import_export/base_exporter"
-  class CategoryExporter
+require "import_export/topic_exporter"
 
-    attr_reader :export_data
+module ImportExport
+  class CategoryExporter < BaseExporter
 
     def initialize(category_id)
       @category = Category.find(category_id)
-      @subcategories = Category.where(parent_category_id: category_id)
+      @categories = Category.where(parent_category_id: category_id).to_a
+      @categories << @category
       @export_data = {
-        users: [],
+        categories: [],
         groups: [],
-        category: nil,
+        topics: [],
-        subcategories: [],
+        users: []
-        topics: []
       }
     end
 
     def perform
       puts "Exporting category #{@category.name}...", ""
-      export_categories
+      export_categories!
+      export_category_groups!
       export_topics_and_users
       self
     end
 
-    CATEGORY_ATTRS = [:id, :name, :color, :created_at, :user_id, :slug, :description, :text_color,
-                      :auto_close_hours, :auto_close_based_on_last_post,
-                      :topic_template, :suppress_from_homepage, :all_topics_wiki, :permissions_params]
-
-    def export_categories
-      @export_data[:category] = CATEGORY_ATTRS.inject({}) { |h, a| h[a] = @category.send(a); h }
-      @subcategories.find_each do |subcat|
-        @export_data[:subcategories] << CATEGORY_ATTRS.inject({}) { |h, a| h[a] = subcat.send(a); h }
-      end
-
-      # export groups that are mentioned in category permissions
-      group_names = []
-      auto_group_names = Group::AUTO_GROUPS.keys.map(&:to_s)
-
-      ([@export_data[:category]] + @export_data[:subcategories]).each do |c|
-        c[:permissions_params].each do |group_name, _|
-          group_names << group_name unless auto_group_names.include?(group_name.to_s)
-        end
-      end
-
-      group_names.uniq!
-      export_groups(group_names) unless group_names.empty?
-
-      self
-    end
-
-    GROUP_ATTRS = [ :id, :name, :created_at, :mentionable_level, :messageable_level, :visible,
-                    :automatic_membership_email_domains, :automatic_membership_retroactive,
-                    :primary_group, :title, :grant_trust_level, :incoming_email]
-
-    def export_groups(group_names)
-      group_names.each do |name|
-        group = Group.find_by_name(name)
-        group_attrs = GROUP_ATTRS.inject({}) { |h, a| h[a] = group.send(a); h }
-        group_attrs[:user_ids] = group.users.pluck(:id)
-        @export_data[:groups] << group_attrs
-      end
-
-      self
-    end
-
     def export_topics_and_users
-      all_category_ids = [@category.id] + @subcategories.pluck(:id)
+      all_category_ids = @categories.pluck(:id)
-      description_topic_ids = Category.where(id: all_category_ids).pluck(:topic_id)
+      description_topic_ids = @categories.pluck(:topic_id)
       topic_exporter = ImportExport::TopicExporter.new(Topic.where(category_id: all_category_ids).pluck(:id) - description_topic_ids)
       topic_exporter.perform
       @export_data[:users]  = topic_exporter.export_data[:users]
@@ -73,14 +34,8 @@ module ImportExport
       self
     end
 
-    def save_to_file(filename = nil)
+    def default_filename_prefix
-      require 'json'
+      "category-export"
-      output_basename = filename || File.join("category-export-#{Time.now.strftime("%Y-%m-%d-%H%M%S")}.json")
-      File.open(output_basename, "w:UTF-8") do |f|
-        f.write(@export_data.to_json)
-      end
-      puts "Export saved to #{output_basename}"
-      output_basename
     end
 
   end

lib/import_export/category_importer.rb

@@ -1,89 +0,0 @@
-require File.join(Rails.root, 'script', 'import_scripts', 'base.rb')
-
-module ImportExport
-  class CategoryImporter < ImportScripts::Base
-    def initialize(export_data)
-      @export_data = export_data
-      @topic_importer = TopicImporter.new(@export_data)
-    end
-
-    def perform
-      RateLimiter.disable
-
-      import_users
-      import_groups
-      import_categories
-      import_topics
-      self
-    ensure
-      RateLimiter.enable
-    end
-
-    def import_groups
-      return if @export_data[:groups].empty?
-
-      @export_data[:groups].each do |group_data|
-        g = group_data.dup
-        user_ids = g.delete(:user_ids)
-        external_id = g.delete(:id)
-        new_group = Group.find_by_name(g[:name]) || Group.create!(g)
-        user_ids.each do |external_user_id|
-          new_group.add(User.find(@topic_importer.new_user_id(external_user_id))) rescue ActiveRecord::RecordNotUnique
-        end
-      end
-    end
-
-    def import_users
-      @topic_importer.import_users
-    end
-
-    def import_categories
-      id = @export_data[:category].delete(:id)
-      import_id = "#{id}#{import_source}"
-
-      parent = CategoryCustomField.where(name: 'import_id', value: import_id).first.try(:category)
-
-      unless parent
-        permissions = @export_data[:category].delete(:permissions_params)
-        parent = Category.new(@export_data[:category])
-        parent.user_id = @topic_importer.new_user_id(@export_data[:category][:user_id]) # imported user's new id
-        parent.custom_fields["import_id"] = import_id
-        parent.permissions = permissions.present? ? permissions : { "everyone" => CategoryGroup.permission_types[:full] }
-        parent.save!
-        set_category_description(parent, @export_data[:category][:description])
-      end
-
-      @export_data[:subcategories].each do |cat_attrs|
-        id = cat_attrs.delete(:id)
-        import_id = "#{id}#{import_source}"
-        existing = CategoryCustomField.where(name: 'import_id', value: import_id).first.try(:category)
-
-        unless existing
-          permissions = cat_attrs.delete(:permissions_params)
-          subcategory = Category.new(cat_attrs)
-          subcategory.parent_category_id = parent.id
-          subcategory.user_id = @topic_importer.new_user_id(cat_attrs[:user_id])
-          subcategory.custom_fields["import_id"] = import_id
-          subcategory.permissions = permissions.present? ? permissions : { "everyone" => CategoryGroup.permission_types[:full] }
-          subcategory.save!
-          set_category_description(subcategory, cat_attrs[:description])
-        end
-      end
-    end
-
-    def set_category_description(c, description)
-      post = c.topic.ordered_posts.first
-      post.raw = description
-      post.save!
-      post.rebake!
-    end
-
-    def import_topics
-      @topic_importer.import_topics
-    end
-
-    def import_source
-      @_import_source ||= "#{ENV['IMPORT_SOURCE'] || ''}"
-    end
-  end
-end

lib/import_export/category_structure_exporter.rb

@@ -0,0 +1,30 @@
+require "import_export/base_exporter"
+
+module ImportExport
+  class CategoryStructureExporter < ImportExport::BaseExporter
+
+    def initialize(include_group_users = false)
+      @include_group_users = include_group_users
+
+      @export_data = {
+        groups: [],
+        categories: []
+      }
+      @export_data[:users] = [] if @include_group_users
+    end
+
+    def perform
+      puts "Exporting all the categories...", ""
+      export_categories!
+      export_category_groups!
+      export_group_users! if @include_group_users
+
+      self
+    end
+
+    def default_filename_prefix
+      "category-structure-export"
+    end
+
+  end
+end

lib/import_export/import_export.rb

@@ -1,26 +1,26 @@
+require "import_export/importer"
+require "import_export/category_structure_exporter"
 require "import_export/category_exporter"
-require "import_export/category_importer"
 require "import_export/topic_exporter"
-require "import_export/topic_importer"
 require "json"
 
 module ImportExport
 
+  def self.import(filename)
+    data = ActiveSupport::HashWithIndifferentAccess.new(File.open(filename, "r:UTF-8") { |f| JSON.parse(f.read) })
+    ImportExport::Importer.new(data).perform
+  end
+
+  def self.export_categories(include_users, filename = nil)
+    ImportExport::CategoryStructureExporter.new(include_users).perform.save_to_file(filename)
+  end
+
   def self.export_category(category_id, filename = nil)
     ImportExport::CategoryExporter.new(category_id).perform.save_to_file(filename)
   end
 
-  def self.import_category(filename)
+  def self.export_topics(topic_ids, filename = nil)
-    export_data = ActiveSupport::HashWithIndifferentAccess.new(File.open(filename, "r:UTF-8") { |f| JSON.parse(f.read) })
+    ImportExport::TopicExporter.new(topic_ids).perform.save_to_file(filename)
-    ImportExport::CategoryImporter.new(export_data).perform
   end
 
-  def self.export_topics(topic_ids)
-    ImportExport::TopicExporter.new(topic_ids).perform.save_to_file
-  end
-
-  def self.import_topics(filename)
-    export_data = ActiveSupport::HashWithIndifferentAccess.new(File.open(filename, "r:UTF-8") { |f| JSON.parse(f.read) })
-    ImportExport::TopicImporter.new(export_data).perform
-  end
 end

lib/import_export/importer.rb

@@ -0,0 +1,160 @@
+require File.join(Rails.root, 'script', 'import_scripts', 'base.rb')
+
+module ImportExport
+  class Importer < ImportScripts::Base
+
+    def initialize(data)
+      @users = data[:users]
+      @groups = data[:groups]
+      @categories = data[:categories]
+      @topics = data[:topics]
+
+      # To support legacy `category_export` script
+      if data[:category].present?
+        @categories = [] if @categories.blank?
+        @categories << data[:category]
+      end
+    end
+
+    def perform
+      RateLimiter.disable
+
+      import_users
+      import_groups
+      import_categories
+      import_topics
+
+      self
+    ensure
+      RateLimiter.enable
+    end
+
+    def import_users
+      return if @users.blank?
+
+      @users.each do |u|
+        import_id = "#{u[:id]}#{import_source}"
+        existing = User.with_email(u[:email]).first
+
+        if existing
+          if existing.custom_fields["import_id"] != import_id
+            existing.custom_fields["import_id"] = import_id
+            existing.save!
+          end
+        else
+          u = create_user(u, import_id) # see ImportScripts::Base
+        end
+      end
+
+      self
+    end
+
+    def import_groups
+      return if @groups.blank?
+
+      @groups.each do |group_data|
+        g = group_data.dup
+        user_ids = g.delete(:user_ids)
+        external_id = g.delete(:id)
+        new_group = Group.find_by_name(g[:name]) || Group.create!(g)
+        user_ids.each do |external_user_id|
+          new_group.add(User.find(new_user_id(external_user_id))) rescue ActiveRecord::RecordNotUnique
+        end
+      end
+
+      self
+    end
+
+    def import_categories
+      return if @categories.blank?
+
+      import_ids = @categories.collect { |c| "#{c[:id]}#{import_source}" }
+      existing_categories = CategoryCustomField.where("name = 'import_id' AND value IN (?)", import_ids).select(:category_id, :value).to_a
+      existing_category_ids = existing_categories.pluck(:value)
+
+      @categories.reject! { |c| existing_category_ids.include? c[:id].to_s }
+      @categories.sort_by! { |c| c[:parent_category_id].presence || 0 }
+
+      @categories.each do |cat_attrs|
+        id = cat_attrs.delete(:id)
+        permissions = cat_attrs.delete(:permissions_params)
+
+        category = Category.new(cat_attrs)
+        category.parent_category_id = new_category_id(cat_attrs[:parent_category_id]) if cat_attrs[:parent_category_id].present?
+        category.user_id = new_user_id(cat_attrs[:user_id])
+        import_id = "#{id}#{import_source}"
+        category.custom_fields["import_id"] = import_id
+        category.permissions = permissions.present? ? permissions : { "everyone" => CategoryGroup.permission_types[:full] }
+        category.save!
+        existing_categories << { category_id: category.id, value: import_id }
+
+        if cat_attrs[:description].present?
+          post = category.topic.ordered_posts.first
+          post.raw = cat_attrs[:description]
+          post.save!
+          post.rebake!
+        end
+      end
+
+      self
+    end
+
+    def import_topics
+      return if @topics.blank?
+
+      @topics.each do |t|
+        puts ""
+        print t[:title]
+
+        first_post_attrs = t[:posts].first.merge(t.slice(*(TopicExporter::TOPIC_ATTRS - [:id, :category_id])))
+
+        first_post_attrs[:user_id] = new_user_id(first_post_attrs[:user_id])
+        first_post_attrs[:category] = new_category_id(t[:category_id])
+
+        import_id = "#{first_post_attrs[:id]}#{import_source}"
+        first_post = PostCustomField.where(name: "import_id", value: import_id).first&.post
+
+        unless first_post
+          first_post = create_post(first_post_attrs, import_id)
+        end
+
+        topic_id = first_post.topic_id
+
+        t[:posts].each_with_index do |post_data, i|
+          next if i == 0
+          print "."
+          post_import_id = "#{post_data[:id]}#{import_source}"
+          existing = PostCustomField.where(name: "import_id", value: post_import_id).first&.post
+          unless existing
+            # see ImportScripts::Base
+            create_post(
+              post_data.merge(
+                topic_id: topic_id,
+                user_id: new_user_id(post_data[:user_id])
+              ),
+              post_import_id
+            )
+          end
+        end
+      end
+
+      puts ""
+
+      self
+    end
+
+    def new_user_id(external_user_id)
+      ucf = UserCustomField.where(name: "import_id", value: "#{external_user_id}#{import_source}").first
+      ucf ? ucf.user_id : Discourse::SYSTEM_USER_ID
+    end
+
+    def new_category_id(external_category_id)
+      CategoryCustomField.where(name: "import_id", value: "#{external_category_id}#{import_source}").first.category_id rescue nil
+    end
+
+    def import_source
+      @_import_source ||= "#{ENV['IMPORT_SOURCE'] || ''}"
+    end
+
+  end
+end

lib/import_export/topic_exporter.rb

@@ -1,89 +1,26 @@
-module ImportExport
+require "import_export/base_exporter"
-  class TopicExporter
 
-    attr_reader :exported_user_ids, :export_data
+module ImportExport
+  class TopicExporter < ImportExport::BaseExporter
 
     def initialize(topic_ids)
-      @topic_ids = topic_ids
+      @topics = Topic.where(id: topic_ids).to_a
-      @exported_user_ids = []
       @export_data = {
-        users: [],
+        topics: [],
-        topics: []
+        users: []
       }
     end
 
     def perform
-      export_users
+      export_topics!
-      export_topics
+      export_topic_users!
       # TODO: user actions
 
       self
     end
 
-    USER_ATTRS = [:id, :email, :username, :name, :created_at, :trust_level, :active, :last_emailed_at]
+    def default_filename_prefix
-
+      "topic-export"
-    def export_users
-      # TODO: avatar
-
-      @exported_user_ids = []
-      @topic_ids.each do |topic_id|
-        t = Topic.find(topic_id)
-        t.posts.includes(user: [:user_profile]).find_each do |post|
-          u = post.user
-          unless @exported_user_ids.include?(u.id)
-            x = USER_ATTRS.inject({}) { |h, a| h[a] = u.send(a); h; }
-            @export_data[:users] << x.merge(bio_raw: u.user_profile.bio_raw,
-                                            website: u.user_profile.website,
-                                            location: u.user_profile.location)
-            @exported_user_ids << u.id
-          end
-        end
-      end
-
-      self
-    end
-
-    def export_topics
-      @topic_ids.each do |topic_id|
-        t = Topic.find(topic_id)
-        puts t.title
-        export_topic(t)
-      end
-      puts ""
-    end
-
-    TOPIC_ATTRS = [:id, :title, :created_at, :views, :category_id, :closed, :archived, :archetype]
-    POST_ATTRS = [:id, :user_id, :post_number, :raw, :created_at, :reply_to_post_number,
-                  :hidden, :hidden_reason_id, :wiki]
-
-    def export_topic(topic)
-      topic_data = {}
-
-      TOPIC_ATTRS.each do |a|
-        topic_data[a] = topic.send(a)
-      end
-
-      topic_data[:posts] = []
-
-      topic.ordered_posts.find_each do |post|
-        h = POST_ATTRS.inject({}) { |h, a| h[a] = post.send(a); h; }
-        h[:raw] = h[:raw].gsub('src="/uploads', "src=\"#{Discourse.base_url_no_prefix}/uploads")
-        topic_data[:posts] << h
-      end
-
-      @export_data[:topics] << topic_data
-
-      self
-    end
-
-    def save_to_file(filename = nil)
-      require 'json'
-      output_basename = filename || File.join("topic-export-#{Time.now.strftime("%Y-%m-%d-%H%M%S")}.json")
-      File.open(output_basename, "w:UTF-8") do |f|
-        f.write(@export_data.to_json)
-      end
-      puts "Export saved to #{output_basename}"
-      output_basename
     end
 
   end

lib/import_export/topic_importer.rb

@@ -1,90 +0,0 @@
-require File.join(Rails.root, 'script', 'import_scripts', 'base.rb')
-
-module ImportExport
-  class TopicImporter < ImportScripts::Base
-    def initialize(export_data)
-      @export_data = export_data
-    end
-
-    def perform
-      RateLimiter.disable
-
-      import_users
-      import_topics
-      self
-    ensure
-      RateLimiter.enable
-    end
-
-    def import_users
-      @export_data[:users].each do |u|
-        import_id = "#{u[:id]}#{import_source}"
-        existing = User.with_email(u[:email]).first
-        if existing
-          if existing.custom_fields["import_id"] != import_id
-            existing.custom_fields["import_id"] = import_id
-            existing.save!
-          end
-        else
-          u = create_user(u, import_id) # see ImportScripts::Base
-        end
-      end
-      self
-    end
-
-    def import_topics
-      @export_data[:topics].each do |t|
-        puts ""
-        print t[:title]
-
-        first_post_attrs = t[:posts].first.merge(t.slice(*(TopicExporter::TOPIC_ATTRS - [:id, :category_id])))
-
-        first_post_attrs[:user_id] = new_user_id(first_post_attrs[:user_id])
-        first_post_attrs[:category] = new_category_id(t[:category_id])
-
-        import_id = "#{first_post_attrs[:id]}#{import_source}"
-        first_post = PostCustomField.where(name: "import_id", value: import_id).first&.post
-
-        unless first_post
-          first_post = create_post(first_post_attrs, import_id)
-        end
-
-        topic_id = first_post.topic_id
-
-        t[:posts].each_with_index do |post_data, i|
-          next if i == 0
-          print "."
-          post_import_id = "#{post_data[:id]}#{import_source}"
-          existing = PostCustomField.where(name: "import_id", value: post_import_id).first&.post
-          unless existing
-            # see ImportScripts::Base
-            create_post(
-              post_data.merge(
-                topic_id: topic_id,
-                user_id: new_user_id(post_data[:user_id])
-              ),
-              post_import_id
-            )
-          end
-        end
-      end
-
-      puts ""
-
-      self
-    end
-
-    def new_user_id(external_user_id)
-      ucf = UserCustomField.where(name: "import_id", value: "#{external_user_id}#{import_source}").first
-      ucf ? ucf.user_id : Discourse::SYSTEM_USER_ID
-    end
-
-    def new_category_id(external_category_id)
-      CategoryCustomField.where(name: "import_id", value: "#{external_category_id}#{import_source}").first.category_id rescue nil
-    end
-
-    def import_source
-      @_import_source ||= "#{ENV['IMPORT_SOURCE'] || ''}"
-    end
-  end
-end

lib/tasks/export.rake

@@ -0,0 +1,7 @@
+desc 'Export all the categories'
+task 'export:categories', [:include_group_users, :file_name] => [:environment] do |_, args|
+  require "import_export/import_export"
+
+  ImportExport.export_categories(args[:include_group_users], args[:file_name])
+  puts "", "Done", ""
+end

lib/tasks/import.rake

@@ -500,3 +500,11 @@ task "import:create_vbulletin_permalinks" => :environment do
 
   log "Done!"
 end
+
+desc 'Import existing exported file'
+task 'import:file', [:file_name] => [:environment] do |_, args|
+  require "import_export/import_export"
+
+  ImportExport.import(args[:file_name])
+  puts "", "Done", ""
+end

script/discourse

@@ -198,7 +198,7 @@ class DiscourseCLI < Thor
     puts "Starting import from #{filename}..."
     load_rails
     load_import_export
-    ImportExport.import_category(filename)
+    ImportExport.import(filename)
     puts "", "Done", ""
   end
 
@@ -218,7 +218,7 @@ class DiscourseCLI < Thor
     puts "Starting import from #{filename}..."
     load_rails
     load_import_export
-    ImportExport.import_topics(filename)
+    ImportExport.import(filename)
     puts "", "Done", ""
   end
 

spec/fabricators/group_user_fabricator.rb

@@ -0,0 +1,4 @@
+Fabricator(:group_user) do
+  user
+  group
+end

spec/fixtures/json/import-export.json

@@ -0,0 +1,31 @@
+{
+  "groups":[
+    {"id":41,"name":"custom_group","created_at":"2017-10-26T15:33:46.328Z","mentionable_level":0,"messageable_level":0,"visibility_level":0,"automatic_membership_email_domains":"","automatic_membership_retroactive":false,"primary_group":false,"title":null,"grant_trust_level":null,"incoming_email":null,"user_ids":[1]},
+    {"id":42,"name":"custom_group_import","created_at":"2017-10-26T15:33:46.328Z","mentionable_level":0,"messageable_level":0,"visibility_level":0,"automatic_membership_email_domains":"","automatic_membership_retroactive":false,"primary_group":false,"title":null,"grant_trust_level":null,"incoming_email":null,"user_ids":[2]}
+  ],
+  "categories":[
+    {"id":8,"name":"Custom Category","color":"AB9364","created_at":"2017-10-26T15:32:44.083Z","user_id":1,"slug":"custom-category","description":null,"text_color":"FFFFFF","auto_close_hours":null,"parent_category_id":3,"auto_close_based_on_last_post":false,"topic_template":"","suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{"custom_group":1,"everyone":2}},
+    {"id":10,"name":"Site Feedback Import","color":"808281","created_at":"2017-10-26T17:12:39.995Z","user_id":-1,"slug":"site-feedback-import","description":"Discussion about this site, its organization, how it works, and how we can improve it.","text_color":"FFFFFF","auto_close_hours":null,"parent_category_id":null,"auto_close_based_on_last_post":false,"topic_template":null,"suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{}},
+    {"id":11,"name":"Uncategorized Import","color":"AB9364","created_at":"2017-10-26T17:12:32.359Z","user_id":-1,"slug":"uncategorized-import","description":"","text_color":"FFFFFF","auto_close_hours":null,"parent_category_id":null,"auto_close_based_on_last_post":false,"topic_template":null,"suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{}},
+    {"id":12,"name":"Lounge Import","color":"EEEEEE","created_at":"2017-10-26T17:12:39.490Z","user_id":-1,"slug":"lounge-import","description":"A category exclusive to members with trust level 3 and higher.","text_color":"652D90","auto_close_hours":null,"parent_category_id":null,"auto_close_based_on_last_post":false,"topic_template":null,"suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{"trust_level_3":1}},
+    {"id":13,"name":"Staff Import","color":"283890","created_at":"2017-10-26T17:12:42.806Z","user_id":2,"slug":"staff-import","description":"Private category for staff discussions. Topics are only visible to admins and moderators.","text_color":"FFFFFF","auto_close_hours":null,"parent_category_id":null,"auto_close_based_on_last_post":false,"topic_template":null,"suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{"custom_group_import":1,"staff":1}},
+    {"id":15,"name":"Custom Category Import","color":"AB9364","created_at":"2017-10-26T15:32:44.083Z","user_id":2,"slug":"custom-category-import","description":null,"text_color":"FFFFFF","auto_close_hours":null,"parent_category_id":10,"auto_close_based_on_last_post":false,"topic_template":"","suppress_from_homepage":false,"all_topics_wiki":false,"permissions_params":{"custom_group_import":1,"everyone":2}}
+  ],
+  "users":[
+    {"id":1,"email":"vinothkannan@example.com","username":"example","name":"Example","created_at":"2017-10-07T15:01:24.597Z","trust_level":4,"active":true,"last_emailed_at":null},
+    {"id":2,"email":"vinoth.kannan@discourse.org","username":"vinothkannans","name":"Vinoth Kannan","created_at":"2017-10-07T15:01:24.597Z","trust_level":4,"active":true,"last_emailed_at":null}
+  ],
+  "topics":[
+    {"id":7,"title":"Assets for the site design","created_at":"2017-10-26T17:15:04.590Z","views":0,"category_id":8,"closed":false,"archived":false,"archetype":"regular",
+      "posts":[
+        {"id":10,"user_id":-1,"post_number":1,"raw":"This topic, visible only to staff, is for storing images and files used in the site design.","created_at":"2017-10-26T17:15:04.720Z","reply_to_post_number":null,"hidden":false,"hidden_reason_id":null,"wiki":false}
+      ]
+    },
+    {"id":6,"title":"Privacy Policy","created_at":"2017-10-26T17:15:04.009Z","views":0,"category_id":15,"closed":false,"archived":false,"archetype":"regular",
+      "posts":[
+        {"id":8,"user_id":-1,"post_number":1,"raw":"[Third party links](#third-party)\n\nOccasionally, at our discretion, we may include or offer third party products or services on our site.","created_at":"2017-10-26T17:15:03.535Z","reply_to_post_number":null,"hidden":false,"hidden_reason_id":null,"wiki":false},
+        {"id":7,"user_id":-1,"post_number":2,"raw":"Edit the first post in this topic to change the contents of the FAQ/Guidelines page.","created_at":"2017-10-26T17:15:03.822Z","reply_to_post_number":null,"hidden":false,"hidden_reason_id":null,"wiki":false}
+      ]
+    }
+  ]
+}

spec/import_export/category_exporter_spec.rb

@@ -0,0 +1,42 @@
+require "rails_helper"
+require "import_export/category_exporter"
+
+describe ImportExport::CategoryExporter do
+
+  let(:category) { Fabricate(:category) }
+  let(:group) { Fabricate(:group) }
+  let(:user) { Fabricate(:user) }
+
+  context '.perform' do
+    it 'raises an error when the category is not found' do
+      expect { ImportExport::CategoryExporter.new(100).perform }.to raise_error(ActiveRecord::RecordNotFound)
+    end
+
+    it 'export the category when it is found' do
+      data = ImportExport::CategoryExporter.new(category.id).perform.export_data
+
+      expect(data[:categories].count).to eq(1)
+      expect(data[:groups].count).to eq(0)
+    end
+
+    it 'export the category with permission groups' do
+      category_group = Fabricate(:category_group, category: category, group: group)
+      data = ImportExport::CategoryExporter.new(category.id).perform.export_data
+
+      expect(data[:categories].count).to eq(1)
+      expect(data[:groups].count).to eq(1)
+    end
+
+    it 'export the category with topics and users' do
+      topic1 = Fabricate(:topic, category: category, user_id: -1)
+      topic2 = Fabricate(:topic, category: category, user: user)
+      data = ImportExport::CategoryExporter.new(category.id).perform.export_data
+
+      expect(data[:categories].count).to eq(1)
+      expect(data[:groups].count).to eq(0)
+      expect(data[:topics].count).to eq(2)
+      expect(data[:users].count).to eq(1)
+    end
+  end
+
+end

spec/import_export/category_structure_exporter_spec.rb

@@ -0,0 +1,39 @@
+require "rails_helper"
+require "import_export/category_structure_exporter"
+
+describe ImportExport::CategoryStructureExporter do
+
+  it 'export all the categories' do
+    category = Fabricate(:category)
+    data = ImportExport::CategoryStructureExporter.new.perform.export_data
+
+    expect(data[:categories].count).to eq(2)
+    expect(data[:groups].count).to eq(0)
+    expect(data[:users].blank?).to eq(true)
+  end
+
+  it 'export all the categories with permission groups' do
+    category = Fabricate(:category)
+    group = Fabricate(:group)
+    category_group = Fabricate(:category_group, category: category, group: group)
+    data = ImportExport::CategoryStructureExporter.new.perform.export_data
+
+    expect(data[:categories].count).to eq(2)
+    expect(data[:groups].count).to eq(1)
+    expect(data[:users].blank?).to eq(true)
+  end
+
+  it 'export all the categories with permission groups and users' do
+    category = Fabricate(:category)
+    group = Fabricate(:group)
+    user = Fabricate(:user)
+    category_group = Fabricate(:category_group, category: category, group: group)
+    group_user = Fabricate(:group_user, group: group, user: user)
+    data = ImportExport::CategoryStructureExporter.new(true).perform.export_data
+
+    expect(data[:categories].count).to eq(2)
+    expect(data[:groups].count).to eq(1)
+    expect(data[:users].count).to eq(1)
+  end
+
+end

spec/import_export/importer_spec.rb

@@ -0,0 +1,68 @@
+require "rails_helper"
+require "import_export/category_exporter"
+require "import_export/category_structure_exporter"
+require "import_export/importer"
+
+describe ImportExport::Importer do
+
+  let(:import_data) do
+    import_file = Rack::Test::UploadedFile.new(file_from_fixtures("import-export.json", "json"))
+    data = ActiveSupport::HashWithIndifferentAccess.new(JSON.parse(import_file.read))
+  end
+
+  def import(data)
+    ImportExport::Importer.new(data).perform
+  end
+
+  context '.perform' do
+
+    it 'topics and users' do
+      data = import_data.dup
+      data[:categories] = nil
+      data[:groups] = nil
+
+      expect {
+        import(data)
+      }.to change { Category.count }.by(0)
+        .and change { Group.count }.by(0)
+        .and change { Topic.count }.by(2)
+        .and change { User.count }.by(2)
+    end
+
+    it 'categories and groups' do
+      data = import_data.dup
+      data[:topics] = nil
+      data[:users] = nil
+
+      expect {
+        import(data)
+      }.to change { Category.count }.by(6)
+        .and change { Group.count }.by(2)
+        .and change { Topic.count }.by(6)
+        .and change { User.count }.by(0)
+    end
+
+    it 'categories, groups and users' do
+      data = import_data.dup
+      data[:topics] = nil
+
+      expect {
+        import(data)
+      }.to change { Category.count }.by(6)
+        .and change { Group.count }.by(2)
+        .and change { Topic.count }.by(6)
+        .and change { User.count }.by(2)
+    end
+
+    it 'all' do
+      expect {
+        import(import_data)
+      }.to change { Category.count }.by(6)
+        .and change { Group.count }.by(2)
+        .and change { Topic.count }.by(8)
+        .and change { User.count }.by(2)
+    end
+
+  end
+
+end

spec/import_export/topic_exporter_spec.rb

@@ -0,0 +1,30 @@
+require "rails_helper"
+require "import_export/topic_exporter"
+
+describe ImportExport::TopicExporter do
+
+  let(:user) { Fabricate(:user) }
+  let(:topic) { Fabricate(:topic, user: user) }
+
+  context '.perform' do
+    it 'export a single topic' do
+      data = ImportExport::TopicExporter.new([topic.id]).perform.export_data
+
+      expect(data[:categories].blank?).to eq(true)
+      expect(data[:groups].blank?).to eq(true)
+      expect(data[:topics].count).to eq(1)
+      expect(data[:users].count).to eq(1)
+    end
+
+    it 'export multiple topics' do
+      topic2 = Fabricate(:topic, user: user)
+      data = ImportExport::TopicExporter.new([topic.id, topic2.id]).perform.export_data
+
+      expect(data[:categories].blank?).to eq(true)
+      expect(data[:groups].blank?).to eq(true)
+      expect(data[:topics].count).to eq(2)
+      expect(data[:users].count).to eq(1)
+    end
+  end
+
+end