From a8560d741f07afb25e0ec23cb45d054154d9fada Mon Sep 17 00:00:00 2001
From: Kane York <kanepyork@gmail.com>
Date: Thu, 27 Aug 2020 15:37:58 -0700
Subject: [PATCH] DEV: Create ExportUserArchive as clone of ExportCsvFile

This is in preparation for improvements to the user archive export data.
Some refactors happened along the way, including calling the different _export methods 'components' of the zip file.

Additionally, make the test for post export much more comprehensive.

Copy sources:
  app/jobs/regular/export_csv_file.rb
  spec/jobs/export_csv_file_spec.rb
---
 app/jobs/regular/export_user_archive.rb | 213 ++++++++++++++++++++++++
 spec/jobs/export_user_archive_spec.rb   | 146 ++++++++++++++++
 2 files changed, 359 insertions(+)
 create mode 100644 app/jobs/regular/export_user_archive.rb
 create mode 100644 spec/jobs/export_user_archive_spec.rb

diff --git a/app/jobs/regular/export_user_archive.rb b/app/jobs/regular/export_user_archive.rb
new file mode 100644
index 00000000000..81bb03b8b98
--- /dev/null
+++ b/app/jobs/regular/export_user_archive.rb
@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Jobs
+  class ExportUserArchive < ::Jobs::Base
+    sidekiq_options retry: false
+
+    attr_accessor :current_user
+    attr_accessor :extra
+
+    COMPONENTS ||= %w(
+      user_archive
+      user_archive_profile
+    )
+
+    HEADER_ATTRS_FOR ||= HashWithIndifferentAccess.new(
+      user_archive: ['topic_title', 'categories', 'is_pm', 'post', 'like_count', 'reply_count', 'url', 'created_at'],
+      user_archive_profile: ['location', 'website', 'bio', 'views'],
+    )
+
+    def execute(args)
+      @current_user = User.find_by(id: args[:user_id])
+      @extra = HashWithIndifferentAccess.new(args[:args]) if args[:args]
+      @timestamp ||= Time.now.strftime("%y%m%d-%H%M%S")
+
+      components = []
+
+      COMPONENTS.each do |name|
+        h = { name: name, method: :"#{name}_export" }
+        h[:filetype] = :csv
+        filename_method = :"#{name}_filename"
+        if respond_to? filename_method
+          h[:filename] = public_send(filename_method)
+        else
+          h[:filename] = "#{name}-#{@current_user.username}-#{@timestamp}"
+        end
+        components.push(h)
+      end
+
+      export_title = 'user_archive'.titleize
+      filename = components.first[:filename]
+      user_export = UserExport.create(file_name: filename, user_id: @current_user.id)
+
+      filename = "#{filename}-#{user_export.id}"
+      dirname = "#{UserExport.base_directory}/#{filename}"
+
+      # ensure directory exists
+      FileUtils.mkdir_p(dirname) unless Dir.exists?(dirname)
+
+      # Generate a compressed CSV file
+      zip_filename = nil
+      begin
+        components.each do |component|
+          case component[:filetype]
+          when :csv
+            CSV.open("#{dirname}/#{component[:filename]}.csv", "w") do |csv|
+              csv << get_header(component[:name])
+              public_send(component[:method]).each { |d| csv << d }
+            end
+          else
+            raise 'unknown export filetype'
+          end
+        end
+
+        zip_filename = Compression::Zip.new.compress(UserExport.base_directory, filename)
+      ensure
+        FileUtils.rm_rf(dirname)
+      end
+
+      # create upload
+      upload = nil
+
+      if File.exist?(zip_filename)
+        File.open(zip_filename) do |file|
+          upload = UploadCreator.new(
+            file,
+            File.basename(zip_filename),
+            type: 'csv_export',
+            for_export: 'true'
+          ).create_for(@current_user.id)
+
+          if upload.persisted?
+            user_export.update_columns(upload_id: upload.id)
+          else
+            Rails.logger.warn("Failed to upload the file #{zip_filename}")
+          end
+        end
+
+        File.delete(zip_filename)
+      end
+    ensure
+      post = notify_user(upload, export_title)
+
+      if user_export.present? && post.present?
+        topic = post.topic
+        user_export.update_columns(topic_id: topic.id)
+        topic.update_status('closed', true, Discourse.system_user)
+      end
+    end
+
+    def user_archive_export
+      return enum_for(:user_archive_export) unless block_given?
+
+      Post.includes(topic: :category)
+        .where(user_id: @current_user.id)
+        .select(:topic_id, :post_number, :raw, :like_count, :reply_count, :created_at)
+        .order(:created_at)
+        .with_deleted
+        .each do |user_archive|
+        yield get_user_archive_fields(user_archive)
+      end
+    end
+
+    def user_archive_profile_export
+      return enum_for(:user_archive_profile_export) unless block_given?
+
+      UserProfile
+        .where(user_id: @current_user.id)
+        .select(:location, :website, :bio_raw, :views)
+        .each do |user_profile|
+        yield get_user_archive_profile_fields(user_profile)
+      end
+    end
+
+    def get_header(entity)
+      if entity == 'user_list'
+        header_array = HEADER_ATTRS_FOR['user_list'] + HEADER_ATTRS_FOR['user_stats'] + HEADER_ATTRS_FOR['user_profile']
+        header_array.concat(HEADER_ATTRS_FOR['user_sso']) if SiteSetting.enable_sso
+        user_custom_fields = UserField.all
+        if user_custom_fields.present?
+          user_custom_fields.each do |custom_field|
+            header_array.push("#{custom_field.name} (custom user field)")
+          end
+        end
+        header_array.push("group_names")
+      else
+        header_array = HEADER_ATTRS_FOR[entity]
+      end
+
+      header_array
+    end
+
+    private
+
+    def get_user_archive_fields(user_archive)
+      user_archive_array = []
+      topic_data = user_archive.topic
+      user_archive = user_archive.as_json
+      topic_data = Topic.with_deleted.find_by(id: user_archive['topic_id']) if topic_data.nil?
+      return user_archive_array if topic_data.nil?
+
+      all_categories = Category.all.to_h { |category| [category.id, category] }
+
+      categories = "-"
+      if topic_data.category_id && category = all_categories[topic_data.category_id]
+        categories = [category.name]
+        while category.parent_category_id && category = all_categories[category.parent_category_id]
+          categories << category.name
+        end
+        categories = categories.reverse.join("|")
+      end
+
+      is_pm = topic_data.archetype == "private_message" ? I18n.t("csv_export.boolean_yes") : I18n.t("csv_export.boolean_no")
+      url = "#{Discourse.base_url}/t/#{topic_data.slug}/#{topic_data.id}/#{user_archive['post_number']}"
+
+      topic_hash = { "post" => user_archive['raw'], "topic_title" => topic_data.title, "categories" => categories, "is_pm" => is_pm, "url" => url }
+      user_archive.merge!(topic_hash)
+
+      HEADER_ATTRS_FOR['user_archive'].each do |attr|
+        user_archive_array.push(user_archive[attr])
+      end
+
+      user_archive_array
+    end
+
+    def get_user_archive_profile_fields(user_profile)
+      user_archive_profile = []
+
+      HEADER_ATTRS_FOR['user_archive_profile'].each do |attr|
+        data =
+          if attr == 'bio'
+            user_profile.attributes['bio_raw']
+          else
+            user_profile.attributes[attr]
+          end
+
+          user_archive_profile.push(data)
+      end
+
+      user_archive_profile
+    end
+
+    def notify_user(upload, export_title)
+      post = nil
+
+      if @current_user
+        post = if upload
+          SystemMessage.create_from_system_user(
+            @current_user,
+            :csv_export_succeeded,
+            download_link: UploadMarkdown.new(upload).attachment_markdown,
+            export_title: export_title
+          )
+        else
+          SystemMessage.create_from_system_user(@current_user, :csv_export_failed)
+        end
+      end
+
+      post
+    end
+  end
+end
diff --git a/spec/jobs/export_user_archive_spec.rb b/spec/jobs/export_user_archive_spec.rb
new file mode 100644
index 00000000000..db37aaa65bb
--- /dev/null
+++ b/spec/jobs/export_user_archive_spec.rb
@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'csv'
+
+describe Jobs::ExportUserArchive do
+  context '#execute' do
+    let(:user) { Fabricate(:user, username: "john_doe") }
+    let(:post) { Fabricate(:post, user: user) }
+
+    before do
+      _ = post
+      user.user_profile.website = 'https://doe.example.com/john'
+      user.user_profile.save
+    end
+
+    after do
+      user.uploads.each(&:destroy!)
+    end
+
+    it 'raises an error when the user is missing' do
+      expect { Jobs::ExportCsvFile.new.execute(user_id: user.id + (1 << 20)) }.to raise_error(Discourse::InvalidParameters)
+    end
+
+    it 'works' do
+      expect do
+        Jobs::ExportUserArchive.new.execute(
+          user_id: user.id,
+        )
+      end.to change { Upload.count }.by(1)
+
+      system_message = user.topics_allowed.last
+
+      expect(system_message.title).to eq(I18n.t(
+        "system_messages.csv_export_succeeded.subject_template",
+        export_title: "User Archive"
+      ))
+
+      upload = system_message.first_post.uploads.first
+
+      expect(system_message.first_post.raw).to eq(I18n.t(
+        "system_messages.csv_export_succeeded.text_body_template",
+        download_link: "[#{upload.original_filename}|attachment](#{upload.short_url}) (#{upload.filesize} Bytes)"
+      ).chomp)
+
+      expect(system_message.id).to eq(UserExport.last.topic_id)
+      expect(system_message.closed).to eq(true)
+
+      files = []
+      Zip::File.open(Discourse.store.path_for(upload)) do |zip_file|
+        zip_file.each { |entry| files << entry.name }
+      end
+
+      expect(files.size).to eq(Jobs::ExportUserArchive::COMPONENTS.length)
+      expect(files.find { |f| f.match 'user_archive-john_doe-' }).to_not be_nil
+      expect(files.find { |f| f.match 'user_archive_profile-john_doe-' }).to_not be_nil
+    end
+  end
+
+  context 'user_archive posts' do
+    let(:component) { 'user_archive' }
+    let(:user) { Fabricate(:user, username: "john_doe") }
+    let(:user2) { Fabricate(:user) }
+    let(:job) {
+      j = Jobs::ExportUserArchive.new
+      j.current_user = user
+      j
+    }
+    let(:category) { Fabricate(:category_with_definition) }
+    let(:subcategory) { Fabricate(:category_with_definition, parent_category_id: category.id) }
+    let(:subsubcategory) { Fabricate(:category_with_definition, parent_category_id: subcategory.id) }
+    let(:subsubtopic) { Fabricate(:topic, category: subsubcategory) }
+    let(:subsubpost) { Fabricate(:post, user: user, topic: subsubtopic) }
+
+    let(:topic) { Fabricate(:topic, category: category) }
+    let(:normal_post) { Fabricate(:post, user: user, topic: topic) }
+    let(:reply) { PostCreator.new(user2, raw: 'asdf1234qwert7896', topic_id: topic.id, reply_to_post_number: normal_post.post_number).create }
+
+    let(:message) { Fabricate(:private_message_topic) }
+    let(:message_post) { Fabricate(:post, user: user, topic: message) }
+
+    it 'properly exports posts' do
+      SiteSetting.max_category_nesting = 3
+      [reply, subsubpost, message_post]
+
+      PostActionCreator.like(user2, normal_post)
+
+      rows = []
+      job.user_archive_export do |row|
+        rows << Jobs::ExportUserArchive::HEADER_ATTRS_FOR['user_archive'].zip(row).to_h
+      end
+
+      expect(rows.length).to eq(3)
+
+      post1 = rows.find { |r| r['topic_title'] == topic.title }
+      post2 = rows.find { |r| r['topic_title'] == subsubtopic.title }
+      post3 = rows.find { |r| r['topic_title'] == message.title }
+
+      expect(post1["categories"]).to eq("#{category.name}")
+      expect(post2["categories"]).to eq("#{category.name}|#{subcategory.name}|#{subsubcategory.name}")
+      expect(post3["categories"]).to eq("-")
+
+      expect(post1["is_pm"]).to eq(I18n.t("csv_export.boolean_no"))
+      expect(post2["is_pm"]).to eq(I18n.t("csv_export.boolean_no"))
+      expect(post3["is_pm"]).to eq(I18n.t("csv_export.boolean_yes"))
+
+      expect(post1["post"]).to eq(normal_post.raw)
+      expect(post2["post"]).to eq(subsubpost.raw)
+      expect(post3["post"]).to eq(message_post.raw)
+
+      expect(post1['like_count']).to eq(1)
+      expect(post2['like_count']).to eq(0)
+
+      expect(post1['reply_count']).to eq(1)
+      expect(post2['reply_count']).to eq(0)
+    end
+
+  end
+
+  context 'user_archive_profile' do
+    let(:component) { 'user_archive_profile' }
+    let(:user) { Fabricate(:user, username: "john_doe") }
+    let(:job) {
+      j = Jobs::ExportUserArchive.new
+      j.current_user = user
+      j
+    }
+
+    before do
+      user.user_profile.website = 'https://doe.example.com/john'
+      user.user_profile.bio_raw = "I am John Doe\n\nHere I am"
+      user.user_profile.save
+    end
+
+    it 'properly includes the profile fields' do
+      csv_out = CSV.generate do |csv|
+        csv << job.get_header(component)
+        job.user_archive_profile_export.each { |d| csv << d }
+      end
+
+      expect(csv_out).to match('doe.example.com')
+      expect(csv_out).to match("Doe\n\nHere")
+    end
+  end
+
+end