DEV: Create ExportUserArchive as clone of ExportCsvFile
This is in preparation for improvements to the user archive export data. Some refactors happened along the way, including calling the different _export methods 'components' of the zip file. Additionally, make the test for post export much more comprehensive. Copy sources: app/jobs/regular/export_csv_file.rb spec/jobs/export_csv_file_spec.rb
This commit is contained in:
Kane York
Kane York
parent
58cdf87674
commit
a8560d741f
|
|
@ -0,0 +1,213 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'csv'
|
||||
|
||||
module Jobs
|
||||
class ExportUserArchive < ::Jobs::Base
|
||||
sidekiq_options retry: false
|
||||
|
||||
attr_accessor :current_user
|
||||
attr_accessor :extra
|
||||
|
||||
COMPONENTS ||= %w(
|
||||
user_archive
|
||||
user_archive_profile
|
||||
)
|
||||
|
||||
HEADER_ATTRS_FOR ||= HashWithIndifferentAccess.new(
|
||||
user_archive: ['topic_title', 'categories', 'is_pm', 'post', 'like_count', 'reply_count', 'url', 'created_at'],
|
||||
user_archive_profile: ['location', 'website', 'bio', 'views'],
|
||||
)
|
||||
|
||||
def execute(args)
|
||||
@current_user = User.find_by(id: args[:user_id])
|
||||
@extra = HashWithIndifferentAccess.new(args[:args]) if args[:args]
|
||||
@timestamp ||= Time.now.strftime("%y%m%d-%H%M%S")
|
||||
|
||||
components = []
|
||||
|
||||
COMPONENTS.each do |name|
|
||||
h = { name: name, method: :"#{name}_export" }
|
||||
h[:filetype] = :csv
|
||||
filename_method = :"#{name}_filename"
|
||||
if respond_to? filename_method
|
||||
h[:filename] = public_send(filename_method)
|
||||
else
|
||||
h[:filename] = "#{name}-#{@current_user.username}-#{@timestamp}"
|
||||
end
|
||||
components.push(h)
|
||||
end
|
||||
|
||||
export_title = 'user_archive'.titleize
|
||||
filename = components.first[:filename]
|
||||
user_export = UserExport.create(file_name: filename, user_id: @current_user.id)
|
||||
|
||||
filename = "#{filename}-#{user_export.id}"
|
||||
dirname = "#{UserExport.base_directory}/#{filename}"
|
||||
|
||||
# ensure directory exists
|
||||
FileUtils.mkdir_p(dirname) unless Dir.exists?(dirname)
|
||||
|
||||
# Generate a compressed CSV file
|
||||
zip_filename = nil
|
||||
begin
|
||||
components.each do |component|
|
||||
case component[:filetype]
|
||||
when :csv
|
||||
CSV.open("#{dirname}/#{component[:filename]}.csv", "w") do |csv|
|
||||
csv << get_header(component[:name])
|
||||
public_send(component[:method]).each { |d| csv << d }
|
||||
end
|
||||
else
|
||||
raise 'unknown export filetype'
|
||||
end
|
||||
end
|
||||
|
||||
zip_filename = Compression::Zip.new.compress(UserExport.base_directory, filename)
|
||||
ensure
|
||||
FileUtils.rm_rf(dirname)
|
||||
end
|
||||
|
||||
# create upload
|
||||
upload = nil
|
||||
|
||||
if File.exist?(zip_filename)
|
||||
File.open(zip_filename) do |file|
|
||||
upload = UploadCreator.new(
|
||||
file,
|
||||
File.basename(zip_filename),
|
||||
type: 'csv_export',
|
||||
for_export: 'true'
|
||||
).create_for(@current_user.id)
|
||||
|
||||
if upload.persisted?
|
||||
user_export.update_columns(upload_id: upload.id)
|
||||
else
|
||||
Rails.logger.warn("Failed to upload the file #{zip_filename}")
|
||||
end
|
||||
end
|
||||
|
||||
File.delete(zip_filename)
|
||||
end
|
||||
ensure
|
||||
post = notify_user(upload, export_title)
|
||||
|
||||
if user_export.present? && post.present?
|
||||
topic = post.topic
|
||||
user_export.update_columns(topic_id: topic.id)
|
||||
topic.update_status('closed', true, Discourse.system_user)
|
||||
end
|
||||
end
|
||||
|
||||
def user_archive_export
|
||||
return enum_for(:user_archive_export) unless block_given?
|
||||
|
||||
Post.includes(topic: :category)
|
||||
.where(user_id: @current_user.id)
|
||||
.select(:topic_id, :post_number, :raw, :like_count, :reply_count, :created_at)
|
||||
.order(:created_at)
|
||||
.with_deleted
|
||||
.each do |user_archive|
|
||||
yield get_user_archive_fields(user_archive)
|
||||
end
|
||||
end
|
||||
|
||||
def user_archive_profile_export
|
||||
return enum_for(:user_archive_profile_export) unless block_given?
|
||||
|
||||
UserProfile
|
||||
.where(user_id: @current_user.id)
|
||||
.select(:location, :website, :bio_raw, :views)
|
||||
.each do |user_profile|
|
||||
yield get_user_archive_profile_fields(user_profile)
|
||||
end
|
||||
end
|
||||
|
||||
def get_header(entity)
|
||||
if entity == 'user_list'
|
||||
header_array = HEADER_ATTRS_FOR['user_list'] + HEADER_ATTRS_FOR['user_stats'] + HEADER_ATTRS_FOR['user_profile']
|
||||
header_array.concat(HEADER_ATTRS_FOR['user_sso']) if SiteSetting.enable_sso
|
||||
user_custom_fields = UserField.all
|
||||
if user_custom_fields.present?
|
||||
user_custom_fields.each do |custom_field|
|
||||
header_array.push("#{custom_field.name} (custom user field)")
|
||||
end
|
||||
end
|
||||
header_array.push("group_names")
|
||||
else
|
||||
header_array = HEADER_ATTRS_FOR[entity]
|
||||
end
|
||||
|
||||
header_array
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def get_user_archive_fields(user_archive)
|
||||
user_archive_array = []
|
||||
topic_data = user_archive.topic
|
||||
user_archive = user_archive.as_json
|
||||
topic_data = Topic.with_deleted.find_by(id: user_archive['topic_id']) if topic_data.nil?
|
||||
return user_archive_array if topic_data.nil?
|
||||
|
||||
all_categories = Category.all.to_h { |category| [category.id, category] }
|
||||
|
||||
categories = "-"
|
||||
if topic_data.category_id && category = all_categories[topic_data.category_id]
|
||||
categories = [category.name]
|
||||
while category.parent_category_id && category = all_categories[category.parent_category_id]
|
||||
categories << category.name
|
||||
end
|
||||
categories = categories.reverse.join("|")
|
||||
end
|
||||
|
||||
is_pm = topic_data.archetype == "private_message" ? I18n.t("csv_export.boolean_yes") : I18n.t("csv_export.boolean_no")
|
||||
url = "#{Discourse.base_url}/t/#{topic_data.slug}/#{topic_data.id}/#{user_archive['post_number']}"
|
||||
|
||||
topic_hash = { "post" => user_archive['raw'], "topic_title" => topic_data.title, "categories" => categories, "is_pm" => is_pm, "url" => url }
|
||||
user_archive.merge!(topic_hash)
|
||||
|
||||
HEADER_ATTRS_FOR['user_archive'].each do |attr|
|
||||
user_archive_array.push(user_archive[attr])
|
||||
end
|
||||
|
||||
user_archive_array
|
||||
end
|
||||
|
||||
def get_user_archive_profile_fields(user_profile)
|
||||
user_archive_profile = []
|
||||
|
||||
HEADER_ATTRS_FOR['user_archive_profile'].each do |attr|
|
||||
data =
|
||||
if attr == 'bio'
|
||||
user_profile.attributes['bio_raw']
|
||||
else
|
||||
user_profile.attributes[attr]
|
||||
end
|
||||
|
||||
user_archive_profile.push(data)
|
||||
end
|
||||
|
||||
user_archive_profile
|
||||
end
|
||||
|
||||
def notify_user(upload, export_title)
|
||||
post = nil
|
||||
|
||||
if @current_user
|
||||
post = if upload
|
||||
SystemMessage.create_from_system_user(
|
||||
@current_user,
|
||||
:csv_export_succeeded,
|
||||
download_link: UploadMarkdown.new(upload).attachment_markdown,
|
||||
export_title: export_title
|
||||
)
|
||||
else
|
||||
SystemMessage.create_from_system_user(@current_user, :csv_export_failed)
|
||||
end
|
||||
end
|
||||
|
||||
post
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'rails_helper'
|
||||
require 'csv'
|
||||
|
||||
describe Jobs::ExportUserArchive do
|
||||
context '#execute' do
|
||||
let(:user) { Fabricate(:user, username: "john_doe") }
|
||||
let(:post) { Fabricate(:post, user: user) }
|
||||
|
||||
before do
|
||||
_ = post
|
||||
user.user_profile.website = 'https://doe.example.com/john'
|
||||
user.user_profile.save
|
||||
end
|
||||
|
||||
after do
|
||||
user.uploads.each(&:destroy!)
|
||||
end
|
||||
|
||||
it 'raises an error when the user is missing' do
|
||||
expect { Jobs::ExportCsvFile.new.execute(user_id: user.id + (1 << 20)) }.to raise_error(Discourse::InvalidParameters)
|
||||
end
|
||||
|
||||
it 'works' do
|
||||
expect do
|
||||
Jobs::ExportUserArchive.new.execute(
|
||||
user_id: user.id,
|
||||
)
|
||||
end.to change { Upload.count }.by(1)
|
||||
|
||||
system_message = user.topics_allowed.last
|
||||
|
||||
expect(system_message.title).to eq(I18n.t(
|
||||
"system_messages.csv_export_succeeded.subject_template",
|
||||
export_title: "User Archive"
|
||||
))
|
||||
|
||||
upload = system_message.first_post.uploads.first
|
||||
|
||||
expect(system_message.first_post.raw).to eq(I18n.t(
|
||||
"system_messages.csv_export_succeeded.text_body_template",
|
||||
download_link: "[#{upload.original_filename}|attachment](#{upload.short_url}) (#{upload.filesize} Bytes)"
|
||||
).chomp)
|
||||
|
||||
expect(system_message.id).to eq(UserExport.last.topic_id)
|
||||
expect(system_message.closed).to eq(true)
|
||||
|
||||
files = []
|
||||
Zip::File.open(Discourse.store.path_for(upload)) do |zip_file|
|
||||
zip_file.each { |entry| files << entry.name }
|
||||
end
|
||||
|
||||
expect(files.size).to eq(Jobs::ExportUserArchive::COMPONENTS.length)
|
||||
expect(files.find { |f| f.match 'user_archive-john_doe-' }).to_not be_nil
|
||||
expect(files.find { |f| f.match 'user_archive_profile-john_doe-' }).to_not be_nil
|
||||
end
|
||||
end
|
||||
|
||||
context 'user_archive posts' do
|
||||
let(:component) { 'user_archive' }
|
||||
let(:user) { Fabricate(:user, username: "john_doe") }
|
||||
let(:user2) { Fabricate(:user) }
|
||||
let(:job) {
|
||||
j = Jobs::ExportUserArchive.new
|
||||
j.current_user = user
|
||||
j
|
||||
}
|
||||
let(:category) { Fabricate(:category_with_definition) }
|
||||
let(:subcategory) { Fabricate(:category_with_definition, parent_category_id: category.id) }
|
||||
let(:subsubcategory) { Fabricate(:category_with_definition, parent_category_id: subcategory.id) }
|
||||
let(:subsubtopic) { Fabricate(:topic, category: subsubcategory) }
|
||||
let(:subsubpost) { Fabricate(:post, user: user, topic: subsubtopic) }
|
||||
|
||||
let(:topic) { Fabricate(:topic, category: category) }
|
||||
let(:normal_post) { Fabricate(:post, user: user, topic: topic) }
|
||||
let(:reply) { PostCreator.new(user2, raw: 'asdf1234qwert7896', topic_id: topic.id, reply_to_post_number: normal_post.post_number).create }
|
||||
|
||||
let(:message) { Fabricate(:private_message_topic) }
|
||||
let(:message_post) { Fabricate(:post, user: user, topic: message) }
|
||||
|
||||
it 'properly exports posts' do
|
||||
SiteSetting.max_category_nesting = 3
|
||||
[reply, subsubpost, message_post]
|
||||
|
||||
PostActionCreator.like(user2, normal_post)
|
||||
|
||||
rows = []
|
||||
job.user_archive_export do |row|
|
||||
rows << Jobs::ExportUserArchive::HEADER_ATTRS_FOR['user_archive'].zip(row).to_h
|
||||
end
|
||||
|
||||
expect(rows.length).to eq(3)
|
||||
|
||||
post1 = rows.find { |r| r['topic_title'] == topic.title }
|
||||
post2 = rows.find { |r| r['topic_title'] == subsubtopic.title }
|
||||
post3 = rows.find { |r| r['topic_title'] == message.title }
|
||||
|
||||
expect(post1["categories"]).to eq("#{category.name}")
|
||||
expect(post2["categories"]).to eq("#{category.name}|#{subcategory.name}|#{subsubcategory.name}")
|
||||
expect(post3["categories"]).to eq("-")
|
||||
|
||||
expect(post1["is_pm"]).to eq(I18n.t("csv_export.boolean_no"))
|
||||
expect(post2["is_pm"]).to eq(I18n.t("csv_export.boolean_no"))
|
||||
expect(post3["is_pm"]).to eq(I18n.t("csv_export.boolean_yes"))
|
||||
|
||||
expect(post1["post"]).to eq(normal_post.raw)
|
||||
expect(post2["post"]).to eq(subsubpost.raw)
|
||||
expect(post3["post"]).to eq(message_post.raw)
|
||||
|
||||
expect(post1['like_count']).to eq(1)
|
||||
expect(post2['like_count']).to eq(0)
|
||||
|
||||
expect(post1['reply_count']).to eq(1)
|
||||
expect(post2['reply_count']).to eq(0)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
context 'user_archive_profile' do
|
||||
let(:component) { 'user_archive_profile' }
|
||||
let(:user) { Fabricate(:user, username: "john_doe") }
|
||||
let(:job) {
|
||||
j = Jobs::ExportUserArchive.new
|
||||
j.current_user = user
|
||||
j
|
||||
}
|
||||
|
||||
before do
|
||||
user.user_profile.website = 'https://doe.example.com/john'
|
||||
user.user_profile.bio_raw = "I am John Doe\n\nHere I am"
|
||||
user.user_profile.save
|
||||
end
|
||||
|
||||
it 'properly includes the profile fields' do
|
||||
csv_out = CSV.generate do |csv|
|
||||
csv << job.get_header(component)
|
||||
job.user_archive_profile_export.each { |d| csv << d }
|
||||
end
|
||||
|
||||
expect(csv_out).to match('doe.example.com')
|
||||
expect(csv_out).to match("Doe\n\nHere")
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
Loading…
Reference in New Issue