FEATURE: Allows CSV file result to be attached in automated PMs (#318)

This commit adds an optional setting that allows to attach query results
in CSV format as a file to PMs sent by Data Explorer's automation
scripts.

meta topic: https://meta.discourse.org/t/turn-data-explorer-query-results-into-csv-to-attach-to-discourse-automated-emails/267529

Co-authored-by: Drenmi <drenmi@gmail.com>
This commit is contained in:
锦心 2024-08-27 11:41:12 +08:00 committed by GitHub
parent 68760cd3a5
commit cbae98f6ed
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 155 additions and 43 deletions

View File

@ -179,49 +179,27 @@ module ::DiscourseDataExplorer
render json: { success: false, errors: [err_msg] }, status: 422
else
pg_result = result[:pg_result]
cols = pg_result.fields
content_disposition =
"attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult"
respond_to do |format|
format.json do
if params[:download]
response.headers[
"Content-Disposition"
] = "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.json"
end
json = {
success: true,
errors: [],
duration: (result[:duration_secs].to_f * 1000).round(1),
result_count: pg_result.values.length || 0,
params: query_params,
columns: cols,
default_limit: SiteSetting.data_explorer_query_result_limit,
}
json[:explain] = result[:explain] if opts[:explain]
response.headers["Content-Disposition"] = "#{content_disposition}.json" if params[
:download
]
if !params[:download]
relations, colrender = DataExplorer.add_extra_data(pg_result)
json[:relations] = relations
json[:colrender] = colrender
end
json[:rows] = pg_result.values
render json: json
render json:
ResultFormatConverter.convert(
:json,
result,
query_params:,
download: params[:download],
)
end
format.csv do
response.headers[
"Content-Disposition"
] = "attachment; filename=#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"
response.headers["Content-Disposition"] = "#{content_disposition}.csv"
require "csv"
text =
CSV.generate do |csv|
csv << cols
pg_result.values.each { |row| csv << row }
end
render plain: text
render plain: ResultFormatConverter.convert(:csv, result)
end
end
end

View File

@ -116,3 +116,5 @@ en:
label: Data Explorer Query parameters
skip_empty:
label: Skip sending PM if there are no results
attach_csv:
label: Attach the CSV file to the PM

View File

@ -9,13 +9,13 @@ module ::DiscourseDataExplorer
recipients = filter_recipients_by_query_access(recipients, query)
params = params_to_hash(query_params)
result = DataExplorer.run_query(query, params)[:pg_result]
result = DataExplorer.run_query(query, params)
query.update!(last_run_at: Time.now)
return [] if opts[:skip_empty] && result.values.empty?
table = ResultToMarkdown.convert(result)
return [] if opts[:skip_empty] && result[:pg_result].values.empty?
table = ResultToMarkdown.convert(result[:pg_result])
build_report_pms(query, table, recipients)
build_report_pms(query, table, recipients, attach_csv: opts[:attach_csv], result:)
end
private
@ -40,8 +40,20 @@ module ::DiscourseDataExplorer
params_hash
end
def self.build_report_pms(query, table = "", targets = [])
def self.build_report_pms(query, table = "", targets = [], attach_csv: false, result: nil)
pms = []
upload =
if attach_csv
tmp_filename =
"#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"
tmp = Tempfile.new(tmp_filename)
tmp.write(ResultFormatConverter.convert(:csv, result))
tmp.rewind
UploadCreator.new(tmp, tmp_filename, type: "csv_export").create_for(
Discourse.system_user.id,
)
end
targets.each do |target|
name = target[0]
pm_type = "target_#{target[1]}s"
@ -53,6 +65,9 @@ module ::DiscourseDataExplorer
"Query Name:\n#{query.name}\n\nHere are the results:\n#{table}\n\n" +
"<a href='#{Discourse.base_url}/admin/plugins/explorer?id=#{query.id}'>View query in Data Explorer</a>\n\n" +
"Report created at #{Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S")} (#{Time.zone.name})"
if upload
pm["raw"] << "\n\nAppendix: [#{upload.original_filename}|attachment](#{upload.short_url})"
end
pms << pm
end
pms

View File

@ -0,0 +1,59 @@
# frozen_string_literal: true
module ::DiscourseDataExplorer
class ResultFormatConverter
def self.convert(file_type, result, opts = {})
self.new(result, opts).send("to_#{file_type}")
end
def initialize(result, opts)
@result = result
@opts = opts
end
private
attr_reader :result
attr_reader :opts
def pg_result
@pg_result ||= @result[:pg_result]
end
def cols
@cols ||= pg_result.fields
end
def to_csv
require "csv"
CSV.generate do |csv|
csv << cols
pg_result.values.each { |row| csv << row }
end
end
def to_json
json = {
success: true,
errors: [],
duration: (result[:duration_secs].to_f * 1000).round(1),
result_count: pg_result.values.length || 0,
params: opts[:query_params],
columns: cols,
default_limit: SiteSetting.data_explorer_query_result_limit,
}
json[:explain] = result[:explain] if opts[:explain]
if !opts[:download]
relations, colrender = DataExplorer.add_extra_data(pg_result)
json[:relations] = relations
json[:colrender] = colrender
end
json[:rows] = pg_result.values
json
end
#TODO: we can move ResultToMarkdown here
end
end

View File

@ -79,6 +79,7 @@ after_initialize do
require_relative "lib/report_generator"
require_relative "lib/result_to_markdown"
require_relative "lib/result_format_converter"
reloadable_patch do
if defined?(DiscourseAutomation)
add_automation_scriptable("recurring_data_explorer_result_pm") do
@ -90,6 +91,7 @@ after_initialize do
field :query_id, component: :choices, required: true, extra: { content: queries }
field :query_params, component: :"key-value", accepts_placeholders: true
field :skip_empty, component: :boolean
field :attach_csv, component: :boolean
version 1
triggerables [:recurring]
@ -99,6 +101,7 @@ after_initialize do
query_id = fields.dig("query_id", "value")
query_params = fields.dig("query_params", "value") || {}
skip_empty = fields.dig("skip_empty", "value") || false
attach_csv = fields.dig("attach_csv", "value") || false
unless SiteSetting.data_explorer_enabled
Rails.logger.warn "#{DiscourseDataExplorer::PLUGIN_NAME} - plugin must be enabled to run automation #{automation.id}"
@ -111,7 +114,7 @@ after_initialize do
end
DiscourseDataExplorer::ReportGenerator
.generate(query_id, query_params, recipients, { skip_empty: })
.generate(query_id, query_params, recipients, { skip_empty:, attach_csv: })
.each do |pm|
begin
utils.send_pm(pm, automation_id: automation.id, prefers_encrypt: false)

View File

@ -130,5 +130,25 @@ describe DiscourseDataExplorer::ReportGenerator do
expect(result[1]["target_group_names"]).to eq([group.name])
expect(result[2]["target_emails"]).to eq(["john@doe.com"])
end
it "works with attached csv file" do
SiteSetting.personal_message_enabled_groups = group.id
DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
freeze_time
result =
described_class.generate(query.id, query_params, [user.username], { attach_csv: true })
filename =
"#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"
expect(result[0]["raw"]).to include(
"Hi #{user.username}, your data explorer report is ready.\n\n" +
"Query Name:\n#{query.name}\n\nHere are the results:\nle table\n\n" +
"<a href='#{Discourse.base_url}/admin/plugins/explorer?id=#{query.id}'>View query in Data Explorer</a>\n\n" +
"Report created at #{Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S")} (#{Time.zone.name})\n\n" +
"Appendix: [#{filename}|attachment](upload://",
)
end
end
end

View File

@ -0,0 +1,35 @@
# frozen_string_literal: true
describe DiscourseDataExplorer::ResultFormatConverter do
fab!(:user)
fab!(:post)
fab!(:query) { DiscourseDataExplorer::Query.find(-1) }
let(:query_params) { [{ from_days_ago: 0 }, { duration_days: 15 }] }
let(:query_result) { DiscourseDataExplorer::DataExplorer.run_query(query, query_params) }
before { SiteSetting.data_explorer_enabled = true }
describe ".convert" do
context "for csv files" do
it "format results as a csv table with headers and columns" do
result = described_class.convert(:csv, query_result)
table = <<~CSV
liker_user_id,liked_user_id,count
CSV
expect(result).to include(table)
end
end
context "for json files" do
it "format results as a json file" do
result = described_class.convert(:json, query_result, { query_params: })
expect(result[:columns]).to contain_exactly("liker_user_id", "liked_user_id", "count")
expect(result[:params]).to eq(query_params)
end
end
end
end