From 705753216cc632b4f6505d2000926ab3b73d8628 Mon Sep 17 00:00:00 2001 From: David Battersby Date: Fri, 24 Mar 2023 16:38:42 +0800 Subject: [PATCH] FEATURE: Allow data explorer query result to be sent as recurring PM (#233) This feature enables admins to create reports automatically based on a recurring schedule. It introduces a new automation script that includes the new email_group_user field added to discourse-automation, along with a query_id and query_params to pass in parameters to the existing data explorer query. The output of the report will be sent via pm (as a markdown table) to the recipients entered within the automation script. The automation (supports individual users, email addresses and groups). --- about.json | 7 ++ config/locales/client.en.yml | 11 +++ config/locales/server.en.yml | 5 ++ lib/report_generator.rb | 76 ++++++++++++++++ lib/result_to_markdown.rb | 49 ++++++++++ plugin.rb | 45 ++++++++++ .../recurring_data_explorer_result_pm_spec.rb | 89 +++++++++++++++++++ spec/fabricators/query_fabricator.rb | 10 +-- spec/report_generator_spec.rb | 58 ++++++++++++ 9 files changed, 345 insertions(+), 5 deletions(-) create mode 100644 about.json create mode 100644 lib/report_generator.rb create mode 100644 lib/result_to_markdown.rb create mode 100644 spec/automation/recurring_data_explorer_result_pm_spec.rb create mode 100644 spec/report_generator_spec.rb diff --git a/about.json b/about.json new file mode 100644 index 0000000..681e3b4 --- /dev/null +++ b/about.json @@ -0,0 +1,7 @@ +{ + "tests": { + "requiredPlugins": [ + "https://github.com/discourse/discourse-automation" + ] + } +} diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index f18bc5a..51f83e6 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -96,3 +96,14 @@ en: descriptions: discourse_data_explorer: run_queries: "Run Data Explorer queries. Restrict the API key to a set of queries by specifying queries IDs." + discourse_automation: + scriptables: + recurring_data_explorer_result_pm: + fields: + recipients: + label: Send to User, Group or Email + query_id: + label: Data Explorer Query + query_params: + label: Data Explorer Query parameters + diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index d60cd3b..97044b9 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -1,3 +1,8 @@ en: site_settings: data_explorer_enabled: "Enable the Data Explorer at /admin/plugins/explorer" + discourse_automation: + scriptables: + recurring_data_explorer_result_pm: + title: Schedule a PM with Data Explorer results + description: Get scheduled reports sent to your messages each month diff --git a/lib/report_generator.rb b/lib/report_generator.rb new file mode 100644 index 0000000..8680f05 --- /dev/null +++ b/lib/report_generator.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module ::DiscourseDataExplorer + class ReportGenerator + def initialize(creator_user_id) + @creator_user_id = creator_user_id + end + + def generate(query_id, query_params, recipients) + query = DiscourseDataExplorer::Query.find(query_id) + return [] unless query + + usernames = filter_recipients_by_query_access(recipients, query) + return [] if usernames.empty? + params = params_to_hash(query_params) + + result = DataExplorer.run_query(query, params) + query.update!(last_run_at: Time.now) + + table = ResultToMarkdown.convert(result[:pg_result]) + + build_report_pms(query, table, usernames) + end + + def filter_recipients_by_query_access(recipients, query) + return [] if recipients.empty? + creator = User.find(@creator_user_id) + return [] unless Guardian.new(creator).can_send_private_messages? + + recipients.reduce([]) do |names, recipient| + if (group = Group.find_by(name: recipient)) + next names unless query.query_groups.exists?(group_id: group.id) + next names.concat group.users.pluck(:username) + elsif (user = User.find_by(username: recipient)) + next names unless Guardian.new(user).user_can_access_query?(query) + next names << recipient + end + end + end + + def params_to_hash(query_params) + params = JSON.parse(query_params) + params_hash = {} + + if !params.blank? + param_key, param_value = [], [] + params.flatten.each.with_index do |data, i| + if i % 2 == 0 + param_key << data + else + param_value << data + end + end + + params_hash = Hash[param_key.zip(param_value)] + end + + params_hash + end + + def build_report_pms(query, table = "", usernames = []) + pms = [] + usernames.flatten.compact.uniq.each do |username| + pm = {} + pm["title"] = "Scheduled Report for #{query.name}" + pm["target_usernames"] = Array(username) + pm["raw"] = "Hi #{username}, your data explorer report is ready.\n\n" + + "Query Name:\n#{query.name}\n\nHere are the results:\n#{table}\n\n" + + "View query in Data Explorer\n\n" + + "Report created at #{Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S")} (#{Time.zone.name})" + pms << pm + end + pms + end + end +end diff --git a/lib/result_to_markdown.rb b/lib/result_to_markdown.rb new file mode 100644 index 0000000..59d6a10 --- /dev/null +++ b/lib/result_to_markdown.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +include HasSanitizableFields + +module ::DiscourseDataExplorer + class ResultToMarkdown + def self.convert(pg_result) + relations, colrender = DataExplorer.add_extra_data(pg_result) + result_data = [] + + # column names to search in place of id columns (topic_id, user_id etc) + cols = %w[name title username] + + # find values from extra data, based on result id + pg_result.values.each do |row| + row_data = [] + + row.each_with_index do |col, col_index| + col_name = pg_result.fields[col_index] + related = relations.dig(colrender[col_index].to_sym) if col_index < colrender.size + + if related.is_a?(ActiveModel::ArraySerializer) + related_row = related.object.find_by(id: col) + if col_name.include?("_id") + column = cols.find { |c| related_row.try c } + else + column = related_row.try(col_name) + end + + if column.nil? + row_data[col_index] = col + else + row_data[col_index] = related_row[column] + end + else + row_data[col_index] = col + end + end + + result_data << row_data.map { |c| "| #{sanitize_field(c.to_s)} " }.join + "|\n" + end + + table_headers = pg_result.fields.map { |c| " #{c.gsub("_id", "")} |" }.join + table_body = pg_result.fields.size.times.map { " :-----: |" }.join + + "|#{table_headers}\n|#{table_body}\n#{result_data.join}" + end + end +end diff --git a/plugin.rb b/plugin.rb index bf70e95..13042ca 100644 --- a/plugin.rb +++ b/plugin.rb @@ -70,4 +70,49 @@ after_initialize do :discourse_data_explorer, { run_queries: { actions: %w[discourse_data_explorer/query#run], params: %i[id] } }, ) + + require_relative "lib/report_generator" + require_relative "lib/result_to_markdown" + reloadable_patch do + if defined?(DiscourseAutomation) + DiscourseAutomation::Scriptable::RECURRING_DATA_EXPLORER_RESULT_PM = + "recurring_data_explorer_result_pm" + add_automation_scriptable( + DiscourseAutomation::Scriptable::RECURRING_DATA_EXPLORER_RESULT_PM, + ) do + queries = + DiscourseDataExplorer::Query + .where(hidden: false) + .map { |q| { id: q.id, translated_name: q.name } } + field :recipients, component: :email_group_user, required: true + field :query_id, component: :choices, required: true, extra: { content: queries } + field :query_params, component: :"key-value", accepts_placeholders: true + + version 1 + triggerables [:recurring] + + script do |_, fields, automation| + recipients = Array(fields.dig("recipients", "value")) + query_id = fields.dig("query_id", "value") + query_params = fields.dig("query_params", "value") + + unless SiteSetting.data_explorer_enabled + Rails.logger.warn "#{DiscourseDataExplorer.plugin_name} - plugin must be enabled to run automation #{automation.id}" + next + end + + unless recipients.present? + Rails.logger.warn "#{DiscourseDataExplorer.plugin_name} - couldn't find any recipients for automation #{automation.id}" + next + end + + data_explorer_report = + DiscourseDataExplorer::ReportGenerator.new(automation.last_updated_by_id) + report_pms = data_explorer_report.generate(query_id, query_params, recipients) + + report_pms.each { |pm| utils.send_pm(pm, automation_id: automation.id) } + end + end + end + end end diff --git a/spec/automation/recurring_data_explorer_result_pm_spec.rb b/spec/automation/recurring_data_explorer_result_pm_spec.rb new file mode 100644 index 0000000..e2a50a4 --- /dev/null +++ b/spec/automation/recurring_data_explorer_result_pm_spec.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe "RecurringDataExplorerResultPm" do + fab!(:admin) { Fabricate(:admin) } + + fab!(:user) { Fabricate(:user) } + fab!(:another_user) { Fabricate(:user) } + fab!(:group_user) { Fabricate(:user) } + fab!(:not_allowed_user) { Fabricate(:user) } + + fab!(:group) { Fabricate(:group, users: [user, another_user]) } + fab!(:another_group) { Fabricate(:group, users: [group_user]) } + + fab!(:automation) do + Fabricate( + :automation, + script: DiscourseAutomation::Scriptable::RECURRING_DATA_EXPLORER_RESULT_PM, + trigger: "recurring", + ) + end + fab!(:query) { Fabricate(:query) } + fab!(:query_group) { Fabricate(:query_group, query: query, group: group) } + fab!(:query_group) { Fabricate(:query_group, query: query, group: another_group) } + + let!(:recipients) do + [user.username, not_allowed_user.username, another_user.username, another_group.name] + end + + before do + SiteSetting.data_explorer_enabled = true + SiteSetting.discourse_automation_enabled = true + + automation.upsert_field!("query_id", "choices", { value: query.id }) + automation.upsert_field!("recipients", "email_group_user", { value: recipients }) + automation.upsert_field!( + "query_params", + "key-value", + { value: [%w[from_days_ago 0], %w[duration_days 15]] }, + ) + automation.upsert_field!( + "recurrence", + "period", + { value: { interval: 1, frequency: "day" } }, + target: "trigger", + ) + automation.upsert_field!("start_date", "date_time", { value: 2.minutes.ago }, target: "trigger") + end + + context "when using recurring trigger" do + it "sends the pm at recurring date_date" do + freeze_time 1.day.from_now do + expect { Jobs::DiscourseAutomationTracker.new.execute }.to change { Topic.count }.by(3) + + title = "Scheduled Report for #{query.name}" + expect(Topic.last(3).pluck(:title)).to eq([title, title, title]) + end + end + + it "ensures only allowed users in recipients field receive reports via pm" do + expect do + automation.update(last_updated_by_id: admin.id) + automation.trigger! + end.to change { Topic.count }.by(3) + + created_topics = Topic.last(3) + expect(created_topics.pluck(:archetype)).to eq( + [Archetype.private_message, Archetype.private_message, Archetype.private_message], + ) + expect(created_topics.map { |t| t.allowed_users.pluck(:username) }).to match_array( + [ + [user.username, Discourse.system_user.username], + [another_user.username, Discourse.system_user.username], + [group_user.username, Discourse.system_user.username], + ], + ) + end + + it "has appropriate content from the report generator" do + automation.update(last_updated_by_id: admin.id) + automation.trigger! + + expect(Post.last.raw).to include( + "Hi #{group_user.username}, your data explorer report is ready.\n\nQuery Name:\n#{query.name}", + ) + end + end +end diff --git a/spec/fabricators/query_fabricator.rb b/spec/fabricators/query_fabricator.rb index 30f2eac..5e053c7 100644 --- a/spec/fabricators/query_fabricator.rb +++ b/spec/fabricators/query_fabricator.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true -Fabricator(:query, from: "DiscourseDataExplorer::Query") do - name - description - sql +Fabricator(:query, from: DiscourseDataExplorer::Query) do + name { sequence(:name) { |i| "cat#{i}" } } + description { sequence(:desc) { |i| "description #{i}" } } + sql { sequence(:sql) { |i| "SELECT * FROM users limit #{i}" } } user end -Fabricator(:query_group, from: "DiscourseDataExplorer::QueryGroup") do +Fabricator(:query_group, from: DiscourseDataExplorer::QueryGroup) do query group end diff --git a/spec/report_generator_spec.rb b/spec/report_generator_spec.rb new file mode 100644 index 0000000..74e54e1 --- /dev/null +++ b/spec/report_generator_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe DiscourseDataExplorer::ReportGenerator do + fab!(:user) { Fabricate(:user) } + fab!(:unauthorised_user) { Fabricate(:user) } + fab!(:unauthorised_group) { Fabricate(:group) } + fab!(:group) { Fabricate(:group, users: [user]) } + + fab!(:query) { DiscourseDataExplorer::Query.find(-1) } + fab!(:query_group) { Fabricate(:query_group, query: query, group: group) } + + let(:query_params) { [%w[from_days_ago 0], %w[duration_days 15]] } + + before { SiteSetting.data_explorer_enabled = true } + + describe ".generate" do + it "returns [] if the creator cannot send PMs" do + result = described_class.new(user.id).generate(query.id, query_params, [user.username]) + + expect(result).to eq [] + end + + it "returns [] if the recipient is not in query group" do + result = + described_class.new(user.id).generate( + query.id, + query_params, + [unauthorised_user.username, unauthorised_group.name], + ) + + expect(result).to eq [] + end + + it "returns a list of pms for authorised users" do + SiteSetting.personal_message_enabled_groups = group.id + DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table") + freeze_time + + result = described_class.new(user.id).generate(query.id, query_params, [user.username]) + + expect(result).to eq( + [ + { + "title" => "Scheduled Report for #{query.name}", + "target_usernames" => [user.username], + "raw" => + "Hi #{user.username}, your data explorer report is ready.\n\n" + + "Query Name:\n#{query.name}\n\nHere are the results:\nle table\n\n" + + "View query in Data Explorer\n\n" + + "Report created at #{Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S")} (#{Time.zone.name})", + }, + ], + ) + end + end +end