From 901a45eeb3af3d0754868e4adcedc6fa85cdb2c0 Mon Sep 17 00:00:00 2001 From: Kane York Date: Wed, 9 Dec 2020 15:34:13 -0800 Subject: [PATCH] FEATURE: Add likes, flags to user data export (#11439) This commit is dedicated to https://twitter.com/FiloSottile/status/1335666583126073354 for reminding me that like timestamps are valuable data. Likes additionally include the topic_id and post_number of the acted post, to aid in analysis. Flag export does not include the disposition by staff. --- app/jobs/regular/export_user_archive.rb | 89 +++++++++++++++++++++++++ spec/jobs/export_user_archive_spec.rb | 63 +++++++++++++++-- 2 files changed, 148 insertions(+), 4 deletions(-) diff --git a/app/jobs/regular/export_user_archive.rb b/app/jobs/regular/export_user_archive.rb index e5a8a6b7e7d..bb09b02fc0d 100644 --- a/app/jobs/regular/export_user_archive.rb +++ b/app/jobs/regular/export_user_archive.rb @@ -18,6 +18,9 @@ module Jobs badges bookmarks category_preferences + flags + likes + post_actions queued_posts visits ) @@ -30,6 +33,9 @@ module Jobs badges: ['badge_id', 'badge_name', 'granted_at', 'post_id', 'seq', 'granted_manually', 'notification_id', 'featured_rank'], bookmarks: ['post_id', 'topic_id', 'post_number', 'link', 'name', 'created_at', 'updated_at', 'reminder_type', 'reminder_at', 'reminder_last_sent_at', 'reminder_set_at', 'auto_delete_preference'], category_preferences: ['category_id', 'category_names', 'notification_level', 'dismiss_new_timestamp'], + flags: ['id', 'post_id', 'flag_type', 'created_at', 'updated_at', 'deleted_at', 'deleted_by', 'related_post_id', 'targets_topic', 'was_take_action'], + likes: ['id', 'post_id', 'topic_id', 'post_number', 'created_at', 'updated_at', 'deleted_at', 'deleted_by'], + post_actions: ['id', 'post_id', 'post_action_type', 'created_at', 'updated_at', 'deleted_at', 'deleted_by', 'related_post_id'], queued_posts: ['id', 'verdict', 'category_id', 'topic_id', 'post_raw', 'other_json'], visits: ['visited_at', 'posts_read', 'mobile', 'time_read'], ) @@ -267,6 +273,79 @@ module Jobs end end + def flags_export + return enum_for(:flags_export) unless block_given? + + PostAction + .with_deleted + .where(user_id: @current_user.id) + .where(post_action_type_id: PostActionType.flag_types.values) + .each do |pa| + yield [ + pa.id, + pa.post_id, + PostActionType.flag_types[pa.post_action_type_id], + pa.created_at, + pa.updated_at, + pa.deleted_at, + self_or_other(pa.deleted_by_id), + pa.related_post_id, + pa.targets_topic, + # renamed to 'was_take_action' to avoid possibility of thinking this is a synonym of agreed_at + pa.staff_took_action, + ] + end + end + + def likes_export + return enum_for(:likes_export) unless block_given? + PostAction + .with_deleted + .where(user_id: @current_user.id) + .where(post_action_type_id: PostActionType.types[:like]) + .each do |pa| + post = Post.with_deleted.find(pa.post_id) + yield [ + pa.id, + pa.post_id, + post&.topic_id, + post&.post_number, + pa.created_at, + pa.updated_at, + pa.deleted_at, + self_or_other(pa.deleted_by_id), + ] + end + end + + def include_post_actions? + # Most forums should not have post_action records other than flags and likes, but they are possible in historical oddities. + PostAction + .where(user_id: @current_user.id) + .where.not(post_action_type_id: PostActionType.flag_types.values + [PostActionType.types[:like], PostActionType.types[:bookmark]]) + .exists? + end + + def post_actions_export + return enum_for(:likes_export) unless block_given? + PostAction + .with_deleted + .where(user_id: @current_user.id) + .where.not(post_action_type_id: PostActionType.flag_types.values + [PostActionType.types[:like], PostActionType.types[:bookmark]]) + .each do |pa| + yield [ + pa.id, + pa.post_id, + PostActionType.types[pa.post_action_type] || pa.post_action_type, + pa.created_at, + pa.updated_at, + pa.deleted_at, + self_or_other(pa.deleted_by_id), + pa.related_post_id, + ] + end + end + def queued_posts_export return enum_for(:queued_posts_export) unless block_given? @@ -337,6 +416,16 @@ module Jobs categories.reverse.join("|") end + def self_or_other(user_id) + if user_id.nil? + nil + elsif user_id == @current_user.id + 'self' + else + 'other' + end + end + def get_user_archive_fields(user_archive) user_archive_array = [] topic_data = user_archive.topic diff --git a/spec/jobs/export_user_archive_spec.rb b/spec/jobs/export_user_archive_spec.rb index b837fa2dc8a..efe6d5d473e 100644 --- a/spec/jobs/export_user_archive_spec.rb +++ b/spec/jobs/export_user_archive_spec.rb @@ -14,9 +14,11 @@ describe Jobs::ExportUserArchive do } let(:component) { raise 'component not set' } + let(:admin) { Fabricate(:admin) } let(:category) { Fabricate(:category_with_definition) } let(:subcategory) { Fabricate(:category_with_definition, parent_category_id: category.id) } let(:topic) { Fabricate(:topic, category: category) } + let(:post) { Fabricate(:post, user: user, topic: topic) } def make_component_csv data_rows = [] @@ -35,8 +37,6 @@ describe Jobs::ExportUserArchive do end context '#execute' do - let(:post) { Fabricate(:post, user: user) } - before do _ = post user.user_profile.website = 'https://doe.example.com/john' @@ -46,6 +46,9 @@ describe Jobs::ExportUserArchive do 'HTTP_USER_AGENT' => 'MyWebBrowser', 'REQUEST_PATH' => '/some_path/456852', }).log_on_user(user, {}, {}) + + # force a nonstandard post action + PostAction.new(user: user, post: post, post_action_type_id: 5).save end after do @@ -210,7 +213,6 @@ describe Jobs::ExportUserArchive do context 'badges' do let(:component) { 'badges' } - let(:admin) { Fabricate(:admin) } let(:badge1) { Fabricate(:badge) } let(:badge2) { Fabricate(:badge, multiple_grant: true) } let(:badge3) { Fabricate(:badge, multiple_grant: true) } @@ -351,11 +353,64 @@ describe Jobs::ExportUserArchive do end end + context 'flags' do + let(:component) { 'flags' } + let(:other_post) { Fabricate(:post, user: admin) } + let(:post3) { Fabricate(:post) } + let(:post4) { Fabricate(:post) } + + it 'correctly exports flags' do + result0 = PostActionCreator.notify_moderators(user, other_post, "helping out the admins") + PostActionCreator.spam(user, post3) + PostActionDestroyer.destroy(user, post3, :spam) + PostActionCreator.inappropriate(user, post3) + result3 = PostActionCreator.off_topic(user, post4) + result3.reviewable.perform(admin, :agree_and_keep) + + data, csv_out = make_component_csv + expect(data.length).to eq(4) + + expect(data[0]['post_id']).to eq(other_post.id.to_s) + expect(data[0]['flag_type']).to eq('notify_moderators') + expect(data[0]['related_post_id']).to eq(result0.post_action.related_post_id.to_s) + + expect(data[1]['flag_type']).to eq('spam') + expect(data[2]['flag_type']).to eq('inappropriate') + expect(data[1]['deleted_at']).to_not be_empty + expect(data[1]['deleted_by']).to eq('self') + expect(data[2]['deleted_at']).to be_empty + + expect(data[3]['post_id']).to eq(post4.id.to_s) + expect(data[3]['flag_type']).to eq('off_topic') + expect(data[3]['deleted_at']).to be_empty + end + end + + context 'likes' do + let(:component) { 'likes' } + let(:other_post) { Fabricate(:post, user: admin) } + let(:post3) { Fabricate(:post) } + + it 'correctly exports likes' do + PostActionCreator.like(user, other_post) + PostActionCreator.like(user, post3) + PostActionCreator.like(admin, post3) + PostActionDestroyer.destroy(user, post3, :like) + + data, csv_out = make_component_csv + expect(data.length).to eq(2) + + expect(data[0]['post_id']).to eq(other_post.id.to_s) + expect(data[1]['post_id']).to eq(post3.id.to_s) + expect(data[1]['deleted_at']).to_not be_empty + expect(data[1]['deleted_by']).to eq('self') + end + end + context 'queued posts' do let(:component) { 'queued_posts' } let(:reviewable_post) { Fabricate(:reviewable_queued_post, topic: topic, created_by: user) } let(:reviewable_topic) { Fabricate(:reviewable_queued_post_topic, category: category, created_by: user) } - let(:admin) { Fabricate(:admin) } it 'correctly exports queued posts' do SiteSetting.tagging_enabled = true