2019-05-02 18:17:27 -04:00
# frozen_string_literal: true
2013-02-25 11:42:20 -05:00
#
2015-09-02 14:25:18 -04:00
# Helps us find topics.
# Returns a TopicList object containing the topics found.
2013-02-05 14:16:51 -05:00
#
2015-09-02 14:25:18 -04:00
2013-02-05 14:16:51 -05:00
class TopicQuery
2021-08-02 00:41:41 -04:00
include PrivateMessageLists
2018-09-03 00:45:32 -04:00
PG_MAX_INT || = 2147483647
2021-06-16 18:20:09 -04:00
DEFAULT_PER_PAGE_COUNT || = 30
2017-02-15 15:25:43 -05:00
2018-08-14 03:01:04 -04:00
def self . validators
@validators || = begin
2018-08-15 00:56:24 -04:00
int = lambda do | x |
Integer === x || ( String === x && x . match? ( / ^-?[0-9]+$ / ) )
end
2018-09-03 00:45:32 -04:00
zero_up_to_max_int = lambda do | x |
int . call ( x ) && x . to_i . between? ( 0 , PG_MAX_INT )
end
2018-08-14 03:01:04 -04:00
{
2018-09-03 00:45:32 -04:00
max_posts : zero_up_to_max_int ,
min_posts : zero_up_to_max_int ,
2019-11-18 01:58:35 -05:00
page : zero_up_to_max_int
2018-08-14 03:01:04 -04:00
}
end
end
def self . validate? ( option , value )
if fn = validators [ option . to_sym ]
fn . call ( value )
else
true
end
end
2017-02-15 17:27:10 -05:00
def self . public_valid_options
@public_valid_options || =
% i ( page
2017-03-02 14:54:26 -05:00
before
2017-03-02 15:11:38 -05:00
bumped_before
2017-02-15 15:25:43 -05:00
topic_ids
category
order
ascending
2017-02-15 17:27:10 -05:00
min_posts
max_posts
2017-02-15 15:25:43 -05:00
status
2017-02-15 17:27:10 -05:00
filter
2017-02-15 15:25:43 -05:00
state
search
2017-02-15 17:27:10 -05:00
q
2020-08-06 02:33:45 -04:00
f
2017-02-15 15:25:43 -05:00
group_name
2017-02-15 17:27:10 -05:00
tags
match_all_tags
no_subcategories
2018-03-11 00:29:34 -05:00
no_tags )
2017-02-15 17:27:10 -05:00
end
def self . valid_options
@valid_options || =
public_valid_options +
% i ( except_topic_ids
limit
page
per_page
visible
2018-01-15 00:13:29 -05:00
guardian
2018-03-13 15:59:12 -04:00
no_definitions
2020-09-14 07:07:35 -04:00
destination_category_id
include_pms )
2017-02-15 15:25:43 -05:00
end
2014-04-16 12:05:54 -04:00
# Maps `order` to a columns in `topics`
2013-11-13 14:17:06 -05:00
SORTABLE_MAPPING = {
'likes' = > 'like_count' ,
2014-10-02 23:16:53 -04:00
'op_likes' = > 'op_likes' ,
2013-11-13 14:17:06 -05:00
'views' = > 'views' ,
'posts' = > 'posts_count' ,
2014-08-18 13:26:12 -04:00
'activity' = > 'bumped_at' ,
2013-11-14 15:50:36 -05:00
'posters' = > 'participant_count' ,
2014-08-27 12:42:54 -04:00
'category' = > 'category_id' ,
'created' = > 'created_at'
2013-11-13 14:17:06 -05:00
}
2015-12-21 11:43:17 -05:00
cattr_accessor :results_filter_callbacks
self . results_filter_callbacks = [ ]
2017-02-15 15:25:43 -05:00
attr_accessor :options , :user , :guardian
def self . add_custom_filter ( key , & blk )
@custom_filters || = { }
valid_options << key
2017-02-15 17:27:10 -05:00
public_valid_options << key
2017-02-15 15:25:43 -05:00
@custom_filters [ key ] = blk
end
def self . remove_custom_filter ( key )
@custom_filters . delete ( key )
2017-02-15 17:27:10 -05:00
public_valid_options . delete ( key )
2017-02-15 15:25:43 -05:00
valid_options . delete ( key )
@custom_filters = nil if @custom_filters . length == 0
end
def self . apply_custom_filters ( results , topic_query )
if @custom_filters
@custom_filters . each do | key , filter |
results = filter . call ( results , topic_query )
end
end
results
end
2013-07-16 15:20:18 -04:00
def initialize ( user = nil , options = { } )
2017-02-15 15:25:43 -05:00
options . assert_valid_keys ( TopicQuery . valid_options )
2015-09-21 15:14:05 -04:00
@options = options . dup
2013-07-16 15:20:18 -04:00
@user = user
2018-01-15 00:13:29 -05:00
@guardian = options [ :guardian ] || Guardian . new ( @user )
2013-02-05 14:16:51 -05:00
end
2014-02-21 14:17:45 -05:00
def joined_topic_user ( list = nil )
( list || Topic ) . joins ( " LEFT OUTER JOIN topic_users AS tu ON (topics.id = tu.topic_id AND tu.user_id = #{ @user . id . to_i } ) " )
end
2018-11-11 21:04:30 -05:00
def get_pm_params ( topic )
if topic . private_message?
my_group_ids = topic . topic_allowed_groups
. joins ( "
LEFT JOIN group_users gu
ON topic_allowed_groups . group_id = gu . group_id
AND gu . user_id = #{@user.id.to_i}
" )
. where ( " gu.group_id IS NOT NULL " )
. pluck ( :group_id )
target_group_ids = topic . topic_allowed_groups . pluck ( :group_id )
target_users = topic
. topic_allowed_users
if my_group_ids . present?
# strip out users in groups you already belong to
target_users = target_users
2020-12-10 18:56:26 -05:00
. joins ( " LEFT JOIN group_users gu ON gu.user_id = topic_allowed_users.user_id AND #{ DB . sql_fragment ( 'gu.group_id IN (?)' , my_group_ids ) } " )
2018-11-11 21:04:30 -05:00
. where ( 'gu.group_id IS NULL' )
end
target_user_ids = target_users
. where ( 'NOT topic_allowed_users.user_id = ?' , @user . id )
. pluck ( :user_id )
{
topic : topic ,
my_group_ids : my_group_ids ,
target_group_ids : target_group_ids ,
target_user_ids : target_user_ids
}
end
end
def list_related_for ( topic , pm_params : nil )
return if ! topic . private_message?
return if @user . blank?
return if ! SiteSetting . enable_personal_messages?
builder = SuggestedTopicsBuilder . new ( topic )
pm_params = pm_params || get_pm_params ( topic )
if pm_params [ :my_group_ids ] . present?
builder . add_results ( related_messages_group (
pm_params . merge ( count : [ 6 , builder . results_left ] . max ,
exclude : builder . excluded_topic_ids )
) )
else
builder . add_results ( related_messages_user (
pm_params . merge ( count : [ 6 , builder . results_left ] . max ,
exclude : builder . excluded_topic_ids )
) )
end
params = { unordered : true }
params [ :preload_posters ] = true
create_list ( :suggested , params , builder . results )
end
2013-02-05 14:16:51 -05:00
# Return a list of suggested topics for a topic
2018-11-11 21:04:30 -05:00
def list_suggested_for ( topic , pm_params : nil )
2018-01-23 12:05:44 -05:00
# Don't suggest messages unless we have a user, and private messages are
# enabled.
return if topic . private_message? &&
2018-01-31 01:03:12 -05:00
( @user . blank? || ! SiteSetting . enable_personal_messages? )
2016-02-03 02:50:05 -05:00
2013-07-12 14:38:20 -04:00
builder = SuggestedTopicsBuilder . new ( topic )
2013-02-05 14:16:51 -05:00
2018-11-11 21:04:30 -05:00
pm_params = pm_params || get_pm_params ( topic )
2016-02-03 02:50:05 -05:00
2013-07-12 14:38:20 -04:00
# When logged in we start with different results
2013-07-16 15:20:18 -04:00
if @user
2016-02-03 02:50:05 -05:00
if topic . private_message?
2018-10-29 01:09:58 -04:00
builder . add_results ( new_messages (
pm_params . merge ( count : builder . results_left )
) ) unless builder . full?
builder . add_results ( unread_messages (
pm_params . merge ( count : builder . results_left )
) ) unless builder . full?
2016-02-03 02:50:05 -05:00
else
2019-04-16 03:51:57 -04:00
builder . add_results (
unread_results (
topic : topic ,
per_page : builder . results_left ,
max_age : SiteSetting . suggested_topics_unread_max_days_old
) , :high
)
2016-02-03 02:50:05 -05:00
builder . add_results ( new_results ( topic : topic , per_page : builder . category_results_left ) ) unless builder . full?
end
2013-02-05 14:16:51 -05:00
end
2018-10-28 19:47:59 -04:00
if ! topic . private_message?
2016-02-03 02:50:05 -05:00
builder . add_results ( random_suggested ( topic , builder . results_left , builder . excluded_topic_ids ) ) unless builder . full?
end
params = { unordered : true }
if topic . private_message?
params [ :preload_posters ] = true
end
create_list ( :suggested , params , builder . results )
2013-02-05 14:16:51 -05:00
end
2013-03-27 16:17:49 -04:00
# The latest view of topics
def list_latest
2014-10-08 12:44:47 -04:00
create_list ( :latest , { } , latest_results )
2013-02-05 14:16:51 -05:00
end
def list_read
2013-04-02 16:52:51 -04:00
create_list ( :read , unordered : true ) do | topics |
2017-04-26 12:26:37 -04:00
topics . where ( 'tu.last_visited_at IS NOT NULL' ) . order ( 'tu.last_visited_at DESC' )
2013-02-05 14:16:51 -05:00
end
end
def list_new
2015-02-23 00:50:52 -05:00
create_list ( :new , { unordered : true } , new_results )
2013-02-05 14:16:51 -05:00
end
def list_unread
2015-02-23 00:50:52 -05:00
create_list ( :unread , { unordered : true } , unread_results )
2013-02-05 14:16:51 -05:00
end
def list_posted
2014-03-29 18:26:13 -04:00
create_list ( :posted ) { | l | l . where ( 'tu.posted' ) }
2013-02-05 14:16:51 -05:00
end
2015-01-24 23:53:11 -05:00
def list_bookmarks
create_list ( :bookmarks ) { | l | l . where ( 'tu.bookmarked' ) }
end
2013-12-27 12:10:35 -05:00
def list_top_for ( period )
2021-07-23 13:52:35 -04:00
score_column = TopTopic . score_column_for_period ( period )
2013-12-23 18:50:36 -05:00
create_list ( :top , unordered : true ) do | topics |
2020-05-07 15:04:53 -04:00
topics = remove_muted_categories ( topics , @user )
2021-07-23 13:52:35 -04:00
topics = topics . joins ( :top_topic ) . where ( " top_topics. #{ score_column } > 0 " )
2014-09-05 01:20:39 -04:00
if period == :yearly && @user . try ( :trust_level ) == TrustLevel [ 0 ]
2021-07-23 13:52:35 -04:00
topics . order ( << ~ SQL )
CASE WHEN (
COALESCE ( topics . pinned_at , '1900-01-01' ) > COALESCE ( tu . cleared_pinned_at , '1900-01-01' )
) THEN 0 ELSE 1 END ,
top_topics . #{score_column} DESC,
topics . bumped_at DESC
SQL
2014-01-18 13:03:09 -05:00
else
2021-07-23 13:52:35 -04:00
topics . order ( << ~ SQL )
COALESCE ( top_topics . #{score_column}, 0) DESC, topics.bumped_at DESC
SQL
2014-01-18 13:03:09 -05:00
end
2013-12-23 18:50:36 -05:00
end
2014-01-13 19:02:14 -05:00
end
2013-07-24 17:15:21 -04:00
def list_topics_by ( user )
2015-03-23 18:12:37 -04:00
@options [ :filtered_to_user ] = user . id
2013-07-24 17:15:21 -04:00
create_list ( :user_topics ) do | topics |
topics . where ( user_id : user . id )
end
end
2018-03-14 07:40:28 -04:00
def list_group_topics ( group )
list = default_results . where ( "
topics . user_id IN (
SELECT user_id FROM group_users gu WHERE gu . group_id = #{group.id.to_i}
)
" )
create_list ( :group_topics , { } , list )
end
2015-02-23 00:50:52 -05:00
def list_category_topic_ids ( category )
2015-02-24 22:24:25 -05:00
query = default_results ( category : category . id )
2018-03-22 16:38:53 -04:00
pinned_ids = query . where ( 'topics.pinned_at IS NOT NULL AND topics.category_id = ?' , category . id ) . limit ( nil ) . order ( 'pinned_at DESC' ) . pluck ( :id )
non_pinned_ids = query . where ( 'topics.pinned_at IS NULL OR topics.category_id <> ?' , category . id ) . pluck ( :id )
2015-06-10 14:36:47 -04:00
( pinned_ids + non_pinned_ids )
2013-02-05 14:16:51 -05:00
end
2013-02-27 22:36:12 -05:00
def list_new_in_category ( category )
2014-02-04 15:55:30 -05:00
create_list ( :new_in_category , unordered : true , category : category . id ) do | list |
list . by_newest . first ( 25 )
2014-01-17 17:52:06 -05:00
end
2013-02-27 22:36:12 -05:00
end
DEV: Topic tracking state improvements (#13218)
I merged this PR in yesterday, finally thinking this was done https://github.com/discourse/discourse/pull/12958 but then a wild performance regression occurred. These are the problem methods:
https://github.com/discourse/discourse/blob/1aa20bd681e634f7fff22953ed62d90c2573b331/app/serializers/topic_tracking_state_serializer.rb#L13-L21
Turns out date comparison is super expensive on the backend _as well as_ the frontend.
The fix was to just move the `treat_as_new_topic_start_date` into the SQL query rather than using the slower `UserOption#treat_as_new_topic_start_date` method in ruby. After this change, 1% of the total time is spent with the `created_in_new_period` comparison instead of ~20%.
----
History:
Original PR which had to be reverted **https://github.com/discourse/discourse/pull/12555**. See the description there for what this PR is achieving, plus below.
The issue with the original PR is addressed in https://github.com/discourse/discourse/pull/12958/commits/92ef54f4020111ffacb0f2a27da5d5c2855f9d5d
If you went to the `x unread` link for a tag Chrome would freeze up and possibly crash, or eventually unfreeze after nearly 10 mins. Other routes for unread/new were similarly slow. From profiling the issue was the `sync` function of `topic-tracking-state.js`, which calls down to `isNew` which in turn calls `moment`, a change I had made in the PR above. The time it takes locally with ~1400 topics in the tracking state is 2.3 seconds.
To solve this issue, I have moved these calculations for "created in new period" and "unread not too old" into the tracking state serializer.
When I was looking at the profiler I also noticed this issue which was just compounding the problem. Every time we modify topic tracking state we recalculate the sidebar tracking/everything/tag counts. However this calls `forEachTracked` and `countTags` which can be quite expensive as they go through the whole tracking state (and were also calling the removed moment functions).
I added some logs and this was being called 30 times when navigating to a new /unread route because `sync` is being called from `build-topic-route` (one for each topic loaded due to pagination). So I just added a debounce here and it makes things even faster.
Finally, I changed topic tracking state to use a Map so our counts of the state keys is faster (Maps have .size whereas objects you have to do Object.keys(obj) which is O(n).)
<!-- NOTE: All pull requests should have tests (rspec in Ruby, qunit in JavaScript). If your code does not include test coverage, please include an explanation of why it was omitted. -->
2021-06-01 19:06:29 -04:00
def self . new_filter ( list , treat_as_new_topic_start_date : nil , treat_as_new_topic_clause_sql : nil )
if treat_as_new_topic_start_date
list = list . where ( " topics.created_at >= :created_at " , created_at : treat_as_new_topic_start_date )
else
list = list . where ( " topics.created_at >= #{ treat_as_new_topic_clause_sql } " )
end
list
2013-05-23 01:21:07 -04:00
. where ( " tu.last_read_post_number IS NULL " )
. where ( " COALESCE(tu.notification_level, :tracking) >= :tracking " , tracking : TopicUser . notification_levels [ :tracking ] )
end
DEV: Topic tracking state improvements (#13218)
I merged this PR in yesterday, finally thinking this was done https://github.com/discourse/discourse/pull/12958 but then a wild performance regression occurred. These are the problem methods:
https://github.com/discourse/discourse/blob/1aa20bd681e634f7fff22953ed62d90c2573b331/app/serializers/topic_tracking_state_serializer.rb#L13-L21
Turns out date comparison is super expensive on the backend _as well as_ the frontend.
The fix was to just move the `treat_as_new_topic_start_date` into the SQL query rather than using the slower `UserOption#treat_as_new_topic_start_date` method in ruby. After this change, 1% of the total time is spent with the `created_in_new_period` comparison instead of ~20%.
----
History:
Original PR which had to be reverted **https://github.com/discourse/discourse/pull/12555**. See the description there for what this PR is achieving, plus below.
The issue with the original PR is addressed in https://github.com/discourse/discourse/pull/12958/commits/92ef54f4020111ffacb0f2a27da5d5c2855f9d5d
If you went to the `x unread` link for a tag Chrome would freeze up and possibly crash, or eventually unfreeze after nearly 10 mins. Other routes for unread/new were similarly slow. From profiling the issue was the `sync` function of `topic-tracking-state.js`, which calls down to `isNew` which in turn calls `moment`, a change I had made in the PR above. The time it takes locally with ~1400 topics in the tracking state is 2.3 seconds.
To solve this issue, I have moved these calculations for "created in new period" and "unread not too old" into the tracking state serializer.
When I was looking at the profiler I also noticed this issue which was just compounding the problem. Every time we modify topic tracking state we recalculate the sidebar tracking/everything/tag counts. However this calls `forEachTracked` and `countTags` which can be quite expensive as they go through the whole tracking state (and were also calling the removed moment functions).
I added some logs and this was being called 30 times when navigating to a new /unread route because `sync` is being called from `build-topic-route` (one for each topic loaded due to pagination). So I just added a debounce here and it makes things even faster.
Finally, I changed topic tracking state to use a Map so our counts of the state keys is faster (Maps have .size whereas objects you have to do Object.keys(obj) which is O(n).)
<!-- NOTE: All pull requests should have tests (rspec in Ruby, qunit in JavaScript). If your code does not include test coverage, please include an explanation of why it was omitted. -->
2021-06-01 19:06:29 -04:00
def self . unread_filter ( list , staff : false )
col_name = staff ? " highest_staff_post_number " : " highest_post_number "
2016-12-02 01:03:31 -05:00
2017-05-25 15:07:12 -04:00
list
. where ( " tu.last_read_post_number < topics. #{ col_name } " )
2017-11-19 22:49:09 -05:00
. where ( " COALESCE(tu.notification_level, :regular) >= :tracking " ,
regular : TopicUser . notification_levels [ :regular ] , tracking : TopicUser . notification_levels [ :tracking ] )
2013-05-21 02:39:51 -04:00
end
2020-09-25 15:39:37 -04:00
def self . tracked_filter ( list , user_id )
sql = + << ~ SQL
topics . category_id IN (
SELECT cu . category_id FROM category_users cu
WHERE cu . user_id = :user_id AND cu . notification_level > = :tracking
)
2020-10-07 12:15:28 -04:00
OR topics . category_id IN (
SELECT c . id FROM categories c WHERE c . parent_category_id IN (
SELECT cd . category_id FROM category_users cd
WHERE cd . user_id = :user_id AND cd . notification_level > = :tracking
)
)
2020-09-25 15:39:37 -04:00
SQL
if SiteSetting . tagging_enabled
sql << << ~ SQL
OR topics . id IN (
SELECT tt . topic_id FROM topic_tags tt WHERE tt . tag_id IN (
SELECT tu . tag_id
FROM tag_users tu
WHERE tu . user_id = :user_id AND tu . notification_level > = :tracking
)
)
SQL
end
list . where (
sql ,
user_id : user_id ,
tracking : NotificationLevels . all [ :tracking ]
)
end
2015-02-23 00:50:52 -05:00
def prioritize_pinned_topics ( topics , options )
2019-05-02 18:17:27 -04:00
pinned_clause = if options [ :category_id ]
+ " topics.category_id = #{ options [ :category_id ] . to_i } AND "
else
+ " pinned_globally AND "
end
2015-02-23 00:50:52 -05:00
pinned_clause << " pinned_at IS NOT NULL "
2019-05-02 18:17:27 -04:00
2015-02-23 00:50:52 -05:00
if @user
pinned_clause << " AND (topics.pinned_at > tu.cleared_pinned_at OR tu.cleared_pinned_at IS NULL) "
end
unpinned_topics = topics . where ( " NOT ( #{ pinned_clause } ) " )
2015-06-25 16:25:50 -04:00
pinned_topics = topics . dup . offset ( nil ) . where ( pinned_clause )
2015-02-23 00:50:52 -05:00
per_page = options [ :per_page ] || per_page_setting
limit = per_page unless options [ :limit ] == false
page = options [ :page ] . to_i
if page == 0
( pinned_topics + unpinned_topics ) [ 0 ... limit ] if limit
else
2017-08-31 00:06:56 -04:00
offset = ( page * per_page ) - pinned_topics . length
2015-02-25 22:48:56 -05:00
offset = 0 unless offset > 0
unpinned_topics . offset ( offset ) . to_a
2015-02-23 00:50:52 -05:00
end
end
2015-01-08 16:44:27 -05:00
def create_list ( filter , options = { } , topics = nil )
topics || = default_results ( options )
topics = yield ( topics ) if block_given?
2015-02-23 00:50:52 -05:00
options = options . merge ( @options )
2015-12-22 19:09:17 -05:00
if [ " activity " , " default " ] . include? ( options [ :order ] || " activity " ) &&
! options [ :unordered ] &&
filter != :private_messages
2015-02-23 00:50:52 -05:00
topics = prioritize_pinned_topics ( topics , options )
end
2016-02-03 02:50:05 -05:00
topics = topics . to_a
if options [ :preload_posters ]
user_ids = [ ]
topics . each do | ft |
user_ids << ft . user_id << ft . last_post_user_id << ft . featured_user_ids << ft . allowed_user_ids
end
2020-07-17 05:48:08 -04:00
user_lookup = UserLookup . new ( user_ids )
2017-09-14 00:07:35 -04:00
2019-06-05 04:22:47 -04:00
# memoize for loop so we don't keep looking these up
translations = TopicPostersSummary . translations
2016-02-03 02:50:05 -05:00
topics . each do | t |
2017-09-14 00:07:35 -04:00
t . posters = t . posters_summary (
2020-07-17 05:48:08 -04:00
user_lookup : user_lookup ,
2019-06-05 04:22:47 -04:00
translations : translations
2017-09-14 00:07:35 -04:00
)
2016-02-03 02:50:05 -05:00
end
end
topics . each do | t |
2015-03-31 17:29:07 -04:00
t . allowed_user_ids = filter == :private_messages ? t . allowed_users . map { | u | u . id } : [ ]
2015-02-23 00:50:52 -05:00
end
2016-02-03 02:50:05 -05:00
list = TopicList . new ( filter , @user , topics , options . merge ( @options ) )
2018-08-14 19:22:03 -04:00
list . per_page = options [ :per_page ] || per_page_setting
2015-01-08 16:44:27 -05:00
list
end
def latest_results ( options = { } )
result = default_results ( options )
2020-04-30 02:48:34 -04:00
result = remove_muted_topics ( result , @user ) unless options && options [ :state ] == " muted "
2015-01-08 16:44:27 -05:00
result = remove_muted_categories ( result , @user , exclude : options [ :category ] )
2016-04-25 15:55:15 -04:00
result = remove_muted_tags ( result , @user , options )
2018-12-07 07:44:23 -05:00
result = apply_shared_drafts ( result , get_category_id ( options [ :category ] ) , options )
2015-12-21 11:43:17 -05:00
# plugins can remove topics here:
self . class . results_filter_callbacks . each do | filter_callback |
result = filter_callback . call ( :latest , result , @user , options )
end
2015-01-08 16:44:27 -05:00
result
end
def unread_results ( options = { } )
2017-05-25 15:07:12 -04:00
result = TopicQuery . unread_filter (
default_results ( options . reverse_merge ( unordered : true ) ) ,
staff : @user & . staff? )
2015-01-08 16:44:27 -05:00
. order ( 'CASE WHEN topics.user_id = tu.user_id THEN 1 ELSE 2 END' )
2015-12-21 11:43:17 -05:00
2019-04-04 21:44:36 -04:00
if @user
2019-04-05 00:55:12 -04:00
# micro optimisation so we don't load up all of user stats which we do not need
unread_at = DB . query_single (
" select first_unread_at from user_stats where user_id = ? " ,
@user . id ) . first
2019-04-16 03:51:57 -04:00
if max_age = options [ :max_age ]
max_age_date = max_age . days . ago
unread_at || = max_age_date
unread_at = unread_at > max_age_date ? unread_at : max_age_date
end
2019-04-05 00:55:12 -04:00
# perf note, in the past we tried doing this in a subquery but performance was
# terrible, also tried with a join and it was bad
result = result . where ( " topics.updated_at >= ? " , unread_at )
2019-04-04 21:44:36 -04:00
end
2015-12-21 11:43:17 -05:00
self . class . results_filter_callbacks . each do | filter_callback |
result = filter_callback . call ( :unread , result , @user , options )
end
2015-01-08 16:44:27 -05:00
suggested_ordering ( result , options )
end
def new_results ( options = { } )
2015-02-23 00:50:52 -05:00
# TODO does this make sense or should it be ordered on created_at
# it is ordering on bumped_at now
DEV: Topic tracking state improvements (#13218)
I merged this PR in yesterday, finally thinking this was done https://github.com/discourse/discourse/pull/12958 but then a wild performance regression occurred. These are the problem methods:
https://github.com/discourse/discourse/blob/1aa20bd681e634f7fff22953ed62d90c2573b331/app/serializers/topic_tracking_state_serializer.rb#L13-L21
Turns out date comparison is super expensive on the backend _as well as_ the frontend.
The fix was to just move the `treat_as_new_topic_start_date` into the SQL query rather than using the slower `UserOption#treat_as_new_topic_start_date` method in ruby. After this change, 1% of the total time is spent with the `created_in_new_period` comparison instead of ~20%.
----
History:
Original PR which had to be reverted **https://github.com/discourse/discourse/pull/12555**. See the description there for what this PR is achieving, plus below.
The issue with the original PR is addressed in https://github.com/discourse/discourse/pull/12958/commits/92ef54f4020111ffacb0f2a27da5d5c2855f9d5d
If you went to the `x unread` link for a tag Chrome would freeze up and possibly crash, or eventually unfreeze after nearly 10 mins. Other routes for unread/new were similarly slow. From profiling the issue was the `sync` function of `topic-tracking-state.js`, which calls down to `isNew` which in turn calls `moment`, a change I had made in the PR above. The time it takes locally with ~1400 topics in the tracking state is 2.3 seconds.
To solve this issue, I have moved these calculations for "created in new period" and "unread not too old" into the tracking state serializer.
When I was looking at the profiler I also noticed this issue which was just compounding the problem. Every time we modify topic tracking state we recalculate the sidebar tracking/everything/tag counts. However this calls `forEachTracked` and `countTags` which can be quite expensive as they go through the whole tracking state (and were also calling the removed moment functions).
I added some logs and this was being called 30 times when navigating to a new /unread route because `sync` is being called from `build-topic-route` (one for each topic loaded due to pagination). So I just added a debounce here and it makes things even faster.
Finally, I changed topic tracking state to use a Map so our counts of the state keys is faster (Maps have .size whereas objects you have to do Object.keys(obj) which is O(n).)
<!-- NOTE: All pull requests should have tests (rspec in Ruby, qunit in JavaScript). If your code does not include test coverage, please include an explanation of why it was omitted. -->
2021-06-01 19:06:29 -04:00
result = TopicQuery . new_filter (
default_results ( options . reverse_merge ( unordered : true ) ) ,
treat_as_new_topic_start_date : @user . user_option . treat_as_new_topic_start_date
)
2015-11-01 17:20:22 -05:00
result = remove_muted_topics ( result , @user )
2015-01-08 16:44:27 -05:00
result = remove_muted_categories ( result , @user , exclude : options [ :category ] )
2016-04-25 15:55:15 -04:00
result = remove_muted_tags ( result , @user , options )
2021-02-03 19:27:34 -05:00
result = remove_dismissed ( result , @user )
2015-12-21 11:43:17 -05:00
self . class . results_filter_callbacks . each do | filter_callback |
result = filter_callback . call ( :new , result , @user , options )
end
2015-01-08 16:44:27 -05:00
suggested_ordering ( result , options )
end
2013-02-05 14:16:51 -05:00
protected
2014-12-15 11:54:26 -05:00
def per_page_setting
2021-06-16 18:20:09 -04:00
DEFAULT_PER_PAGE_COUNT
2014-12-15 11:54:26 -05:00
end
2018-03-26 10:43:30 -04:00
def apply_shared_drafts ( result , category_id , options )
2019-08-28 21:35:31 -04:00
# PERF: avoid any penalty if there are no shared drafts enabled
# on some sites the cost can be high eg: gearbox
return result if SiteSetting . shared_drafts_category == " "
2018-03-28 15:36:12 -04:00
drafts_category_id = SiteSetting . shared_drafts_category . to_i
viewing_shared = category_id && category_id == drafts_category_id
2018-12-06 13:59:29 -05:00
2021-02-01 09:16:34 -05:00
if guardian . can_see_shared_draft?
2020-12-14 14:08:20 -05:00
if options [ :destination_category_id ]
destination_category_id = get_category_id ( options [ :destination_category_id ] )
topic_ids = SharedDraft . where ( category_id : destination_category_id ) . pluck ( :topic_id )
return result . where ( id : topic_ids )
end
if viewing_shared
return result . includes ( :shared_draft ) . references ( :shared_draft )
end
elsif viewing_shared
return result . joins ( 'LEFT OUTER JOIN shared_drafts sd ON sd.topic_id = topics.id' ) . where ( 'sd.id IS NULL' )
2018-03-26 10:43:30 -04:00
end
2020-12-14 14:08:20 -05:00
result . where ( 'topics.category_id != ?' , drafts_category_id )
2013-11-14 15:50:36 -05:00
end
2013-11-13 14:17:06 -05:00
2013-11-14 15:50:36 -05:00
def apply_ordering ( result , options )
sort_column = SORTABLE_MAPPING [ options [ :order ] ] || 'default'
sort_dir = ( options [ :ascending ] == " true " ) ? " ASC " : " DESC "
2014-10-02 23:16:53 -04:00
# If we are sorting in the default order desc, we should consider including pinned
# topics. Otherwise, just use bumped_at.
if sort_column == 'default'
2015-01-05 01:39:49 -05:00
if sort_dir == 'DESC'
# If something requires a custom order, for example "unread" which sorts the least read
# to the top, do nothing
return result if options [ :unordered ]
2014-10-02 23:16:53 -04:00
end
2013-11-13 14:17:06 -05:00
sort_column = 'bumped_at'
2018-06-07 01:28:18 -04:00
end
2014-10-02 23:16:53 -04:00
2016-02-25 11:22:23 -05:00
# If we are sorting by category, actually use the name
if sort_column == 'category_id'
# TODO forces a table scan, slow
2021-07-23 13:52:35 -04:00
return result . references ( :categories ) . order ( << ~ SQL )
CASE WHEN categories . id = #{SiteSetting.uncategorized_category_id.to_i} THEN '' ELSE categories.name END #{sort_dir}
SQL
2016-02-25 11:22:23 -05:00
end
2013-11-13 14:17:06 -05:00
if sort_column == 'op_likes'
return result . includes ( :first_post ) . order ( " (SELECT like_count FROM posts p3 WHERE p3.topic_id = topics.id AND p3.post_number = 1) #{ sort_dir } " )
end
2014-06-17 21:23:31 -04:00
if sort_column . start_with? ( 'custom_fields' )
field = sort_column . split ( '.' ) [ 1 ]
return result . order ( " (SELECT CASE WHEN EXISTS (SELECT true FROM topic_custom_fields tcf WHERE tcf.topic_id::integer = topics.id::integer AND tcf.name = ' #{ field } ') THEN (SELECT value::integer FROM topic_custom_fields tcf WHERE tcf.topic_id::integer = topics.id::integer AND tcf.name = ' #{ field } ') ELSE 0 END) #{ sort_dir } " )
end
2013-07-16 15:20:18 -04:00
result . order ( " topics. #{ sort_column } #{ sort_dir } " )
end
2018-06-07 01:28:18 -04:00
2013-07-16 15:20:18 -04:00
def get_category_id ( category_id_or_slug )
2019-10-28 11:20:27 -04:00
return nil unless category_id_or_slug . present?
2013-07-16 15:20:18 -04:00
category_id = category_id_or_slug . to_i
2019-10-28 11:20:27 -04:00
if category_id == 0
category_id =
Category
. where ( slug : category_id_or_slug , parent_category_id : nil )
. pluck_first ( :id )
end
2014-06-17 21:23:31 -04:00
category_id
2018-06-07 01:28:18 -04:00
end
2013-02-05 14:16:51 -05:00
2015-03-23 18:12:37 -04:00
# Create results based on a bunch of default options
def default_results ( options = { } )
options . reverse_merge! ( @options )
2021-06-16 18:20:09 -04:00
options . reverse_merge! ( per_page : per_page_setting ) unless options [ :limit ] == false
2015-03-23 18:12:37 -04:00
2013-03-06 15:17:07 -05:00
# Whether to return visible topics
options [ :visible ] = true if @user . nil? || @user . regular?
2014-11-19 17:46:55 -05:00
options [ :visible ] = false if @user && @user . id == options [ :filtered_to_user ]
2013-03-06 15:17:07 -05:00
2013-07-16 15:20:18 -04:00
# Start with a list of all topics
2019-11-01 12:21:10 -04:00
result = Topic . unscoped . includes ( :category )
2013-03-06 15:17:07 -05:00
2018-06-07 01:28:18 -04:00
if @user
2014-02-21 14:17:45 -05:00
result = result . joins ( " LEFT OUTER JOIN topic_users AS tu ON (topics.id = tu.topic_id AND tu.user_id = #{ @user . id . to_i } ) " )
2014-02-26 11:09:02 -05:00
. references ( 'tu' )
2018-06-07 01:28:18 -04:00
end
2014-06-17 21:23:31 -04:00
category_id = get_category_id ( options [ :category ] )
2014-10-08 12:44:47 -04:00
@options [ :category_id ] = category_id
2014-06-17 21:23:31 -04:00
if category_id
if options [ :no_subcategories ]
result = result . where ( 'categories.id = ?' , category_id )
else
2020-10-07 14:19:48 -04:00
result = result . where ( " categories.id IN (?) " , Category . subcategory_ids ( category_id ) )
if ! SiteSetting . show_category_definitions_in_topic_lists
result = result . where ( " categories.topic_id <> topics.id OR categories.id = ? " , category_id )
end
2013-11-08 15:05:14 -05:00
end
2014-06-17 21:23:31 -04:00
result = result . references ( :categories )
2018-06-07 01:28:18 -04:00
2016-11-01 12:18:31 -04:00
if ! @options [ :order ]
# category default sort order
2019-10-21 06:32:27 -04:00
sort_order , sort_ascending = Category . where ( id : category_id ) . pluck_first ( :sort_order , :sort_ascending )
2016-11-01 12:18:31 -04:00
if sort_order
options [ :order ] = sort_order
options [ :ascending ] = ! ! sort_ascending ? 'true' : 'false'
2020-07-07 03:56:38 -04:00
else
options [ :order ] = 'default'
options [ :ascending ] = 'false'
2016-11-01 12:18:31 -04:00
end
2013-11-08 15:05:14 -05:00
end
2018-06-07 01:28:18 -04:00
end
2013-11-08 15:05:14 -05:00
2016-05-26 18:03:36 -04:00
if SiteSetting . tagging_enabled
result = result . preload ( :tags )
2019-12-04 13:33:51 -05:00
tags_arg = @options [ :tags ]
2019-04-29 20:25:53 -04:00
2019-12-04 13:33:51 -05:00
if tags_arg && tags_arg . size > 0
tags_arg = tags_arg . split if String === tags_arg
2019-04-29 20:25:53 -04:00
2019-12-04 13:33:51 -05:00
tags_arg = tags_arg . map do | t |
2019-04-29 20:25:53 -04:00
if String === t
t . downcase
else
t
end
end
2016-08-11 01:38:16 -04:00
2019-12-04 13:33:51 -05:00
tags_query = tags_arg [ 0 ] . is_a? ( String ) ? Tag . where_name ( tags_arg ) : Tag . where ( id : tags_arg )
tags = tags_query . select ( :id , :target_tag_id ) . map { | t | t . target_tag_id || t . id } . uniq
2016-08-11 01:38:16 -04:00
if @options [ :match_all_tags ]
2016-08-12 15:56:56 -04:00
# ALL of the given tags:
2019-12-04 13:33:51 -05:00
if tags_arg . length == tags . length
2019-04-29 20:25:53 -04:00
tags . each_with_index do | tag , index |
2016-08-15 15:30:17 -04:00
sql_alias = [ 't' , index ] . join
result = result . joins ( " INNER JOIN topic_tags #{ sql_alias } ON #{ sql_alias } .topic_id = topics.id AND #{ sql_alias } .tag_id = #{ tag } " )
2016-08-11 01:38:16 -04:00
end
2016-05-26 18:03:36 -04:00
else
2016-08-15 15:30:17 -04:00
result = result . none # don't return any results unless all tags exist in the database
2018-06-07 01:28:18 -04:00
end
else
2016-08-11 01:38:16 -04:00
# ANY of the given tags:
2019-12-04 13:33:51 -05:00
result = result . joins ( :tags ) . where ( " tags.id in (?) " , tags )
2016-05-04 14:02:47 -04:00
end
2019-04-29 20:25:53 -04:00
# TODO: this is very side-effecty and should be changed
# It is done cause further up we expect normalized tags
@options [ :tags ] = tags
2016-07-20 16:21:43 -04:00
elsif @options [ :no_tags ]
# the following will do: ("topics"."id" NOT IN (SELECT DISTINCT "topic_tags"."topic_id" FROM "topic_tags"))
2017-08-31 00:06:56 -04:00
result = result . where . not ( id : TopicTag . distinct . pluck ( :topic_id ) )
2016-05-04 14:02:47 -04:00
end
2018-06-07 01:28:18 -04:00
end
2016-05-04 14:02:47 -04:00
2013-11-14 15:50:36 -05:00
result = apply_ordering ( result , options )
2020-09-14 07:07:35 -04:00
all_listable_topics = @guardian . filter_allowed_categories ( Topic . unscoped . listable_topics )
if options [ :include_pms ]
all_pm_topics = Topic . unscoped . private_messages_for_user ( @user )
result = result . merge ( all_listable_topics . or ( all_pm_topics ) )
else
result = result . merge ( all_listable_topics )
end
2018-03-13 15:59:12 -04:00
2014-02-10 18:06:20 -05:00
# Don't include the category topics if excluded
if options [ :no_definitions ]
2014-02-04 15:55:30 -05:00
result = result . where ( 'COALESCE(categories.topic_id, 0) <> topics.id' )
end
2013-07-16 15:20:18 -04:00
result = result . limit ( options [ :per_page ] ) unless options [ :limit ] == false
2015-03-23 18:12:37 -04:00
result = result . visible if options [ :visible ]
2013-12-23 18:50:36 -05:00
result = result . where . not ( topics : { id : options [ :except_topic_ids ] } ) . references ( :topics ) if options [ :except_topic_ids ]
2016-05-09 16:33:55 -04:00
if options [ :page ]
offset = options [ :page ] . to_i * options [ :per_page ]
2016-05-11 13:39:21 -04:00
result = result . offset ( offset ) if offset > 0
2016-05-09 16:33:55 -04:00
end
2013-07-16 15:20:18 -04:00
if options [ :topic_ids ]
2013-08-16 08:53:40 -04:00
result = result . where ( 'topics.id in (?)' , options [ :topic_ids ] ) . references ( :topics )
2013-05-28 03:52:52 -04:00
end
2020-09-10 10:49:11 -04:00
if search = options [ :search ] . presence
2018-02-20 04:47:44 -05:00
result = result . where ( " topics.id in (select pp.topic_id from post_search_data pd join posts pp on pp.id = pd.post_id where pd.search_data @@ #{ Search . ts_query ( term : search . to_s ) } ) " )
2014-05-15 10:31:45 -04:00
end
2014-07-16 19:29:09 -04:00
# NOTE protect against SYM attack can be removed with Ruby 2.2
#
state = options [ :state ]
if @user && state &&
TopicUser . notification_levels . keys . map ( & :to_s ) . include? ( state )
level = TopicUser . notification_levels [ state . to_sym ]
result = result . where ( ' topics . id IN (
SELECT topic_id
FROM topic_users
WHERE user_id = ? AND
notification_level = ?) ' , @user . id , level )
end
2014-11-19 17:46:55 -05:00
require_deleted_clause = true
2017-03-02 14:54:26 -05:00
if before = options [ :before ]
if ( before = before . to_i ) > 0
result = result . where ( 'topics.created_at < ?' , before . to_i . days . ago )
end
2018-06-07 01:28:18 -04:00
end
2017-03-02 14:54:26 -05:00
2017-03-02 15:11:38 -05:00
if bumped_before = options [ :bumped_before ]
if ( bumped_before = bumped_before . to_i ) > 0
result = result . where ( 'topics.bumped_at < ?' , bumped_before . to_i . days . ago )
end
2018-06-07 01:28:18 -04:00
end
2017-03-02 15:11:38 -05:00
2014-01-12 22:40:21 -05:00
if status = options [ :status ]
case status
when 'open'
result = result . where ( 'NOT topics.closed AND NOT topics.archived' )
when 'closed'
result = result . where ( 'topics.closed' )
when 'archived'
result = result . where ( 'topics.archived' )
2015-01-12 04:00:45 -05:00
when 'listed'
2014-09-11 19:17:16 -04:00
result = result . where ( 'topics.visible' )
2015-01-12 04:00:45 -05:00
when 'unlisted'
2014-09-11 19:21:25 -04:00
result = result . where ( 'NOT topics.visible' )
2014-11-19 17:46:55 -05:00
when 'deleted'
2015-09-22 23:13:34 -04:00
guardian = @guardian
2014-11-19 17:46:55 -05:00
if guardian . is_staff?
result = result . where ( 'topics.deleted_at IS NOT NULL' )
require_deleted_clause = false
2014-01-12 22:40:21 -05:00
end
end
2018-06-07 01:28:18 -04:00
end
2014-01-12 22:40:21 -05:00
2020-08-06 02:33:45 -04:00
if ( filter = ( options [ :filter ] || options [ :f ] ) ) && @user
2015-01-07 02:20:10 -05:00
action =
if filter == " bookmarked "
PostActionType . types [ :bookmark ]
elsif filter == " liked "
PostActionType . types [ :like ]
end
if action
result = result . where ( ' topics . id IN ( SELECT pp . topic_id
2015-01-06 21:58:34 -05:00
FROM post_actions pa
JOIN posts pp ON pp . id = pa . post_id
WHERE pa . user_id = :user_id AND
2015-01-07 02:20:10 -05:00
pa . post_action_type_id = :action AND
2015-01-06 21:58:34 -05:00
pa . deleted_at IS NULL
) ' , user_id : @user . id ,
2015-01-07 02:20:10 -05:00
action : action
2018-06-07 01:28:18 -04:00
)
2015-01-06 21:58:34 -05:00
end
2020-07-22 20:30:08 -04:00
if filter == " tracked "
2020-09-25 15:39:37 -04:00
result = TopicQuery . tracked_filter ( result , @user . id )
2020-07-22 20:30:08 -04:00
end
2018-06-07 01:28:18 -04:00
end
2015-01-06 21:58:34 -05:00
2014-11-19 17:46:55 -05:00
result = result . where ( 'topics.deleted_at IS NULL' ) if require_deleted_clause
2014-06-05 09:30:24 -04:00
result = result . where ( 'topics.posts_count <= ?' , options [ :max_posts ] ) if options [ :max_posts ] . present?
result = result . where ( 'topics.posts_count >= ?' , options [ :min_posts ] ) if options [ :min_posts ] . present?
2017-02-15 15:25:43 -05:00
result = TopicQuery . apply_custom_filters ( result , self )
2020-09-14 07:07:35 -04:00
result
2013-02-05 14:16:51 -05:00
end
2015-11-01 17:20:22 -05:00
def remove_muted_topics ( list , user )
if user
2015-11-01 22:59:10 -05:00
list = list . where ( 'COALESCE(tu.notification_level,1) > :muted' , muted : TopicUser . notification_levels [ :muted ] )
2015-11-01 17:20:22 -05:00
end
list
2018-06-07 01:28:18 -04:00
end
2019-09-06 12:12:13 -04:00
2014-07-29 00:34:54 -04:00
def remove_muted_categories ( list , user , opts = nil )
2014-06-17 21:23:31 -04:00
category_id = get_category_id ( opts [ :exclude ] ) if opts
2018-06-07 01:28:18 -04:00
2019-11-18 21:34:24 -05:00
if user
2019-11-13 19:16:13 -05:00
list = list
. references ( " cu " )
. joins ( " LEFT JOIN category_users ON category_users.category_id = topics.category_id AND category_users.user_id = #{ user . id } " )
2019-11-18 21:34:24 -05:00
. where ( " topics.category_id = :category_id
OR COALESCE ( category_users . notification_level , :default ) < > :muted
OR tu . notification_level > :regular " ,
category_id : category_id || - 1 ,
2020-01-06 13:22:42 -05:00
default : CategoryUser . default_notification_level ,
2019-11-18 21:34:24 -05:00
muted : CategoryUser . notification_levels [ :muted ] ,
regular : TopicUser . notification_levels [ :regular ] )
elsif SiteSetting . mute_all_categories_by_default
category_ids = [
SiteSetting . default_categories_watching . split ( " | " ) ,
SiteSetting . default_categories_tracking . split ( " | " ) ,
2020-08-19 15:05:04 -04:00
SiteSetting . default_categories_watching_first_post . split ( " | " ) ,
SiteSetting . default_categories_regular . split ( " | " )
2019-11-18 21:34:24 -05:00
] . flatten . map ( & :to_i )
category_ids << category_id if category_id . present? && category_ids . exclude? ( category_id )
list = list . where ( " topics.category_id IN (?) " , category_ids ) if category_ids . present?
2019-11-18 21:18:16 -05:00
else
category_ids = SiteSetting . default_categories_muted . split ( " | " ) . map ( & :to_i )
category_ids -= [ category_id ] if category_id . present? && category_ids . include? ( category_id )
list = list . where ( " topics.category_id NOT IN (?) " , category_ids ) if category_ids . present?
2015-11-01 17:20:22 -05:00
end
2014-02-03 00:05:49 -05:00
2018-06-07 01:28:18 -04:00
list
end
2019-09-06 12:12:13 -04:00
2021-08-02 00:41:41 -04:00
def remove_muted_tags ( list , user , opts = { } )
2020-08-20 01:10:03 -04:00
if ! SiteSetting . tagging_enabled || SiteSetting . remove_muted_tags_from_latest == 'never'
2019-05-27 12:44:24 -04:00
return list
end
2020-08-20 01:10:03 -04:00
muted_tag_ids = [ ]
if user . present?
muted_tag_ids = TagUser . lookup ( user , :muted ) . pluck ( :tag_id )
else
2020-08-26 13:35:29 -04:00
muted_tag_names = SiteSetting . default_tags_muted . split ( " | " )
muted_tag_ids = Tag . where ( name : muted_tag_names ) . pluck ( :id )
2020-08-20 01:10:03 -04:00
end
2019-05-27 12:44:24 -04:00
if muted_tag_ids . blank?
return list
end
# if viewing the topic list for a muted tag, show all the topics
2019-09-06 12:12:13 -04:00
if ! opts [ :no_tags ] && opts [ :tags ] . present?
2020-07-20 06:01:29 -04:00
return list if TagUser . lookup ( user , :muted ) . joins ( :tag ) . where ( 'lower(tags.name) = ?' , opts [ :tags ] . first . downcase ) . exists?
2019-05-27 12:44:24 -04:00
end
2019-06-02 22:23:23 -04:00
if SiteSetting . remove_muted_tags_from_latest == 'always'
2019-05-27 12:44:24 -04:00
list = list . where ( "
NOT EXISTS (
SELECT 1
FROM topic_tags tt
WHERE tt . tag_id IN ( :tag_ids )
AND tt . topic_id = topics . id ) " , tag_ids: muted_tag_ids)
else
list = list . where ( "
EXISTS (
SELECT 1
FROM topic_tags tt
WHERE tt . tag_id NOT IN ( :tag_ids )
AND tt . topic_id = topics . id
) OR NOT EXISTS ( SELECT 1 FROM topic_tags tt WHERE tt . topic_id = topics . id ) " , tag_ids: muted_tag_ids)
2016-04-25 15:55:15 -04:00
end
2018-06-07 01:28:18 -04:00
end
2014-02-03 00:05:49 -05:00
2021-02-03 19:27:34 -05:00
def remove_dismissed ( list , user )
2019-11-13 19:16:13 -05:00
if user
2021-07-30 05:00:48 -04:00
list
. joins ( << ~ SQL )
LEFT JOIN dismissed_topic_users
ON dismissed_topic_users . topic_id = topics . id
AND dismissed_topic_users . user_id = #{user.id.to_i}
SQL
. where ( " dismissed_topic_users.id IS NULL " )
else
list
2019-11-13 19:16:13 -05:00
end
end
2016-02-03 02:50:05 -05:00
def new_messages ( params )
2020-09-10 02:32:11 -04:00
TopicQuery
DEV: Topic tracking state improvements (#13218)
I merged this PR in yesterday, finally thinking this was done https://github.com/discourse/discourse/pull/12958 but then a wild performance regression occurred. These are the problem methods:
https://github.com/discourse/discourse/blob/1aa20bd681e634f7fff22953ed62d90c2573b331/app/serializers/topic_tracking_state_serializer.rb#L13-L21
Turns out date comparison is super expensive on the backend _as well as_ the frontend.
The fix was to just move the `treat_as_new_topic_start_date` into the SQL query rather than using the slower `UserOption#treat_as_new_topic_start_date` method in ruby. After this change, 1% of the total time is spent with the `created_in_new_period` comparison instead of ~20%.
----
History:
Original PR which had to be reverted **https://github.com/discourse/discourse/pull/12555**. See the description there for what this PR is achieving, plus below.
The issue with the original PR is addressed in https://github.com/discourse/discourse/pull/12958/commits/92ef54f4020111ffacb0f2a27da5d5c2855f9d5d
If you went to the `x unread` link for a tag Chrome would freeze up and possibly crash, or eventually unfreeze after nearly 10 mins. Other routes for unread/new were similarly slow. From profiling the issue was the `sync` function of `topic-tracking-state.js`, which calls down to `isNew` which in turn calls `moment`, a change I had made in the PR above. The time it takes locally with ~1400 topics in the tracking state is 2.3 seconds.
To solve this issue, I have moved these calculations for "created in new period" and "unread not too old" into the tracking state serializer.
When I was looking at the profiler I also noticed this issue which was just compounding the problem. Every time we modify topic tracking state we recalculate the sidebar tracking/everything/tag counts. However this calls `forEachTracked` and `countTags` which can be quite expensive as they go through the whole tracking state (and were also calling the removed moment functions).
I added some logs and this was being called 30 times when navigating to a new /unread route because `sync` is being called from `build-topic-route` (one for each topic loaded due to pagination). So I just added a debounce here and it makes things even faster.
Finally, I changed topic tracking state to use a Map so our counts of the state keys is faster (Maps have .size whereas objects you have to do Object.keys(obj) which is O(n).)
<!-- NOTE: All pull requests should have tests (rspec in Ruby, qunit in JavaScript). If your code does not include test coverage, please include an explanation of why it was omitted. -->
2021-06-01 19:06:29 -04:00
. new_filter (
messages_for_groups_or_user ( params [ :my_group_ids ] ) ,
treat_as_new_topic_start_date : Time . at ( SiteSetting . min_new_topics_time ) . to_datetime
)
2016-02-03 02:50:05 -05:00
. limit ( params [ :count ] )
end
def unread_messages ( params )
2018-10-29 01:09:58 -04:00
query = TopicQuery . unread_filter (
2017-09-15 10:45:01 -04:00
messages_for_groups_or_user ( params [ :my_group_ids ] ) ,
2020-09-03 02:02:15 -04:00
staff : @user . staff?
)
first_unread_pm_at =
if params [ :my_group_ids ] . present?
GroupUser . where ( user_id : @user . id , group_id : params [ :my_group_ids ] ) . minimum ( :first_unread_pm_at )
else
2021-07-01 22:03:54 -04:00
UserStat . where ( user_id : @user . id ) . pluck_first ( :first_unread_pm_at )
2020-09-03 02:02:15 -04:00
end
query = query . where ( " topics.updated_at >= ? " , first_unread_pm_at ) if first_unread_pm_at
query = query . limit ( params [ :count ] ) if params [ :count ]
2018-10-29 01:09:58 -04:00
query
PERF: Avoid unnecessary expensive joins if possible.
```
EXPLAIN ANALYZE SELECT "topics".* FROM "topics"
LEFT JOIN topic_users tu ON topics.id = tu.topic_id AND tu.user_id =
13455
WHERE ("topics"."deleted_at" IS NULL)
AND (topics.archetype = 'private_message')
AND (
topics.id IN (
SELECT topic_id
FROM topic_allowed_groups tg
JOIN group_users gu ON gu.user_id = 13455 AND gu.group_id =
tg.group_id
WHERE gu.group_id IN (47)
)
)
AND (
topics.id IN (
SELECT ta.topic_id
FROM topic_allowed_users ta
WHERE ta.user_id IN (32852,-10)
)
OR
topics.id IN (
SELECT tg.topic_id
FROM topic_allowed_groups tg
WHERE tg.group_id IN (-10)
)
)
AND (topics.id NOT IN (69933,69995,69988,69984,69968,69973,69971,69952))
AND "topics"."visible" = 't'
ORDER BY topics.bumped_at DESC
LIMIT 3;
```
Planning time: 1.277 ms
Execution time: 71.577 ms
```
EXPLAIN ANALYZE SELECT "topics".* FROM "topics"
LEFT JOIN topic_users tu ON topics.id = tu.topic_id AND tu.user_id =
13455
LEFT JOIN (
SELECT * FROM topic_allowed_groups _tg
LEFT JOIN group_users gu
ON gu.user_id = 13455
AND gu.group_id = _tg.group_id
AND gu.group_id IN (47)
) tg ON topics.id = tg.topic_id
LEFT JOIN topic_allowed_users ta2 ON topics.id = ta2.topic_id AND
ta2.user_id IN (32852)
WHERE ("topics"."deleted_at" IS NULL)
AND (topics.archetype = 'private_message')
AND (tg.topic_id IS NOT NULL)
AND (ta2.topic_id IS NOT NULL)
AND (topics.id NOT IN (69933,69995,69988,69984,69968,69973,69971,69952))
AND "topics"."visible" = 't'
ORDER BY topics.bumped_at DESC
LIMIT 3;
```
Planning time: 1.191 ms
Execution time: 0.129 ms
2017-09-13 23:12:59 -04:00
end
2017-09-15 10:21:05 -04:00
def related_messages_user ( params )
2017-09-15 10:45:01 -04:00
messages = messages_for_user . limit ( params [ :count ] )
messages = allowed_messages ( messages , params )
end
def related_messages_group ( params )
2017-05-25 15:07:12 -04:00
messages = messages_for_groups_or_user ( params [ :my_group_ids ] ) . limit ( params [ :count ] )
2017-09-15 10:45:01 -04:00
messages = allowed_messages ( messages , params )
2018-06-07 01:28:18 -04:00
end
2017-09-15 10:45:01 -04:00
def allowed_messages ( messages , params )
user_ids = ( params [ :target_user_ids ] || [ ] )
group_ids = ( ( params [ :target_group_ids ] - params [ :my_group_ids ] ) || [ ] )
PERF: Avoid unnecessary expensive joins if possible.
```
EXPLAIN ANALYZE SELECT "topics".* FROM "topics"
LEFT JOIN topic_users tu ON topics.id = tu.topic_id AND tu.user_id =
13455
WHERE ("topics"."deleted_at" IS NULL)
AND (topics.archetype = 'private_message')
AND (
topics.id IN (
SELECT topic_id
FROM topic_allowed_groups tg
JOIN group_users gu ON gu.user_id = 13455 AND gu.group_id =
tg.group_id
WHERE gu.group_id IN (47)
)
)
AND (
topics.id IN (
SELECT ta.topic_id
FROM topic_allowed_users ta
WHERE ta.user_id IN (32852,-10)
)
OR
topics.id IN (
SELECT tg.topic_id
FROM topic_allowed_groups tg
WHERE tg.group_id IN (-10)
)
)
AND (topics.id NOT IN (69933,69995,69988,69984,69968,69973,69971,69952))
AND "topics"."visible" = 't'
ORDER BY topics.bumped_at DESC
LIMIT 3;
```
Planning time: 1.277 ms
Execution time: 71.577 ms
```
EXPLAIN ANALYZE SELECT "topics".* FROM "topics"
LEFT JOIN topic_users tu ON topics.id = tu.topic_id AND tu.user_id =
13455
LEFT JOIN (
SELECT * FROM topic_allowed_groups _tg
LEFT JOIN group_users gu
ON gu.user_id = 13455
AND gu.group_id = _tg.group_id
AND gu.group_id IN (47)
) tg ON topics.id = tg.topic_id
LEFT JOIN topic_allowed_users ta2 ON topics.id = ta2.topic_id AND
ta2.user_id IN (32852)
WHERE ("topics"."deleted_at" IS NULL)
AND (topics.archetype = 'private_message')
AND (tg.topic_id IS NOT NULL)
AND (ta2.topic_id IS NOT NULL)
AND (topics.id NOT IN (69933,69995,69988,69984,69968,69973,69971,69952))
AND "topics"."visible" = 't'
ORDER BY topics.bumped_at DESC
LIMIT 3;
```
Planning time: 1.191 ms
Execution time: 0.129 ms
2017-09-13 23:12:59 -04:00
2017-09-15 10:45:01 -04:00
if user_ids . present?
messages =
messages . joins ( "
LEFT JOIN topic_allowed_users ta2
ON topics . id = ta2 . topic_id
2020-12-10 18:56:26 -05:00
AND #{DB.sql_fragment('ta2.user_id IN (?)', user_ids)}
2017-09-15 10:45:01 -04:00
" )
end
if group_ids . present?
messages =
messages . joins ( "
LEFT JOIN topic_allowed_groups tg2
ON topics . id = tg2 . topic_id
2020-12-10 18:56:26 -05:00
AND #{DB.sql_fragment('tg2.group_id IN (?)', group_ids)}
2017-09-15 10:45:01 -04:00
" )
2016-02-03 02:50:05 -05:00
end
messages =
if user_ids . present? && group_ids . present?
messages . where ( " ta2.topic_id IS NOT NULL OR tg2.topic_id IS NOT NULL " )
elsif user_ids . present?
messages . where ( " ta2.topic_id IS NOT NULL " )
elsif group_ids . present?
2017-09-15 10:45:01 -04:00
messages . where ( " tg2.topic_id IS NOT NULL " )
2016-02-03 02:50:05 -05:00
end
end
def messages_for_groups_or_user ( group_ids )
2017-09-15 10:45:01 -04:00
if group_ids . present?
PERF: Avoid `NOT IN (<subquery>>` which can get really slow.
```
EXPLAIN ANALYZE SELECT "topics".*
FROM "topics" LEFT JOIN topic_users tu ON topics.id = tu.topic_id AND
tu.user_id = 13455
WHERE ("topics"."deleted_at" IS NULL)
AND (topics.archetype = 'private_message')
AND (
topics.id IN (
SELECT topic_id
FROM topic_allowed_users
WHERE user_id = 13455
)
)
AND (
topics.id IN (
SELECT ta.topic_id
FROM topic_allowed_users ta
WHERE ta.user_id IN (2,1995,8307,17621,22980,-10)
)
OR
topics.id IN (
SELECT tg.topic_id
FROM topic_allowed_groups tg
WHERE tg.group_id IN (-10)
)
)
AND (topics.id NOT IN (68559,60069,42145))
AND "topics"."visible" = 't'
ORDER BY topics.bumped_at
DESC LIMIT 5;
```
Planning time: 1.196 ms
Execution time: 21.176 ms
```
EXPLAIN ANALYZE SELECT "topics".*
FROM "topics"
LEFT JOIN topic_users tu ON topics.id = tu.topic_id AND tu.user_id =
13455
LEFT JOIN topic_allowed_users ta ON topics.id = ta.topic_id AND
ta.user_id = 13455
LEFT JOIN topic_allowed_users ta2 ON topics.id = ta2.topic_id AND
ta2.user_id IN (2,1995,8307,17621,22980,-10)
LEFT JOIN topic_allowed_groups tg ON topics.id = tg.topic_id AND
tg.group_id IN (-10)
WHERE ("topics"."deleted_at" IS NULL)
AND (topics.archetype = 'private_message')
AND (ta.topic_id IS NOT NULL)
AND (ta2.topic_id IS NOT NULL OR tg.topic_id IS NOT NULL)
AND (topics.id NOT IN (68559,60069,42145))
AND "topics"."visible" = 't'
ORDER BY topics.bumped_at DESC
LIMIT 5;
```
Planning time: 1.792 ms
Execution time: 2.546 ms
2017-09-13 10:22:33 -04:00
base_messages
. joins ( "
2017-09-15 10:45:01 -04:00
LEFT JOIN (
PERF: Avoid `NOT IN (<subquery>>` which can get really slow.
```
EXPLAIN ANALYZE SELECT "topics".*
FROM "topics" LEFT JOIN topic_users tu ON topics.id = tu.topic_id AND
tu.user_id = 13455
WHERE ("topics"."deleted_at" IS NULL)
AND (topics.archetype = 'private_message')
AND (
topics.id IN (
SELECT topic_id
FROM topic_allowed_users
WHERE user_id = 13455
)
)
AND (
topics.id IN (
SELECT ta.topic_id
FROM topic_allowed_users ta
WHERE ta.user_id IN (2,1995,8307,17621,22980,-10)
)
OR
topics.id IN (
SELECT tg.topic_id
FROM topic_allowed_groups tg
WHERE tg.group_id IN (-10)
)
)
AND (topics.id NOT IN (68559,60069,42145))
AND "topics"."visible" = 't'
ORDER BY topics.bumped_at
DESC LIMIT 5;
```
Planning time: 1.196 ms
Execution time: 21.176 ms
```
EXPLAIN ANALYZE SELECT "topics".*
FROM "topics"
LEFT JOIN topic_users tu ON topics.id = tu.topic_id AND tu.user_id =
13455
LEFT JOIN topic_allowed_users ta ON topics.id = ta.topic_id AND
ta.user_id = 13455
LEFT JOIN topic_allowed_users ta2 ON topics.id = ta2.topic_id AND
ta2.user_id IN (2,1995,8307,17621,22980,-10)
LEFT JOIN topic_allowed_groups tg ON topics.id = tg.topic_id AND
tg.group_id IN (-10)
WHERE ("topics"."deleted_at" IS NULL)
AND (topics.archetype = 'private_message')
AND (ta.topic_id IS NOT NULL)
AND (ta2.topic_id IS NOT NULL OR tg.topic_id IS NOT NULL)
AND (topics.id NOT IN (68559,60069,42145))
AND "topics"."visible" = 't'
ORDER BY topics.bumped_at DESC
LIMIT 5;
```
Planning time: 1.792 ms
Execution time: 2.546 ms
2017-09-13 10:22:33 -04:00
SELECT * FROM topic_allowed_groups _tg
LEFT JOIN group_users gu
ON gu . user_id = #{@user.id.to_i}
AND gu . group_id = _tg . group_id
2020-12-10 18:56:26 -05:00
WHERE #{DB.sql_fragment('gu.group_id IN (?)', group_ids)}
PERF: Avoid `NOT IN (<subquery>>` which can get really slow.
```
EXPLAIN ANALYZE SELECT "topics".*
FROM "topics" LEFT JOIN topic_users tu ON topics.id = tu.topic_id AND
tu.user_id = 13455
WHERE ("topics"."deleted_at" IS NULL)
AND (topics.archetype = 'private_message')
AND (
topics.id IN (
SELECT topic_id
FROM topic_allowed_users
WHERE user_id = 13455
)
)
AND (
topics.id IN (
SELECT ta.topic_id
FROM topic_allowed_users ta
WHERE ta.user_id IN (2,1995,8307,17621,22980,-10)
)
OR
topics.id IN (
SELECT tg.topic_id
FROM topic_allowed_groups tg
WHERE tg.group_id IN (-10)
)
)
AND (topics.id NOT IN (68559,60069,42145))
AND "topics"."visible" = 't'
ORDER BY topics.bumped_at
DESC LIMIT 5;
```
Planning time: 1.196 ms
Execution time: 21.176 ms
```
EXPLAIN ANALYZE SELECT "topics".*
FROM "topics"
LEFT JOIN topic_users tu ON topics.id = tu.topic_id AND tu.user_id =
13455
LEFT JOIN topic_allowed_users ta ON topics.id = ta.topic_id AND
ta.user_id = 13455
LEFT JOIN topic_allowed_users ta2 ON topics.id = ta2.topic_id AND
ta2.user_id IN (2,1995,8307,17621,22980,-10)
LEFT JOIN topic_allowed_groups tg ON topics.id = tg.topic_id AND
tg.group_id IN (-10)
WHERE ("topics"."deleted_at" IS NULL)
AND (topics.archetype = 'private_message')
AND (ta.topic_id IS NOT NULL)
AND (ta2.topic_id IS NOT NULL OR tg.topic_id IS NOT NULL)
AND (topics.id NOT IN (68559,60069,42145))
AND "topics"."visible" = 't'
ORDER BY topics.bumped_at DESC
LIMIT 5;
```
Planning time: 1.792 ms
Execution time: 2.546 ms
2017-09-13 10:22:33 -04:00
) tg ON topics . id = tg . topic_id
" )
. where ( " tg.topic_id IS NOT NULL " )
2018-06-07 01:28:18 -04:00
else
2016-02-03 02:50:05 -05:00
messages_for_user
end
2018-06-07 01:28:18 -04:00
end
2016-02-03 02:50:05 -05:00
def messages_for_user
base_messages
. joins ( "
LEFT JOIN topic_allowed_users ta
ON topics . id = ta . topic_id
AND ta . user_id = #{@user.id.to_i}
" )
2017-05-22 06:05:38 -04:00
. where ( " ta.topic_id IS NOT NULL " )
2016-02-03 02:50:05 -05:00
end
2014-02-03 00:05:49 -05:00
2014-01-28 18:15:36 -05:00
def base_messages
query = Topic
2014-02-04 12:26:38 -05:00
. where ( 'topics.archetype = ?' , Archetype . private_message )
. joins ( " LEFT JOIN topic_users tu ON topics.id = tu.topic_id AND tu.user_id = #{ @user . id . to_i } " )
2018-01-14 23:32:25 -05:00
query = query . includes ( :tags ) if SiteSetting . tagging_enabled
2014-01-28 18:15:36 -05:00
query . order ( 'topics.bumped_at DESC' )
end
2013-02-05 14:16:51 -05:00
2014-07-29 00:34:54 -04:00
def random_suggested ( topic , count , excluded_topic_ids = [ ] )
result = default_results ( unordered : true , per_page : count ) . where ( closed : false , archived : false )
2013-07-12 14:38:20 -04:00
if SiteSetting . limit_suggested_to_category
2013-07-18 14:47:59 -04:00
excluded_topic_ids += Category . where ( id : topic . category_id ) . pluck ( :id )
else
excluded_topic_ids += Category . topic_ids . to_a
end
result = result . where ( " topics.id NOT IN (?) " , excluded_topic_ids ) unless excluded_topic_ids . empty?
2013-02-27 18:30:14 -05:00
2018-01-14 23:32:25 -05:00
result = remove_muted_categories ( result , @user )
2020-11-24 07:16:10 -05:00
result = remove_muted_topics ( result , @user )
2015-02-25 01:19:12 -05:00
2015-03-02 18:20:42 -05:00
# If we are in a category, prefer it for the random results
2013-07-16 15:20:18 -04:00
if topic . category_id
2015-03-02 18:20:42 -05:00
result = result . order ( " CASE WHEN topics.category_id = #{ topic . category_id . to_i } THEN 0 ELSE 1 END " )
2013-02-05 14:16:51 -05:00
end
2013-08-08 13:18:52 -04:00
# Best effort, it over selects, however if you have a high number
# of muted categories there is tiny chance we will not select enough
# in particular this can happen if current category is empty and tons
# of muted, big edge case
2018-06-07 01:28:18 -04:00
#
2013-08-08 13:18:52 -04:00
# we over select in case cache is stale
max = ( count * 1 . 3 ) . to_i
ids = SiteSetting . limit_suggested_to_category ? [ ] : RandomTopicSelector . next ( max )
ids . concat ( RandomTopicSelector . next ( max , topic . category ) )
2018-06-07 01:28:18 -04:00
2013-08-08 13:18:52 -04:00
result . where ( id : ids . uniq )
end
2015-02-23 00:50:52 -05:00
def suggested_ordering ( result , options )
2013-11-14 15:50:36 -05:00
# Prefer unread in the same category
2013-08-08 13:18:52 -04:00
if options [ :topic ] && options [ :topic ] . category_id
2015-02-23 00:50:52 -05:00
result = result . order ( " CASE WHEN topics.category_id = #{ options [ :topic ] . category_id . to_i } THEN 0 ELSE 1 END " )
2013-08-08 13:18:52 -04:00
end
2017-09-15 10:45:01 -04:00
2018-03-22 16:38:53 -04:00
result . order ( 'topics.bumped_at DESC' )
2018-06-07 01:28:18 -04:00
end
2013-02-05 14:16:51 -05:00
end