From 8e00e036e12d8fbd42fb49a516a9bcd9267ce568 Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Silva Date: Thu, 21 Nov 2024 15:18:31 -0300 Subject: [PATCH] FEATURE: Make emotion /filter ordering match the dashboard table (#939) * FEATURE: Make emotion /filter ordering match the dashboard table This change makes the /filter endpoint use the same criteria we use in the dashboard table for emotion, so it is not confusing for users. It means that only posts made in the period with the emotion shall be shown in the /filter, and the order is simply a count of posts that match the emotion in the period. It also uses a trick to extract the filter period, and apply it to the CTE clause that calculates post emotion count on the period, making it a bit more efficient. Downside is that /filter filters are evaluated from left to right, so it will only get the speed-up if the emotion order is last. As we do this on the dashboard table, it should cover most uses of the ordering, kicking the need for materialized views down the road. * Remove zero score in filter * add table tooltip * lint --- .../templates/admin-dashboard-sentiment.hbs | 10 ++++++- config/locales/client.en.yml | 4 ++- config/locales/server.en.yml | 2 +- lib/sentiment/emotion_filter_order.rb | 28 +++++++++++++------ 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/assets/javascripts/discourse/templates/admin-dashboard-sentiment.hbs b/assets/javascripts/discourse/templates/admin-dashboard-sentiment.hbs index 90850b92..5d20c19f 100644 --- a/assets/javascripts/discourse/templates/admin-dashboard-sentiment.hbs +++ b/assets/javascripts/discourse/templates/admin-dashboard-sentiment.hbs @@ -27,8 +27,16 @@ diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index 3cc17f86..82cde4fd 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -12,7 +12,9 @@ en: categories: discourse_ai: "Discourse AI" dashboard: - emotion: "Emotion" + emotion: + title: "Emotion" + description: "The table lists a count of posts classified with a determined emotion. Classified with the model 'SamLowe/roberta-base-go_emotions'." js: discourse_automation: scriptables: diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index fdbd16c5..ef99602a 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -112,7 +112,7 @@ en: reports: overall_sentiment: title: "Overall sentiment" - description: 'The chart compares the number of posts classified as either positive or negative. These are calculated when positive or negative scores > the set threshold score. This means neutral posts are not shown. Private messages (PMs) are also excluded. Classified with "cardiffnlp/twitter-roberta-base-sentiment-latest"' + description: 'The chart compares the number of posts classified as either positive or negative. These are calculated when positive or negative scores > the set threshold score. This means neutral posts are not shown. Personal messages (PMs) are also excluded. Classified with "cardiffnlp/twitter-roberta-base-sentiment-latest"' xaxis: "Positive(%)" yaxis: "Date" emotion_admiration: diff --git a/lib/sentiment/emotion_filter_order.rb b/lib/sentiment/emotion_filter_order.rb index 56285608..34cad293 100644 --- a/lib/sentiment/emotion_filter_order.rb +++ b/lib/sentiment/emotion_filter_order.rb @@ -6,14 +6,25 @@ module DiscourseAi def self.register!(plugin) Emotions::LIST.each do |emotion| filter_order_emotion = ->(scope, order_direction) do + scope_period = + scope + .arel + &.constraints + &.flat_map(&:children) + &.find do |node| + node.is_a?(Arel::Nodes::Grouping) && + node.expr.to_s.match?(/topics\.bumped_at\s*>=/) + end + &.expr + &.split(">=") + &.last if scope.arel.constraints.present? && + scope.arel.constraints.any? { |c| c.is_a?(Arel::Nodes::Grouping) } + + # Fallback in case we can't find the scope period + scope_period ||= "CURRENT_DATE - INTERVAL '1 year'" + emotion_clause = <<~SQL - SUM( - CASE - WHEN (classification_results.classification::jsonb->'#{emotion}')::float > 0.1 - THEN 1 - ELSE 0 - END - )::float / COUNT(posts.id) + COUNT(*) FILTER (WHERE (classification_results.classification::jsonb->'#{emotion}')::float > 0.1) SQL # TODO: This is slow, we will need to materialize this in the future @@ -35,10 +46,11 @@ module DiscourseAi AND topics.deleted_at IS NULL AND posts.deleted_at IS NULL AND posts.post_type = 1 + AND posts.created_at >= #{scope_period} GROUP BY 1 HAVING - #{emotion_clause} > 0.05 + #{emotion_clause} > 0 SQL scope