FIX: More resilient sentiment backfill query (#998)
This commit is contained in:
parent
e7c2cd861a
commit
938d4c018c
|
@ -5,11 +5,10 @@ module DiscourseAi
|
|||
class PostClassification
|
||||
def self.backfill_query(from_post_id: nil, max_age_days: nil)
|
||||
available_classifier_names =
|
||||
DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema
|
||||
.values
|
||||
.map { |mc| mc.model_name.downcase }
|
||||
.sort
|
||||
DiscourseAi::Sentiment::SentimentSiteSettingJsonSchema.values.map { _1.model_name }
|
||||
|
||||
queries =
|
||||
available_classifier_names.map do |classifier_name|
|
||||
base_query =
|
||||
Post
|
||||
.includes(:sentiment_classifications)
|
||||
|
@ -23,16 +22,12 @@ module DiscourseAi
|
|||
ON crs.target_id = posts.id
|
||||
AND crs.target_type = 'Post'
|
||||
AND crs.classification_type = 'sentiment'
|
||||
AND crs.model_used = '#{classifier_name}'
|
||||
SQL
|
||||
.group("posts.id")
|
||||
.having(<<~SQL, available_classifier_names)
|
||||
COUNT(crs.model_used) = 0
|
||||
OR array_agg(
|
||||
DISTINCT LOWER(crs.model_used) ORDER BY LOWER(crs.model_used)
|
||||
)::text[] IS DISTINCT FROM array[?]
|
||||
SQL
|
||||
.where("crs.id IS NULL")
|
||||
|
||||
base_query = base_query.where("posts.id >= ?", from_post_id.to_i) if from_post_id.present?
|
||||
base_query =
|
||||
base_query.where("posts.id >= ?", from_post_id.to_i) if from_post_id.present?
|
||||
|
||||
if max_age_days.present?
|
||||
base_query =
|
||||
|
@ -44,6 +39,11 @@ module DiscourseAi
|
|||
base_query
|
||||
end
|
||||
|
||||
unioned_queries = queries.map(&:to_sql).join(" UNION ")
|
||||
|
||||
Post.from(Arel.sql("(#{unioned_queries}) as posts"))
|
||||
end
|
||||
|
||||
def bulk_classify!(relation)
|
||||
http_pool_size = 100
|
||||
pool =
|
||||
|
|
Loading…
Reference in New Issue