Discource-C
/
discourse-ai
mirror of https://github.com/discourse/discourse-ai.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FIX: Truncate content for sentiment/toxicity classification (#431)

This commit is contained in:
Rafael dos Santos Silva 2024-01-17 15:17:58 -03:00 committed by GitHub
parent 5bdf3dc1f4
commit c70f43f130
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 19 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
lib/sentiment/sentiment_classification.rb
View File

@ -48,11 +48,14 @@ module DiscourseAi
end end
def content_of(target_to_classify) def content_of(target_to_classify)
if target_to_classify.post_number == 1 content =
"#{target_to_classify.topic.title}\n#{target_to_classify.raw}" if target_to_classify.post_number == 1
else "#{target_to_classify.topic.title}\n#{target_to_classify.raw}"
target_to_classify.raw else
end target_to_classify.raw
end
Tokenizer::BertTokenizer.truncate(content, 512)
end end
def endpoint def endpoint

17
lib/toxicity/toxicity_classification.rb
View File

@ -59,13 +59,18 @@ module DiscourseAi
end end
def content_of(target_to_classify) def content_of(target_to_classify)
return target_to_classify.message if target_to_classify.is_a?(Chat::Message) content =
if target_to_classify.is_a?(Chat::Message)
target_to_classify.message
else
if target_to_classify.post_number == 1
"#{target_to_classify.topic.title}\n#{target_to_classify.raw}"
else
target_to_classify.raw
end
end
if target_to_classify.post_number == 1 Tokenizer::BertTokenizer.truncate(content, 512)
"#{target_to_classify.topic.title}\n#{target_to_classify.raw}"
else
target_to_classify.raw
end
end end
def endpoint def endpoint