FIX: Truncate content for sentiment/toxicity classification (#431)

lib/sentiment/sentiment_classification.rb

@@ -48,11 +48,14 @@ module DiscourseAi
       end
 
       def content_of(target_to_classify)
+        content =
           if target_to_classify.post_number == 1
             "#{target_to_classify.topic.title}\n#{target_to_classify.raw}"
           else
             target_to_classify.raw
           end
+
+        Tokenizer::BertTokenizer.truncate(content, 512)
       end
 
       def endpoint

lib/toxicity/toxicity_classification.rb

@@ -59,8 +59,10 @@ module DiscourseAi
       end
 
       def content_of(target_to_classify)
-        return target_to_classify.message if target_to_classify.is_a?(Chat::Message)
-
+        content =
+          if target_to_classify.is_a?(Chat::Message)
+            target_to_classify.message
+          else
             if target_to_classify.post_number == 1
               "#{target_to_classify.topic.title}\n#{target_to_classify.raw}"
             else
@@ -68,6 +70,9 @@ module DiscourseAi
             end
           end
 
+        Tokenizer::BertTokenizer.truncate(content, 512)
+      end
+
       def endpoint
         if SiteSetting.ai_toxicity_inference_service_api_endpoint_srv.present?
           service =