FIX: Truncate content for sentiment/toxicity classification (#431)

lib/sentiment/sentiment_classification.rb

@@ -48,11 +48,14 @@ module DiscourseAi
       end
 
       def content_of(target_to_classify)
-        if target_to_classify.post_number == 1
-          "#{target_to_classify.topic.title}\n#{target_to_classify.raw}"
-        else
-          target_to_classify.raw
-        end
+        content =
+          if target_to_classify.post_number == 1
+            "#{target_to_classify.topic.title}\n#{target_to_classify.raw}"
+          else
+            target_to_classify.raw
+          end
+
+        Tokenizer::BertTokenizer.truncate(content, 512)
       end
 
       def endpoint

lib/toxicity/toxicity_classification.rb

@@ -59,13 +59,18 @@ module DiscourseAi
       end
 
       def content_of(target_to_classify)
-        return target_to_classify.message if target_to_classify.is_a?(Chat::Message)
+        content =
+          if target_to_classify.is_a?(Chat::Message)
+            target_to_classify.message
+          else
+            if target_to_classify.post_number == 1
+              "#{target_to_classify.topic.title}\n#{target_to_classify.raw}"
+            else
+              target_to_classify.raw
+            end
+          end
 
-        if target_to_classify.post_number == 1
-          "#{target_to_classify.topic.title}\n#{target_to_classify.raw}"
-        else
-          target_to_classify.raw
-        end
+        Tokenizer::BertTokenizer.truncate(content, 512)
       end
 
       def endpoint