FIX: remove full quote on direct replies when "typographed"

Use the cooked version of the post and the quote to compare their content in
order to take into account the "typographer" option of the markdown pipeline.

lib/cooked_post_processor.rb

@@ -37,7 +37,7 @@ class CookedPostProcessor
   def post_process(bypass_bump: false, new_post: false)
     DistributedMutex.synchronize("post_process_#{@post.id}") do
       DiscourseEvent.trigger(:before_post_process_cooked, @doc, @post)
-      removed_direct_reply_full_quotes if new_post
+      remove_full_quote_on_direct_reply if new_post
       post_process_oneboxes
       post_process_images
       post_process_quotes
@@ -92,22 +92,34 @@ class CookedPostProcessor
     end
   end
 
-  def removed_direct_reply_full_quotes
-    return if !SiteSetting.remove_full_quote || @post.post_number == 1
+  def remove_full_quote_on_direct_reply
+    return if !SiteSetting.remove_full_quote
+    return if @post.post_number == 1
+    return if @doc.css("aside.quote").size != 1
 
-    num_quotes = @doc.css("aside.quote").size
-    return if num_quotes != 1
+    previous = Post
+      .where("post_number < ? AND topic_id = ? AND post_type = ? AND NOT hidden", @post.post_number, @post.topic_id, Post.types[:regular])
+      .order("post_number DESC")
+      .limit(1)
+      .pluck(:cooked)
+      .first
 
-    prev = Post.where('post_number < ? AND topic_id = ? AND post_type = ? AND not hidden', @post.post_number, @post.topic_id, Post.types[:regular]).order('post_number desc').limit(1).pluck(:raw).first
-    return if !prev
+    return if previous.blank?
 
-    new_raw = @post.raw.gsub(/\A\s*\[quote[^\]]*\]\s*#{Regexp.quote(prev.strip)}\s*\[\/quote\]/, '')
-    return if @post.raw == new_raw
+    previous_text = Nokogiri::HTML::fragment(previous).text.strip
+    quoted_text = @doc.css("aside.quote:first-child blockquote").first&.text&.strip
+
+    return if previous_text != quoted_text
+
+    quote_regexp = /\A\s*\[quote.+?\[\/quote\]/im
+    quoteless_raw = @post.raw.sub(quote_regexp, "").strip
+
+    return if @post.raw.strip == quoteless_raw
 
     PostRevisor.new(@post).revise!(
       Discourse.system_user,
       {
-        raw: new_raw.strip,
+        raw: quoteless_raw,
         edit_reason: I18n.t(:removed_direct_reply_full_quotes)
       },
       skip_validations: true,

spec/components/cooked_post_processor_spec.rb

@@ -1307,12 +1307,14 @@ describe CookedPostProcessor do
 
   context "remove direct reply full quote" do
     fab!(:topic) { Fabricate(:topic) }
-    let!(:post) { Fabricate(:post, topic: topic, raw: "this is the first post") }
+    let!(:post) { Fabricate(:post, topic: topic, raw: 'this is the "first" post') }
 
     let(:raw) do
       <<~RAW.strip
       [quote="#{post.user.username}, post:#{post.post_number}, topic:#{topic.id}"]
-      this is the first post
+
+      this is the “first” post
+
       [/quote]
 
       and this is the third reply
@@ -1324,7 +1326,7 @@ describe CookedPostProcessor do
       and this is the third reply
 
       [quote="#{post.user.username}, post:#{post.post_number}, topic:#{topic.id}"]
-      this is the first post
+      this is the ”first” post
       [/quote]
       RAW
     end
@@ -1340,7 +1342,7 @@ describe CookedPostProcessor do
 
       freeze_time Time.zone.now do
         topic.bumped_at = 1.day.ago
-        CookedPostProcessor.new(reply).removed_direct_reply_full_quotes
+        CookedPostProcessor.new(reply).remove_full_quote_on_direct_reply
 
         expect(topic.ordered_posts.pluck(:id))
           .to eq([post.id, hidden.id, small_action.id, reply.id])
@@ -1355,7 +1357,7 @@ describe CookedPostProcessor do
 
     it 'does not delete quote if not first paragraph' do
       reply = Fabricate(:post, topic: topic, raw: raw2)
-      CookedPostProcessor.new(reply).removed_direct_reply_full_quotes
+      CookedPostProcessor.new(reply).remove_full_quote_on_direct_reply
       expect(topic.ordered_posts.pluck(:id)).to eq([post.id, reply.id])
       expect(reply.raw).to eq(raw2)
     end
@@ -1365,7 +1367,7 @@ describe CookedPostProcessor do
 
       reply = Fabricate(:post, topic: topic, raw: raw)
 
-      CookedPostProcessor.new(reply).removed_direct_reply_full_quotes
+      CookedPostProcessor.new(reply).remove_full_quote_on_direct_reply
       expect(reply.raw).to eq(raw)
     end