From e9bb9a167b3727ac3b041999741075bac4ffa64a Mon Sep 17 00:00:00 2001
From: Claus Strasburger <claus@strasburger.de>
Date: Sun, 30 Apr 2017 23:30:40 +0200
Subject: [PATCH] Fix reply-by-mail for 8-bit transfer encodings

The mail class seems to handle mails sent with Content-Transfer-Encoding: 8bit
somewhat weirdly: It decodes them (to utf-8), changes the raw source to base64,
and does not modify the Content-Type:charset= header.

This leads to Discourse trying the message encoding (in my example ISO-8859-1)
first, and if that does not contain any unparseable characters, it uses that.
Sadly, in ISO-8859-1, every byte sequence is valid.

Fix this by always trying to decode as UTF-8 first. The probability of someone
using another encoding that cleanly (but wrongly) decodes as UTF-8 should be
fairly low.
---
 lib/email/receiver.rb                             |  7 +++++++
 spec/components/email/receiver_spec.rb            |  6 ++++++
 spec/fixtures/emails/reply_with_8bit_encoding.eml | 12 ++++++++++++
 3 files changed, 25 insertions(+)
 create mode 100644 spec/fixtures/emails/reply_with_8bit_encoding.eml

diff --git a/lib/email/receiver.rb b/lib/email/receiver.rb
index 58a3acba122..3d10c9c2780 100644
--- a/lib/email/receiver.rb
+++ b/lib/email/receiver.rb
@@ -217,6 +217,13 @@ module Email
       encodings = ["UTF-8", "ISO-8859-1"]
       encodings.unshift(mail_part.charset) if mail_part.charset.present?
 
+      # mail (>=2.5) decodes mails with 8bit transfer encoding to utf-8, so
+      # always try UTF-8 first
+      if mail_part.content_transfer_encoding == "8bit"
+        encodings.delete("UTF-8")
+        encodings.unshift("UTF-8")
+      end
+
       encodings.uniq.each do |encoding|
         fixed = try_to_encode(string, encoding)
         return fixed if fixed.present?
diff --git a/spec/components/email/receiver_spec.rb b/spec/components/email/receiver_spec.rb
index 7895e6f4e3e..c0735fbc35a 100644
--- a/spec/components/email/receiver_spec.rb
+++ b/spec/components/email/receiver_spec.rb
@@ -158,7 +158,9 @@ describe Email::Receiver do
 
       expect { process(:html_reply) }.to change { topic.posts.count }
       expect(topic.posts.last.raw).to eq("This is a **HTML** reply ;)")
+    end
 
+    it "handles different encodings correctly" do
       expect { process(:hebrew_reply) }.to change { topic.posts.count }
       expect(topic.posts.last.raw).to eq("שלום! מה שלומך היום?")
 
@@ -167,6 +169,10 @@ describe Email::Receiver do
 
       expect { process(:reply_with_weird_encoding) }.to change { topic.posts.count }
       expect(topic.posts.last.raw).to eq("This is a reply with a weird encoding.")
+
+      expect { process(:reply_with_8bit_encoding) }.to change { topic.posts.count }
+      expect(topic.posts.last.raw).to eq("hab vergessen kritische zeichen einzufügen:\näöüÄÖÜß")
+
     end
 
     it "prefers text over html" do
diff --git a/spec/fixtures/emails/reply_with_8bit_encoding.eml b/spec/fixtures/emails/reply_with_8bit_encoding.eml
new file mode 100644
index 00000000000..ed99f6752a8
--- /dev/null
+++ b/spec/fixtures/emails/reply_with_8bit_encoding.eml
@@ -0,0 +1,12 @@
+Return-Path: <discourse@bar.com>
+From: Foo Bar <discourse@bar.com>
+To: reply+4f97315cc828096c9cb34c6f1a0d6fe8@bar.com
+Date: Fri, 15 Jan 2016 00:12:43 +0100
+Message-ID: <43@foo.bar.mail>
+MIME-Version: 1.0
+Content-Type: text/plain; charset=iso-8859-1
+Content-Disposition: inline
+Content-Transfer-Encoding: 8bit
+
+hab vergessen kritische zeichen einzuf�gen:
+�������