Merge pull request #2703 from riking/email-pr-receiver

Heavy refactor for Email::Receiver

lib/email/html_cleaner.rb

@@ -0,0 +1,120 @@
+module Email
+  # HtmlCleaner cleans up the extremely dirty HTML that many email clients
+  # generate by stripping out any excess divs or spans, removing styling in
+  # the process (which also makes the html more suitable to be parsed as
+  # Markdown).
+  class HtmlCleaner
+    # Elements to hoist all children out of
+    HTML_HOIST_ELEMENTS = %w(div span font table tbody th tr td)
+    # Node types to always delete
+    HTML_DELETE_ELEMENT_TYPES = [Nokogiri::XML::Node::DTD_NODE,
+                                 Nokogiri::XML::Node::COMMENT_NODE,
+                                 ]
+
+    # Private variables:
+    #   @doc - nokogiri document
+    #   @out - same as @doc, but only if trimming has occured
+    def initialize(html)
+      if String === html
+        @doc = Nokogiri::HTML(html)
+      else
+        @doc = html
+      end
+    end
+
+    class << self
+      # Email::HtmlCleaner.trim(inp, opts={})
+      #
+      # Arguments:
+      #   inp - Either a HTML string or a Nokogiri document.
+      # Options:
+      #   :return => :doc, :string
+      #     Specify the desired return type.
+      #     Defaults to the type of the input.
+      #     A value of :string is equivalent to calling get_document_text()
+      #     on the returned document.
+      def trim(inp, opts={})
+        cleaner = HtmlCleaner.new(inp)
+
+        opts[:return] ||= ((String === inp) ? :string : :doc)
+
+        if opts[:return] == :string
+          cleaner.output_html
+        else
+          cleaner.output_document
+        end
+      end
+
+      # Email::HtmlCleaner.get_document_text(doc)
+      #
+      # Get the body portion of the document, including html, as a string.
+      def get_document_text(doc)
+        body = doc.xpath('//body')
+        if body
+          body.inner_html
+        else
+          doc.inner_html
+        end
+      end
+    end
+
+    def output_document
+      @out ||= begin
+                 doc = @doc
+                 trim_process_node doc
+                 add_newlines doc
+                 doc
+      end
+    end
+
+    def output_html
+      HtmlCleaner.get_document_text(output_document)
+    end
+
+    private
+
+    def add_newlines(doc)
+      doc.xpath('//br').each do |br|
+        br.replace(Nokogiri::XML::Text.new("\n", doc))
+      end
+    end
+
+    def trim_process_node(node)
+      if should_hoist?(node)
+        hoisted = trim_hoist_element node
+        hoisted.each { |child| trim_process_node child }
+      elsif should_delete?(node)
+        node.remove
+      else
+        if children = node.children
+          children.each { |child| trim_process_node child }
+        end
+      end
+
+      node
+    end
+
+    def trim_hoist_element(element)
+      hoisted = []
+      element.children.each do |child|
+        element.before(child)
+        hoisted << child
+      end
+      element.remove
+      hoisted
+    end
+
+    def should_hoist?(node)
+      return false unless node.element?
+      HTML_HOIST_ELEMENTS.include? node.name
+    end
+
+    def should_delete?(node)
+      return true if HTML_DELETE_ELEMENT_TYPES.include? node.type
+      return true if node.element? && node.name == 'head'
+      return true if node.text? && node.text.strip.blank?
+
+      false
+    end
+  end
+end

lib/email/receiver.rb

@@ -1,3 +1,4 @@
+require 'email/html_cleaner'
 #
 # Handles an incoming message
 #
@@ -26,20 +27,12 @@ module Email
     def process
       raise EmptyEmailError if @raw.blank?
 
-      @message = Mail.new(@raw)
+      message = Mail.new(@raw)
 
-      # First remove the known discourse stuff.
-      parse_body
-      raise EmptyEmailError if @body.blank?
-
-      # Then run the github EmailReplyParser on it in case we didn't catch it
-      @body = EmailReplyParser.read(@body).visible_text.force_encoding('UTF-8')
-
-      discourse_email_parser
-      raise EmailUnparsableError if @body.blank?
+      body = parse_body message
 
       dest_info = {type: :invalid, obj: nil}
-      @message.to.each do |to_address|
+      message.to.each do |to_address|
         if dest_info[:type] == :invalid
           dest_info = check_address to_address
         end
@@ -47,6 +40,10 @@ module Email
 
       raise BadDestinationAddress if dest_info[:type] == :invalid
 
+      # TODO get to a state where we can remove this
+      @message = message
+      @body = body
+
       if dest_info[:type] == :category
         raise BadDestinationAddress unless SiteSetting.email_in
         category = dest_info[:obj]
@@ -74,6 +71,8 @@ module Email
 
         create_reply
       end
+    rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
+      raise EmailUnparsableError.new(e)
     end
 
     def check_address(address)
@@ -94,57 +93,64 @@ module Email
       {type: :invalid, obj: nil}
     end
 
-    private
+    def parse_body(message)
+      body = select_body message
+      encoding = body.encoding
+      raise EmptyEmailError if body.strip.blank?
 
-    def parse_body
+      body = discourse_email_trimmer body
+      raise EmptyEmailError if body.strip.blank?
+
+      body = EmailReplyParser.parse_reply body
+      raise EmptyEmailError if body.strip.blank?
+
+      body.force_encoding(encoding).encode("UTF-8")
+    end
+
+    def select_body(message)
       html = nil
-
-      # If the message is multipart, find the best type for our purposes
-      if @message.multipart?
-        if p = @message.text_part
-          @body = p.charset ? p.body.decoded.force_encoding(p.charset).encode("UTF-8").to_s : p.body.to_s
-          return @body
-        elsif p = @message.html_part
-          html = p.charset ? p.body.decoded.force_encoding(p.charset).encode("UTF-8").to_s : p.body.to_s
+      # If the message is multipart, return that part (favor html)
+      if message.multipart?
+        html = fix_charset message.html_part
+        text = fix_charset message.text_part
+        # TODO picking text if available may be better
+        if text && !html
+          return text
         end
+      elsif message.content_type =~ /text\/html/
+        html = fix_charset message
       end
 
-      if @message.content_type =~ /text\/html/
-        if defined? @message.charset
-          html = @message.body.decoded.force_encoding(@message.charset).encode("UTF-8").to_s
-        else
-          html = @message.body.to_s
-        end
+      if html
+        body = HtmlCleaner.new(html).output_html
+      else
+        body = fix_charset message
       end
 
-      if html.present?
-        @body = scrub_html(html)
-        return @body
-      end
-
-      @body = @message.charset ? @message.body.decoded.force_encoding(@message.charset).encode("UTF-8").to_s.strip : @message.body.to_s
-
       # Certain trigger phrases that means we didn't parse correctly
-      @body = nil if @body =~ /Content\-Type\:/ ||
-                     @body =~ /multipart\/alternative/ ||
-                     @body =~ /text\/plain/
+      if body =~ /Content\-Type\:/ || body =~ /multipart\/alternative/ || body =~ /text\/plain/
+        raise EmptyEmailError
+      end
 
-      @body
+      body
     end
 
-    def scrub_html(html)
-      # If we have an HTML message, strip the markup
-      doc = Nokogiri::HTML(html)
+    # Force encoding to UTF-8 on a Mail::Message or Mail::Part
+    def fix_charset(object)
+      return nil if object.nil?
 
-      # Blackberry is annoying in that it only provides HTML. We can easily extract it though
-      content = doc.at("#BB10_response_div")
-      return content.text if content.present?
-
-      doc.xpath("//text()").text
+      if object.charset
+        object.body.decoded.force_encoding(object.charset).encode("UTF-8").to_s
+      else
+        object.body.to_s
+      end
     end
 
-    def discourse_email_parser
-      lines = @body.scrub.lines.to_a
+    REPLYING_HEADER_LABELS = ['From', 'Sent', 'To', 'Subject', 'Reply To']
+    REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |lbl| "#{lbl}:" })
+
+    def discourse_email_trimmer(body)
+      lines = body.scrub.lines.to_a
       range_end = 0
 
       lines.each_with_index do |l, idx|
@@ -155,11 +161,15 @@ module Email
                  # Let's try it and see how well it works.
                  (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/)
 
+        # Headers on subsequent lines
+        break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX }
+        # Headers on the same line
+        break if REPLYING_HEADER_LABELS.count { |lbl| l.include? lbl } >= 3
+
         range_end = idx
       end
 
-      @body = lines[0..range_end].join
-      @body.strip!
+      lines[0..range_end].join.strip
     end
 
     def wrap_body_in_quote(user_email)
@@ -168,37 +178,37 @@ module Email
 [/quote]"
     end
 
+    private
+
     def create_reply
-      create_post_with_attachments(email_log.user, @body, @email_log.topic_id, @email_log.post.post_number)
+      create_post_with_attachments(@email_log.user,
+                                   raw: @body,
+                                   topic_id: @email_log.topic_id,
+                                   reply_to_post_number: @email_log.post.post_number)
     end
 
     def create_new_topic
-      topic = TopicCreator.new(
-        @user,
-        Guardian.new(@user),
-        category: @category_id,
-        title: @message.subject,
-      ).create
-
-      post = create_post_with_attachments(@user, @body, topic.id)
+      post = create_post_with_attachments(@user,
+                                          raw: @body,
+                                          title: @message.subject,
+                                          category: @category_id)
 
       EmailLog.create(
         email_type: "topic_via_incoming_email",
-        to_address: @message.to.first,
-        topic_id: topic.id,
+        to_address: @message.from.first, # pick from address because we want the user's email
+        topic_id: post.topic.id,
         user_id: @user.id,
       )
 
       post
     end
 
-    def create_post_with_attachments(user, raw, topic_id, reply_to_post_number=nil)
+    def create_post_with_attachments(user, post_opts={})
       options = {
-        raw: raw,
-        topic_id: topic_id,
         cooking_options: { traditional_markdown_linebreaks: true },
-      }
-      options[:reply_to_post_number] = reply_to_post_number if reply_to_post_number
+      }.merge(post_opts)
+
+      raw = options[:raw]
 
       # deal with attachments
       @message.attachments.each do |attachment|
@@ -215,9 +225,10 @@ module Email
         ensure
           tmp.close!
         end
-
       end
 
+      options[:raw] = raw
+
       create_post(user, options)
     end
 
@@ -232,9 +243,11 @@ module Email
     def create_post(user, options)
       creator = PostCreator.new(user, options)
       post = creator.create
+
       if creator.errors.present?
         raise InvalidPost, creator.errors.full_messages.join("\n")
       end
+
       post
     end
 

spec/components/email/receiver_spec.rb

@@ -8,122 +8,233 @@ describe Email::Receiver do
   before do
     SiteSetting.reply_by_email_address = "reply+%{reply_key}@appmail.adventuretime.ooo"
     SiteSetting.email_in = false
+    SiteSetting.title = "Discourse"
   end
 
-  describe 'invalid emails' do
+  describe 'parse_body' do
+    def test_parse_body(mail_string)
+      Email::Receiver.new(nil).parse_body(Mail::Message.new mail_string)
+    end
+
     it "raises EmptyEmailError if the message is blank" do
-      expect { Email::Receiver.new("").process }.to raise_error(Email::Receiver::EmptyEmailError)
+      expect { test_parse_body("") }.to raise_error(Email::Receiver::EmptyEmailError)
     end
 
     it "raises EmptyEmailError if the message is not an email" do
-      expect { Email::Receiver.new("asdf" * 30).process}.to raise_error(Email::Receiver::EmptyEmailError)
+      expect { test_parse_body("asdf" * 30) }.to raise_error(Email::Receiver::EmptyEmailError)
     end
 
-    it "raises EmailUnparsableError if there is no reply content" do
-      expect { Email::Receiver.new(fixture_file("emails/no_content_reply.eml")).process}.to raise_error(Email::Receiver::EmailUnparsableError)
+    it "raises EmptyEmailError if there is no reply content" do
+      expect { test_parse_body(fixture_file("emails/no_content_reply.eml")) }.to raise_error(Email::Receiver::EmptyEmailError)
     end
-  end
 
-  describe "with multipart" do
-    let(:reply_below) { fixture_file("emails/multipart.eml") }
-    let(:receiver) { Email::Receiver.new(reply_below) }
-
-    it "processes correctly" do
-      expect { receiver.process}.to raise_error(Email::Receiver::EmailLogNotFound)
-      expect(receiver.body).to eq(
-"So presumably all the quoted garbage and my (proper) signature will get
-stripped from my reply?")
+    pending "raises EmailUnparsableError if the headers are corrupted" do
+      expect { ; }.to raise_error(Email::Receiver::EmailUnparsableError)
     end
-  end
 
-  describe "html only" do
-    let(:reply_below) { fixture_file("emails/html_only.eml") }
-    let(:receiver) { Email::Receiver.new(reply_below) }
-
-    it "processes correctly" do
-      expect { receiver.process}.to raise_error(Email::Receiver::EmailLogNotFound)
-      expect(receiver.body).to eq("The EC2 instance - I've seen that there tends to be odd and " +
-                                  "unrecommended settings on the Bitnami installs that I've checked out.")
+    it "can parse the html section" do
+      test_parse_body(fixture_file("emails/html_only.eml")).should == "The EC2 instance - I've seen that there tends to be odd and " +
+          "unrecommended settings on the Bitnami installs that I've checked out."
     end
-  end
 
-  describe "it supports a dutch reply" do
-    let(:dutch) { fixture_file("emails/dutch.eml") }
-    let(:receiver) { Email::Receiver.new(dutch) }
-
-    it "processes correctly" do
-      expect { receiver.process}.to raise_error(Email::Receiver::EmailLogNotFound)
-      expect(receiver.body).to eq("Dit is een antwoord in het Nederlands.")
+    it "supports a Dutch reply" do
+      test_parse_body(fixture_file("emails/dutch.eml")).should == "Dit is een antwoord in het Nederlands."
     end
-  end
 
-  describe "It supports a non english reply" do
-    let(:hebrew) { fixture_file("emails/hebrew.eml") }
-    let(:receiver) { Email::Receiver.new(hebrew) }
-
-    it "processes correctly" do
+    it "supports a Hebrew reply" do
       I18n.expects(:t).with('user_notifications.previous_discussion').returns('כלטוב')
-      expect { receiver.process}.to raise_error(Email::Receiver::EmailLogNotFound)
-      expect(receiver.body).to eq("שלום")
+
+      # The force_encoding call is only needed for the test - it is passed on fine to the cooked post
+      test_parse_body(fixture_file("emails/hebrew.eml")).should == "שלום"
     end
-  end
 
-  describe "It supports a non UTF-8 reply" do
-    let(:big5) { fixture_file("emails/big5.eml") }
-    let(:receiver) { Email::Receiver.new(big5) }
-
-    it "processes correctly" do
+    it "supports a BIG5-encoded reply" do
       I18n.expects(:t).with('user_notifications.previous_discussion').returns('媽！我上電視了！')
-      expect { receiver.process}.to raise_error(Email::Receiver::EmailLogNotFound)
-      expect(receiver.body).to eq("媽！我上電視了！")
+
+      # The force_encoding call is only needed for the test - it is passed on fine to the cooked post
+      test_parse_body(fixture_file("emails/big5.eml")).should == "媽！我上電視了！"
     end
-  end
 
-  describe "via" do
-    let(:wrote) { fixture_file("emails/via_line.eml") }
-    let(:receiver) { Email::Receiver.new(wrote) }
+    it "removes 'via' lines if they match the site title" do
+      SiteSetting.title = "Discourse"
 
-    it "removes via lines if we know them" do
-      expect { receiver.process}.to raise_error(Email::Receiver::EmailLogNotFound)
-      expect(receiver.body).to eq("Hello this email has content!")
+      test_parse_body(fixture_file("emails/via_line.eml")).should == "Hello this email has content!"
     end
-  end
 
-  describe "if wrote is on a second line" do
-    let(:wrote) { fixture_file("emails/multiline_wrote.eml") }
-    let(:receiver) { Email::Receiver.new(wrote) }
-
-    it "processes correctly" do
-      expect { receiver.process}.to raise_error(Email::Receiver::EmailLogNotFound)
-      expect(receiver.body).to eq("Thanks!")
+    it "removes the 'Previous Discussion' marker" do
+      test_parse_body(fixture_file("emails/previous.eml")).should == "This will not include the previous discussion that is present in this email."
     end
-  end
 
-  describe "remove previous discussion" do
-    let(:previous) { fixture_file("emails/previous.eml") }
-    let(:receiver) { Email::Receiver.new(previous) }
-
-    it "processes correctly" do
-      expect { receiver.process}.to raise_error(Email::Receiver::EmailLogNotFound)
-      expect(receiver.body).to eq("This will not include the previous discussion that is present in this email.")
-    end
-  end
-
-  describe "multiple paragraphs" do
-    let(:paragraphs) { fixture_file("emails/paragraphs.eml") }
-    let(:receiver) { Email::Receiver.new(paragraphs) }
-
-    it "processes correctly" do
-      expect { receiver.process}.to raise_error(Email::Receiver::EmailLogNotFound)
-      expect(receiver.body).to eq(
+    it "handles multiple paragraphs" do
+      test_parse_body(fixture_file("emails/paragraphs.eml")).
+          should == (
 "Is there any reason the *old* candy can't be be kept in silos while the new candy
 is imported into *new* silos?
 
 The thing about candy is it stays delicious for a long time -- we can just keep
 it there without worrying about it too much, imo.
 
-Thanks for listening.")
+Thanks for listening."
+      )
     end
+
+    it "converts back to UTF-8 at the end" do
+      result = test_parse_body(fixture_file("emails/big5.eml"))
+      result.encoding.should == Encoding::UTF_8
+
+      # should not throw
+      TextCleaner.normalize_whitespaces(
+          test_parse_body(fixture_file("emails/big5.eml"))
+      )
+    end
+  end
+
+  describe "posting replies" do
+    let(:reply_key) { raise "Override this in a lower describe block" }
+    let(:email_raw) { raise "Override this in a lower describe block" }
+    # ----
+    let(:receiver) { Email::Receiver.new(email_raw) }
+    let(:post) { create_post }
+    let(:topic) { post.topic }
+    let(:posting_user) { post.user }
+    let(:replying_user_email) { 'jake@adventuretime.ooo' }
+    let(:replying_user) { Fabricate(:user, email: replying_user_email, trust_level: 2)}
+    let(:email_log) { EmailLog.new(reply_key: reply_key,
+                                   post: post,
+                                   post_id: post.id,
+                                   topic_id: post.topic_id,
+                                   email_type: 'user_posted',
+                                   user: replying_user,
+                                   user_id: replying_user.id,
+                                   to_address: replying_user_email
+    ) }
+
+    before do
+      email_log.save
+    end
+
+    # === Success Posting ===
+
+    describe "valid_reply.eml" do
+      let!(:reply_key) { '59d8df8370b7e95c5a49fbf86aeb2c93' }
+      let!(:email_raw) { fixture_file("emails/valid_reply.eml") }
+
+      it "creates a post with the correct content" do
+        start_count = topic.posts.count
+
+        receiver.process
+
+        topic.posts.count.should == (start_count + 1)
+        topic.posts.last.cooked.strip.should == fixture_file("emails/valid_reply.cooked").strip
+      end
+    end
+
+    describe "paragraphs.eml" do
+      let!(:reply_key) { '59d8df8370b7e95c5a49fbf86aeb2c93' }
+      let!(:email_raw) { fixture_file("emails/paragraphs.eml") }
+
+      it "cooks multiple paragraphs with traditional Markdown linebreaks" do
+        start_count = topic.posts.count
+
+        receiver.process
+
+        topic.posts.count.should == (start_count + 1)
+        topic.posts.last.cooked.strip.should == fixture_file("emails/paragraphs.cooked").strip
+        topic.posts.last.cooked.should_not match /<br/
+      end
+    end
+
+    describe "attachment.eml" do
+      let!(:reply_key) { '636ca428858779856c226bb145ef4fad' }
+      let!(:email_raw) {
+        fixture_file("emails/attachment.eml")
+        .gsub("TO", "reply+#{reply_key}@appmail.adventuretime.ooo")
+        .gsub("FROM", replying_user_email)
+      }
+
+      let(:upload_sha) { '04df605be528d03876685c52166d4b063aabb78a' }
+
+      it "creates a post with an attachment" do
+        start_count = topic.posts.count
+        Upload.find_by(sha1: upload_sha).try :destroy
+
+        receiver.process
+
+        topic.posts.count.should == (start_count + 1)
+        topic.posts.last.cooked.should match /<img src=['"](\/uploads\/default\/\d+\/\w{16}\.png)['"] width=['"]289['"] height=['"]126['"]>/
+        Upload.find_by(sha1: upload_sha).should_not be_nil
+      end
+
+    end
+
+    # === Failure Conditions ===
+
+    describe "too_short.eml" do
+      let!(:reply_key) { '636ca428858779856c226bb145ef4fad' }
+      let!(:email_raw) {
+        fixture_file("emails/too_short.eml")
+        .gsub("TO", "reply+#{reply_key}@appmail.adventuretime.ooo")
+        .gsub("FROM", replying_user_email)
+        .gsub("SUBJECT", "re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'")
+      }
+
+      it "raises an InvalidPost error" do
+        SiteSetting.min_post_length = 5
+        expect { receiver.process }.to raise_error(Email::Receiver::InvalidPost)
+      end
+    end
+
+    describe "too_many_mentions.eml" do
+      let!(:reply_key) { '636ca428858779856c226bb145ef4fad' }
+      let!(:email_raw) { fixture_file("emails/too_many_mentions.eml") }
+
+      it "raises an InvalidPost error" do
+        SiteSetting.max_mentions_per_post = 10
+        (1..11).each do |i|
+          Fabricate(:user, username: "user#{i}").save
+        end
+
+        expect { receiver.process }.to raise_error(Email::Receiver::InvalidPost)
+      end
+    end
+
+  end
+
+  describe "posting a new topic" do
+    let(:category_destination) { raise "Override this in a lower describe block" }
+    let(:email_raw) { raise "Override this in a lower describe block" }
+    let(:allow_strangers) { false }
+    # ----
+    let(:receiver) { Email::Receiver.new(email_raw) }
+    let(:user_email) { 'jake@adventuretime.ooo' }
+    let(:user) { Fabricate(:user, email: user_email, trust_level: 2)}
+    let(:category) { Fabricate(:category, email_in: category_destination, email_in_allow_strangers: allow_strangers) }
+
+    before do
+      SiteSetting.email_in = true
+      user.save
+      category.save
+    end
+
+    describe "too_short.eml" do
+      let!(:category_destination) { 'incoming+amazing@appmail.adventuretime.ooo' }
+      let(:email_raw) {
+        fixture_file("emails/too_short.eml")
+        .gsub("TO", category_destination)
+        .gsub("FROM", user_email)
+        .gsub("SUBJECT", "A long subject that passes the checks")
+      }
+
+      it "does not create a topic if the post fails" do
+        before_topic_count = Topic.count
+
+        expect { receiver.process }.to raise_error(Email::Receiver::InvalidPost)
+
+        Topic.count.should == before_topic_count
+      end
+
+    end
+
   end
 
   def fill_email(mail, from, to, body = nil, subject = nil)
@@ -181,12 +292,6 @@ greatest show ever created. Everyone should watch it.
 
         expect(receiver.body).to eq(reply_body)
         expect(receiver.email_log).to eq(email_log)
-
-        attachment_email = fixture_file("emails/attachment.eml")
-        attachment_email = fill_email(attachment_email, "test@test.com", to)
-        r = Email::Receiver.new(attachment_email)
-        expect { r.process }.to_not raise_error
-        expect(r.body).to match(/here is an image attachment\n<img src='\/uploads\/default\/\d+\/\w{16}\.png' width='289' height='126'>\n/)
       end
 
     end

spec/fixtures/emails/boundary.eml

@@ -18,7 +18,7 @@ Content-Type: text/plain; charset=ISO-8859-1
 I'll look into it, thanks!
 
 
-On Wednesday, June 19, 2013, jake via Adventure Time wrote:
+On Wednesday, June 19, 2013, jake via Discourse wrote:
 
 > jake mentioned you in 'peppermint butler is missing' on Adventure
 > Time:
@@ -58,4 +58,4 @@ p>
 ime.ooo/user_preferences" target=3D"_blank">user preferences</a>.</p>
 </blockquote>
 
---001a11c206a073876a04df81d2a9--
+--001a11c206a073876a04df81d2a9--

spec/fixtures/emails/multiline_wrote.eml

@@ -1,23 +0,0 @@
-
-Delivered-To: discourse-reply+cd480e301683c9902891f15968bf07a5@discourse.org
-Received: by 10.194.216.104 with SMTP id op8csp80593wjc;
-        Wed, 24 Jul 2013 07:59:14 -0700 (PDT)
-Return-Path: <walter.white@googlemail.com>
-References: <topic/5043@discourse.org> <51efeb9b36c34_66dc2dfce6811866@discourse.mail>
-From: Walter White <walter.white@googlemail.com>
-In-Reply-To: <51efeb9b36c34_66dc2dfce6811866@discourse.mail>
-Mime-Version: 1.0 (1.0)
-Date: Wed, 24 Jul 2013 15:59:10 +0100
-Message-ID: <4597127794206131679@unknownmsgid>
-Subject: Re: [Discourse] new reply to your post in 'Crystal Blue'
-To: walter via Discourse <reply+cd480e301683c9902891f15968bf07a5@appmail.adventuretime.ooo>
-Content-Type: multipart/alternative; boundary=001a11c20edc15a39304e2432790
-
-Thanks!
-
-On 24 Jul 2013, at 15:58, walter via Discourse <info@discourse.org>
-wrote:
-
-   walter <http://discourse.org/users/walter>  July 24
-
-You look great today Walter.

spec/fixtures/emails/multipart.eml

@@ -1,67 +0,0 @@
-Message-ID: <51C22E52.1030509@darthvader.ca>
-Date: Wed, 19 Jun 2013 18:18:58 -0400
-From: Anakin Skywalker <evildad@darthvader.ca>
-User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20130510 Thunderbird/17.0.6
-MIME-Version: 1.0
-To: Han Solo via Death Star <reply+603775f8f5f68006461890a3eadf94cf@appmail.adventuretime.ooo>
-Subject: Re: [Death Star] [PM] re: Regarding your post in "Site Customization
- not working"
-References: <51d23d33f41fb_5f4e4b35d7d60798@xwing.mail>
-In-Reply-To: <51d23d33f41fb_5f4e4b35d7d60798@xwing.mail>
-Content-Type: multipart/alternative;
- boundary="------------070503080300090900010604"
-
-This is a multi-part message in MIME format.
---------------070503080300090900010604
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 7bit
-
-So presumably all the quoted garbage and my (proper) signature will get
-stripped from my reply?
-
---
-Anakin Skywalker               | `One of the main causes of the fall of
-evildad@darthvader.ca          | the Roman Empire was that, lacking zero,
-                               | they had no way to indicate successful
-                               | termination of their C programs.' - Firth
-
-
---------------070503080300090900010604
-Content-Type: text/html; charset=UTF-8
-Content-Transfer-Encoding: 7bit
-
-<html>
-  <head>
-    <meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
-  </head>
-  <body bgcolor="#FFFFFF" text="#000000">
-    <div class="moz-cite-prefix">On 13-06-19 06:14 PM, Han Solo via
-      Death Star wrote:<br>
-    </div>
-    <blockquote
-      cite="mid:51d23d33f41fb_5f4e4b35d7d60798@xwing.mail"
-      type="cite">
-      <p>Han Solo just sent you a private message</p>
-      <hr>
-      <p>I got it here! Yay it worked!</p>
-      <hr>
-      <p>Please visit this link to respond: <a moz-do-not-send="true"
-href="http://darthvader.ca/t/regarding-your-post-in-site-customization-not-working/7641/2">http://darthvader.ca/t/regarding-your-post-in-site-customization-not-working/7641/2</a></p>
-      <p>To unsubscribe from these emails, visit your <a
-          moz-do-not-send="true"
-          href="http://darthvader.ca/user_preferences">user
-          preferences</a>.</p>
-    </blockquote>
-    So presumably all the quoted garbage and my (proper) signature will
-    get stripped from my reply?<br>
-    <br>
-    <pre class="moz-signature" cols="72">--
-Anakin Skywalker               | `One of the main causes of the fall of
-<a class="moz-txt-link-abbreviated" href="mailto:evildad@darthvader.ca">evildad@darthvader.ca</a>       | the Roman Empire was that, lacking zero,
-                            | they had no way to indicate successful
-                            | termination of their C programs.' - Firth
-</pre>
-  </body>
-</html>
-
---------------070503080300090900010604--

spec/fixtures/emails/paragraphs.cooked

@@ -0,0 +1,7 @@
+<p>Is there any reason the <em>old</em> candy can't be be kept in silos while the new candy
+is imported into <em>new</em> silos?</p>
+
+<p>The thing about candy is it stays delicious for a long time -- we can just keep
+it there without worrying about it too much, imo.</p>
+
+<p>Thanks for listening.</p>

spec/fixtures/emails/too_many_mentions.eml

@@ -0,0 +1,31 @@
+Return-Path: <jake@adventuretime.ooo>
+Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400
+Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@discourse.example.com>; Thu, 13 Jun 2013 17:03:50 -0400
+Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@discourse.example.com>; Thu, 13 Jun 2013 14:03:48 -0700
+Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700
+Date: Thu, 13 Jun 2013 17:03:48 -0400
+From: Jake the Dog <jake@adventuretime.ooo>
+To: reply+636ca428858779856c226bb145ef4fad@appmail.adventuretime.ooo
+Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
+Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'
+Mime-Version: 1.0
+Content-Type: text/plain;
+ charset=ISO-8859-1
+Content-Transfer-Encoding: 7bit
+X-Sieve: CMU Sieve 2.2
+X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu,
+ 13 Jun 2013 14:03:48 -0700 (PDT)
+X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1
+
+
+@user1
+@user2
+@user3
+@user4
+@user5
+@user6
+@user7
+@user8
+@user9
+@user10
+@user11

spec/fixtures/emails/too_short.eml

@@ -0,0 +1,21 @@
+Return-Path: <jake@adventuretime.ooo>
+Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400
+Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@discourse.example.com>; Thu, 13 Jun 2013 17:03:50 -0400
+Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@discourse.example.com>; Thu, 13 Jun 2013 14:03:48 -0700
+Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700
+Date: Thu, 13 Jun 2013 17:03:48 -0400
+From: Jake the Dog <FROM>
+To: TO
+Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com>
+Subject: SUBJECT
+Mime-Version: 1.0
+Content-Type: text/plain;
+ charset=ISO-8859-1
+Content-Transfer-Encoding: 7bit
+X-Sieve: CMU Sieve 2.2
+X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu,
+ 13 Jun 2013 14:03:48 -0700 (PDT)
+X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1
+
+
++1

spec/jobs/poll_mailbox_spec.rb

@@ -202,9 +202,9 @@ describe Jobs::PollMailbox do
         email.should be_deleted
       end
 
-      it "a no content reply raises an EmailUnparsableError" do
+      it "a no content reply raises an EmptyEmailError" do
         email = MockPop3EmailObject.new fixture_file('emails/no_content_reply.eml')
-        expect_exception Email::Receiver::EmailUnparsableError
+        expect_exception Email::Receiver::EmptyEmailError
 
         poller.handle_mail(email)
         email.should be_deleted

spec/spec_helper.rb

@@ -30,6 +30,7 @@ Spork.prefork do
   # Requires supporting ruby files with custom matchers and macros, etc,
   # in spec/support/ and its subdirectories.
   Dir[Rails.root.join("spec/support/**/*.rb")].each {|f| require f}
+  Dir[Rails.root.join("spec/fabricators/*.rb")].each {|f| require f}
 
   # let's not run seed_fu every test
   SeedFu.quiet = true if SeedFu.respond_to? :quiet