FEATURE: hide posts from incoming email based on dmarc verdict (#8333)

app/models/post.rb

@@ -133,7 +133,8 @@ class Post < ActiveRecord::Base
                                  new_user_spam_threshold_reached: 3,
                                  flagged_by_tl3_user: 4,
                                  email_spam_header_found: 5,
-                                 flagged_by_tl4_user: 6)
+                                 flagged_by_tl4_user: 6,
+                                 email_authentication_result_header: 7)
   end
 
   def self.types

config/locales/server.en.yml

@@ -1900,6 +1900,7 @@ en:
     log_mail_processing_failures: "Log all email processing failures to <a href='%{base_path}/logs' target='_blank'>/logs</a>"
     email_in: 'Allow users to post new topics via email (requires manual or pop3 polling). Configure the addresses in the "Settings" tab of each category.'
     email_in_min_trust: "The minimum trust level a user needs to have to be allowed to post new topics via email."
+    email_in_authserv_id: "The identifier of the service doing authentication checks on incoming emails. See <a href='https://meta.discourse.org/t/134358'>https://meta.discourse.org/t/134358</a> for instructions on how to configure this."
     email_in_spam_header: "The email header to detect spam."
     email_prefix: "The [label] used in the subject of emails. It will default to 'title' if not set."
     email_site_title: "The title of the site used as the sender of emails from the site. Default to 'title' if not set. If your 'title' contains characters that are not allowed in email sender strings, use this setting."

config/site_settings.yml

@@ -965,6 +965,8 @@ email:
   email_in_min_trust:
     default: 2
     enum: "TrustLevelSetting"
+  email_in_authserv_id:
+    default: ""
   email_in_spam_header:
     type: enum
     default: "none"

lib/email/authentication_results.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Email
+  class AuthenticationResults
+    attr_reader :results
+
+    VERDICT = Enum.new(
+      :gray,
+      :pass,
+      :fail,
+      start: 0,
+    )
+
+    def initialize(headers)
+      authserv_id = SiteSetting.email_in_authserv_id
+      @results = Array(headers).map do |header|
+        parse_header(header.to_s)
+      end.filter do |result|
+        authserv_id.blank? || authserv_id == result[:authserv_id]
+      end
+    end
+
+    def action
+      @action ||= calc_action
+    end
+
+    def verdict
+      @verdict ||= calc_verdict
+    end
+
+    private
+
+    def calc_action
+      if verdict == :fail
+        :hide
+      else
+        :accept
+      end
+    end
+
+    def calc_verdict
+      VERDICT[calc_dmarc]
+    end
+
+    def calc_dmarc
+      verdict = VERDICT[:gray]
+      @results.each do |result|
+        result[:resinfo].each do |resinfo|
+          if resinfo[:method] == "dmarc"
+            v = VERDICT[resinfo[:result].to_sym].to_i
+            verdict = v if v > verdict
+          end
+        end
+      end
+      verdict = VERDICT[:gray] if SiteSetting.email_in_authserv_id.blank? && verdict == VERDICT[:pass]
+      verdict
+    end
+
+    def parse_header(header)
+      # based on https://tools.ietf.org/html/rfc8601#section-2.2
+      cfws = /\s*(\([^()]*\))?\s*/
+      value = /(?:"([^"]*)")|(?:([^\s";]*))/
+      authserv_id = value
+      authres_version = /\d+#{cfws}?/
+      no_result = /#{cfws}?;#{cfws}?none/
+      keyword = /([a-zA-Z0-9-]*[a-zA-Z0-9])/
+      authres_payload = /\A#{cfws}?#{authserv_id}(?:#{cfws}#{authres_version})?(?:#{no_result}|([\S\s]*))/
+
+      method_version = authres_version
+      method = /#{keyword}\s*(?:#{cfws}?\/#{cfws}?#{method_version})?/
+      result = keyword
+      methodspec = /#{cfws}?#{method}#{cfws}?=#{cfws}?#{result}/
+      reasonspec = /reason#{cfws}?=#{cfws}?#{value}/
+      resinfo = /#{cfws}?;#{methodspec}(?:#{cfws}#{reasonspec})?(?:#{cfws}([^;]*))?/
+
+      ptype = keyword
+      property = value
+      pvalue = /#{cfws}?#{value}#{cfws}?/
+      propspec = /#{ptype}#{cfws}?\.#{cfws}?#{property}#{cfws}?=#{pvalue}/
+
+      authres_payload_match = authres_payload.match(header)
+      parsed_authserv_id = authres_payload_match[2] || authres_payload_match[3]
+      resinfo_val = authres_payload_match[-1]
+
+      if resinfo_val
+        resinfo_scan = resinfo_val.scan(resinfo)
+        parsed_resinfo = resinfo_scan.map do |x|
+          {
+            method: x[2],
+            result: x[8],
+            reason: x[12] || x[13],
+            props: x[-1].scan(propspec).map do |y|
+              {
+                ptype: y[0],
+                property: y[4],
+                pvalue: y[8] || y[9]
+              }
+            end
+          }
+        end
+      end
+
+      {
+        authserv_id: parsed_authserv_id,
+        resinfo: parsed_resinfo
+      }
+    end
+
+  end
+end

lib/email/receiver.rb

@@ -196,7 +196,14 @@ module Email
     end
 
     def hidden_reason_id
-      @hidden_reason_id ||= is_spam? ? Post.hidden_reasons[:email_spam_header_found] : nil
+      @hidden_reason_id ||=
+        if is_spam?
+          Post.hidden_reasons[:email_spam_header_found]
+        elsif auth_res_action == :hide
+          Post.hidden_reasons[:email_authentication_result_header]
+        else
+          nil
+        end
     end
 
     def log_and_validate_user(user)
@@ -308,6 +315,10 @@ module Email
       end
     end
 
+    def auth_res_action
+      @auth_res_action ||= AuthenticationResults.new(@mail.header[:authentication_results]).action
+    end
+
     def select_body
       text = nil
       html = nil

spec/components/email/authentication_results_spec.rb

@@ -0,0 +1,299 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+require "email/authentication_results"
+
+describe Email::AuthenticationResults do
+  describe "#results" do
+    it "parses 'Nearly Trivial Case: Service Provided, but No Authentication Done' correctly" do
+      # https://tools.ietf.org/html/rfc8601#appendix-B.2
+      results = described_class.new(" example.org 1; none").results
+      expect(results[0][:authserv_id]).to eq "example.org"
+      expect(results[0][:resinfo]).to be nil
+    end
+
+    it "parses 'Service Provided, Authentication Done' correctly" do
+      # https://tools.ietf.org/html/rfc8601#appendix-B.3
+      results = described_class.new(<<~EOF
+        example.com;
+                 spf=pass smtp.mailfrom=example.net
+      EOF
+      ).results
+      expect(results[0][:authserv_id]).to eq "example.com"
+      expect(results[0][:resinfo][0][:method]).to eq "spf"
+      expect(results[0][:resinfo][0][:result]).to eq "pass"
+      expect(results[0][:resinfo][0][:reason]).to be nil
+      expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "smtp"
+      expect(results[0][:resinfo][0][:props][0][:property]).to eq "mailfrom"
+      expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "example.net"
+    end
+
+    it "parses 'Service Provided, Several Authentications Done, Single MTA' correctly" do
+      # https://tools.ietf.org/html/rfc8601#appendix-B.4
+      results = described_class.new([<<~EOF ,
+        example.com;
+                  auth=pass (cram-md5) smtp.auth=sender@example.net;
+                  spf=pass smtp.mailfrom=example.net
+      EOF
+      <<~EOF ,
+        example.com; iprev=pass
+                  policy.iprev=192.0.2.200
+      EOF
+      ]).results
+      expect(results[0][:authserv_id]).to eq "example.com"
+      expect(results[0][:resinfo][0][:method]).to eq "auth"
+      expect(results[0][:resinfo][0][:result]).to eq "pass"
+      expect(results[0][:resinfo][0][:reason]).to be nil
+      expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "smtp"
+      expect(results[0][:resinfo][0][:props][0][:property]).to eq "auth"
+      expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "sender@example.net"
+      expect(results[0][:resinfo][1][:method]).to eq "spf"
+      expect(results[0][:resinfo][1][:result]).to eq "pass"
+      expect(results[0][:resinfo][1][:reason]).to be nil
+      expect(results[0][:resinfo][1][:props][0][:ptype]).to eq "smtp"
+      expect(results[0][:resinfo][1][:props][0][:property]).to eq "mailfrom"
+      expect(results[0][:resinfo][1][:props][0][:pvalue]).to eq "example.net"
+      expect(results[1][:authserv_id]).to eq "example.com"
+      expect(results[1][:resinfo][0][:method]).to eq "iprev"
+      expect(results[1][:resinfo][0][:result]).to eq "pass"
+      expect(results[1][:resinfo][0][:reason]).to be nil
+      expect(results[1][:resinfo][0][:props][0][:ptype]).to eq "policy"
+      expect(results[1][:resinfo][0][:props][0][:property]).to eq "iprev"
+      expect(results[1][:resinfo][0][:props][0][:pvalue]).to eq "192.0.2.200"
+    end
+
+    it "parses 'Service Provided, Several Authentications Done, Different MTAs' correctly" do
+      # https://tools.ietf.org/html/rfc8601#appendix-B.5
+      results = described_class.new([<<~EOF ,
+        example.com;
+                 dkim=pass (good signature) header.d=example.com
+      EOF
+      <<~EOF ,
+        example.com;
+                  auth=pass (cram-md5) smtp.auth=sender@example.com;
+                  spf=fail smtp.mailfrom=example.com
+      EOF
+      ]).results
+
+      expect(results[0][:authserv_id]).to eq "example.com"
+      expect(results[0][:resinfo][0][:method]).to eq "dkim"
+      expect(results[0][:resinfo][0][:result]).to eq "pass"
+      expect(results[0][:resinfo][0][:reason]).to be nil
+      expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "header"
+      expect(results[0][:resinfo][0][:props][0][:property]).to eq "d"
+      expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "example.com"
+      expect(results[1][:authserv_id]).to eq "example.com"
+      expect(results[1][:resinfo][0][:method]).to eq "auth"
+      expect(results[1][:resinfo][0][:result]).to eq "pass"
+      expect(results[1][:resinfo][0][:reason]).to be nil
+      expect(results[1][:resinfo][0][:props][0][:ptype]).to eq "smtp"
+      expect(results[1][:resinfo][0][:props][0][:property]).to eq "auth"
+      expect(results[1][:resinfo][0][:props][0][:pvalue]).to eq "sender@example.com"
+      expect(results[1][:resinfo][1][:method]).to eq "spf"
+      expect(results[1][:resinfo][1][:result]).to eq "fail"
+      expect(results[1][:resinfo][1][:reason]).to be nil
+      expect(results[1][:resinfo][1][:props][0][:ptype]).to eq "smtp"
+      expect(results[1][:resinfo][1][:props][0][:property]).to eq "mailfrom"
+      expect(results[1][:resinfo][1][:props][0][:pvalue]).to eq "example.com"
+    end
+
+    it "parses 'Service Provided, Multi-tiered Authentication Done' correctly" do
+      # https://tools.ietf.org/html/rfc8601#appendix-B.6
+      results = described_class.new([<<~EOF ,
+         example.com;
+              dkim=pass reason="good signature"
+                header.i=@mail-router.example.net;
+              dkim=fail reason="bad signature"
+                header.i=@newyork.example.com
+      EOF
+      <<~EOF ,
+        example.net;
+             dkim=pass (good signature) header.i=@newyork.example.com
+      EOF
+      ]).results
+
+      expect(results[0][:authserv_id]).to eq "example.com"
+      expect(results[0][:resinfo][0][:method]).to eq "dkim"
+      expect(results[0][:resinfo][0][:result]).to eq "pass"
+      expect(results[0][:resinfo][0][:reason]).to eq "good signature"
+      expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "header"
+      expect(results[0][:resinfo][0][:props][0][:property]).to eq "i"
+      expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "@mail-router.example.net"
+      expect(results[0][:resinfo][1][:method]).to eq "dkim"
+      expect(results[0][:resinfo][1][:result]).to eq "fail"
+      expect(results[0][:resinfo][1][:reason]).to eq "bad signature"
+      expect(results[0][:resinfo][1][:props][0][:ptype]).to eq "header"
+      expect(results[0][:resinfo][1][:props][0][:property]).to eq "i"
+      expect(results[0][:resinfo][1][:props][0][:pvalue]).to eq "@newyork.example.com"
+      expect(results[1][:authserv_id]).to eq "example.net"
+      expect(results[1][:resinfo][0][:method]).to eq "dkim"
+      expect(results[1][:resinfo][0][:result]).to eq "pass"
+      expect(results[1][:resinfo][0][:reason]).to be nil
+      expect(results[1][:resinfo][0][:props][0][:ptype]).to eq "header"
+      expect(results[1][:resinfo][0][:props][0][:property]).to eq "i"
+      expect(results[1][:resinfo][0][:props][0][:pvalue]).to eq "@newyork.example.com"
+    end
+
+    it "parses 'Comment-Heavy Example' correctly" do
+      # https://tools.ietf.org/html/rfc8601#appendix-B.7
+      results = described_class.new(<<~EOF
+        foo.example.net (foobar) 1 (baz);
+          dkim (Because I like it) / 1 (One yay) = (wait for it) fail
+            policy (A dot can go here) . (like that) expired
+            (this surprised me) = (as I wasn't expecting it) 1362471462
+      EOF
+      ).results
+
+      expect(results[0][:authserv_id]).to eq "foo.example.net"
+      expect(results[0][:resinfo][0][:method]).to eq "dkim"
+      expect(results[0][:resinfo][0][:result]).to eq "fail"
+      expect(results[0][:resinfo][0][:reason]).to be nil
+      expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "policy"
+      expect(results[0][:resinfo][0][:props][0][:property]).to eq "expired"
+      expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "1362471462"
+    end
+
+    it "parses header with no props correctly" do
+      results = described_class.new(" example.com; dmarc=pass").results
+      expect(results[0][:authserv_id]).to eq "example.com"
+      expect(results[0][:resinfo][0][:method]).to eq "dmarc"
+      expect(results[0][:resinfo][0][:result]).to eq "pass"
+      expect(results[0][:resinfo][0][:reason]).to be nil
+      expect(results[0][:resinfo][0][:props]).to eq []
+    end
+
+    it "parses header with multiple props correctly" do
+      results = described_class.new(<<~EOF
+        mx.google.com;
+      dkim=pass header.i=@email.example.com header.s=20111006 header.b=URn9MW+F;
+      spf=pass (google.com: domain of foo@b.email.example.com designates 1.2.3.4 as permitted sender) smtp.mailfrom=foo@b.email.example.com;
+      dmarc=pass (p=REJECT sp=REJECT dis=NONE) header.from=email.example.com
+      EOF
+      ).results
+
+      expect(results[0][:authserv_id]).to eq "mx.google.com"
+      expect(results[0][:resinfo][0][:method]).to eq "dkim"
+      expect(results[0][:resinfo][0][:result]).to eq "pass"
+      expect(results[0][:resinfo][0][:reason]).to be nil
+      expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "header"
+      expect(results[0][:resinfo][0][:props][0][:property]).to eq "i"
+      expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "@email.example.com"
+      expect(results[0][:resinfo][0][:props][1][:ptype]).to eq "header"
+      expect(results[0][:resinfo][0][:props][1][:property]).to eq "s"
+      expect(results[0][:resinfo][0][:props][1][:pvalue]).to eq "20111006"
+      expect(results[0][:resinfo][0][:props][2][:ptype]).to eq "header"
+      expect(results[0][:resinfo][0][:props][2][:property]).to eq "b"
+      expect(results[0][:resinfo][0][:props][2][:pvalue]).to eq "URn9MW+F"
+      expect(results[0][:resinfo][1][:method]).to eq "spf"
+      expect(results[0][:resinfo][1][:result]).to eq "pass"
+      expect(results[0][:resinfo][1][:reason]).to be nil
+      expect(results[0][:resinfo][1][:props][0][:ptype]).to eq "smtp"
+      expect(results[0][:resinfo][1][:props][0][:property]).to eq "mailfrom"
+      expect(results[0][:resinfo][1][:props][0][:pvalue]).to eq "foo@b.email.example.com"
+      expect(results[0][:resinfo][2][:method]).to eq "dmarc"
+      expect(results[0][:resinfo][2][:result]).to eq "pass"
+      expect(results[0][:resinfo][2][:reason]).to be nil
+      expect(results[0][:resinfo][2][:props][0][:ptype]).to eq "header"
+      expect(results[0][:resinfo][2][:props][0][:property]).to eq "from"
+      expect(results[0][:resinfo][2][:props][0][:pvalue]).to eq "email.example.com"
+    end
+  end
+
+  describe "#verdict" do
+    before do
+      SiteSetting.email_in_authserv_id = "valid.com"
+    end
+
+    shared_examples "is verdict" do |verdict|
+      it "is #{verdict}" do
+        expect(described_class.new(headers).verdict).to eq verdict
+      end
+    end
+
+    context "with no authentication-results headers" do
+      let(:headers) { "" }
+
+      it "is gray" do
+        expect(described_class.new(headers).verdict).to eq :gray
+      end
+    end
+
+    context "with a single authentication-results header" do
+      context "with a valid fail" do
+        let(:headers) { "valid.com; dmarc=fail" }
+        include_examples "is verdict", :fail
+      end
+
+      context "with a valid pass" do
+        let(:headers) { "valid.com; dmarc=pass" }
+        include_examples "is verdict", :pass
+      end
+
+      context "with a valid error" do
+        let(:headers) { "valid.com; dmarc=error" }
+        include_examples "is verdict", :gray
+      end
+
+      context "with no email_in_authserv_id set" do
+        before { SiteSetting.email_in_authserv_id = "" }
+
+        context "with a fail" do
+          let(:headers) { "foobar.com; dmarc=fail" }
+          include_examples "is verdict", :fail
+        end
+
+        context "with a pass" do
+          let(:headers) { "foobar.com; dmarc=pass" }
+          include_examples "is verdict", :gray
+        end
+      end
+    end
+
+    context "with multiple authentication-results headers" do
+      context "with a valid fail, and an invalid pass" do
+        let(:headers) { ["valid.com; dmarc=fail", "invalid.com; dmarc=pass"] }
+        include_examples "is verdict", :fail
+      end
+
+      context "with a valid fail, and a valid pass" do
+        let(:headers) { ["valid.com; dmarc=fail", "valid.com; dmarc=pass"] }
+        include_examples "is verdict", :fail
+      end
+
+      context "with a valid error, and a valid pass" do
+        let(:headers) { ["valid.com; dmarc=foobar", "valid.com; dmarc=pass"] }
+        include_examples "is verdict", :pass
+      end
+
+      context "with no email_in_authserv_id set" do
+        before { SiteSetting.email_in_authserv_id = "" }
+
+        context "with an error, and a pass" do
+          let(:headers) { ["foobar.com; dmarc=foobar", "foobar.com; dmarc=pass"] }
+          include_examples "is verdict", :gray
+        end
+      end
+    end
+  end
+
+  describe "#action" do
+    it "hides a fail verdict" do
+      results = described_class.new("")
+      results.expects(:verdict).returns(:fail)
+      expect(results.action).to eq (:hide)
+    end
+
+    it "accepts a pass verdict" do
+      results = described_class.new("")
+      results.expects(:verdict).returns(:pass)
+      expect(results.action).to eq (:accept)
+    end
+
+    it "accepts a gray verdict" do
+      results = described_class.new("")
+      results.expects(:verdict).returns(:gray)
+      expect(results.action).to eq (:accept)
+    end
+  end
+
+end

spec/components/email/receiver_spec.rb

@@ -1021,6 +1021,19 @@ describe Email::Receiver do
       expect(post.hidden_reason_id).to eq(Post.hidden_reasons[:email_spam_header_found])
     end
 
+    it "creates hidden topic for failed Authentication-Results header" do
+      Fabricate(:user, email: "existing@bar.com", trust_level: SiteSetting.email_in_min_trust)
+      expect { process(:dmarc_fail) }.to change { Topic.count }.by(1) # Topic created
+
+      topic = Topic.last
+      expect(topic.visible).to eq(false)
+
+      post = Post.last
+      expect(post.hidden).to eq(true)
+      expect(post.hidden_at).not_to eq(nil)
+      expect(post.hidden_reason_id).to eq(Post.hidden_reasons[:email_authentication_result_header])
+    end
+
     it "adds the 'elided' part of the original message when always_show_trimmed_content is enabled" do
       SiteSetting.always_show_trimmed_content = true
 

spec/fixtures/emails/dmarc_fail.eml

@@ -0,0 +1,12 @@
+Return-Path: <existing@bar.com>
+From: Foo Bar <existing@bar.com>
+To: category@bar.com
+Subject: This is a topic from an existing user
+Date: Fri, 15 Jan 2016 00:12:43 +0100
+Message-ID: <32@foo.bar.mail>
+Mime-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: quoted-printable
+Authentication-Results: example.com; dmarc=fail
+
+Hey, this is a topic from an existing user ;)