diff --git a/app/models/post.rb b/app/models/post.rb index a189e57a964..e3487232428 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -133,7 +133,8 @@ class Post < ActiveRecord::Base new_user_spam_threshold_reached: 3, flagged_by_tl3_user: 4, email_spam_header_found: 5, - flagged_by_tl4_user: 6) + flagged_by_tl4_user: 6, + email_authentication_result_header: 7) end def self.types diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 6c20fb1c126..54e8ab91608 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -1900,6 +1900,7 @@ en: log_mail_processing_failures: "Log all email processing failures to /logs" email_in: 'Allow users to post new topics via email (requires manual or pop3 polling). Configure the addresses in the "Settings" tab of each category.' email_in_min_trust: "The minimum trust level a user needs to have to be allowed to post new topics via email." + email_in_authserv_id: "The identifier of the service doing authentication checks on incoming emails. See https://meta.discourse.org/t/134358 for instructions on how to configure this." email_in_spam_header: "The email header to detect spam." email_prefix: "The [label] used in the subject of emails. It will default to 'title' if not set." email_site_title: "The title of the site used as the sender of emails from the site. Default to 'title' if not set. If your 'title' contains characters that are not allowed in email sender strings, use this setting." diff --git a/config/site_settings.yml b/config/site_settings.yml index ce91ea6237f..9ad1751d21a 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -965,6 +965,8 @@ email: email_in_min_trust: default: 2 enum: "TrustLevelSetting" + email_in_authserv_id: + default: "" email_in_spam_header: type: enum default: "none" diff --git a/lib/email/authentication_results.rb b/lib/email/authentication_results.rb new file mode 100644 index 00000000000..c5ad6e22499 --- /dev/null +++ b/lib/email/authentication_results.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module Email + class AuthenticationResults + attr_reader :results + + VERDICT = Enum.new( + :gray, + :pass, + :fail, + start: 0, + ) + + def initialize(headers) + authserv_id = SiteSetting.email_in_authserv_id + @results = Array(headers).map do |header| + parse_header(header.to_s) + end.filter do |result| + authserv_id.blank? || authserv_id == result[:authserv_id] + end + end + + def action + @action ||= calc_action + end + + def verdict + @verdict ||= calc_verdict + end + + private + + def calc_action + if verdict == :fail + :hide + else + :accept + end + end + + def calc_verdict + VERDICT[calc_dmarc] + end + + def calc_dmarc + verdict = VERDICT[:gray] + @results.each do |result| + result[:resinfo].each do |resinfo| + if resinfo[:method] == "dmarc" + v = VERDICT[resinfo[:result].to_sym].to_i + verdict = v if v > verdict + end + end + end + verdict = VERDICT[:gray] if SiteSetting.email_in_authserv_id.blank? && verdict == VERDICT[:pass] + verdict + end + + def parse_header(header) + # based on https://tools.ietf.org/html/rfc8601#section-2.2 + cfws = /\s*(\([^()]*\))?\s*/ + value = /(?:"([^"]*)")|(?:([^\s";]*))/ + authserv_id = value + authres_version = /\d+#{cfws}?/ + no_result = /#{cfws}?;#{cfws}?none/ + keyword = /([a-zA-Z0-9-]*[a-zA-Z0-9])/ + authres_payload = /\A#{cfws}?#{authserv_id}(?:#{cfws}#{authres_version})?(?:#{no_result}|([\S\s]*))/ + + method_version = authres_version + method = /#{keyword}\s*(?:#{cfws}?\/#{cfws}?#{method_version})?/ + result = keyword + methodspec = /#{cfws}?#{method}#{cfws}?=#{cfws}?#{result}/ + reasonspec = /reason#{cfws}?=#{cfws}?#{value}/ + resinfo = /#{cfws}?;#{methodspec}(?:#{cfws}#{reasonspec})?(?:#{cfws}([^;]*))?/ + + ptype = keyword + property = value + pvalue = /#{cfws}?#{value}#{cfws}?/ + propspec = /#{ptype}#{cfws}?\.#{cfws}?#{property}#{cfws}?=#{pvalue}/ + + authres_payload_match = authres_payload.match(header) + parsed_authserv_id = authres_payload_match[2] || authres_payload_match[3] + resinfo_val = authres_payload_match[-1] + + if resinfo_val + resinfo_scan = resinfo_val.scan(resinfo) + parsed_resinfo = resinfo_scan.map do |x| + { + method: x[2], + result: x[8], + reason: x[12] || x[13], + props: x[-1].scan(propspec).map do |y| + { + ptype: y[0], + property: y[4], + pvalue: y[8] || y[9] + } + end + } + end + end + + { + authserv_id: parsed_authserv_id, + resinfo: parsed_resinfo + } + end + + end +end diff --git a/lib/email/receiver.rb b/lib/email/receiver.rb index 41035dc3816..836405aae50 100644 --- a/lib/email/receiver.rb +++ b/lib/email/receiver.rb @@ -196,7 +196,14 @@ module Email end def hidden_reason_id - @hidden_reason_id ||= is_spam? ? Post.hidden_reasons[:email_spam_header_found] : nil + @hidden_reason_id ||= + if is_spam? + Post.hidden_reasons[:email_spam_header_found] + elsif auth_res_action == :hide + Post.hidden_reasons[:email_authentication_result_header] + else + nil + end end def log_and_validate_user(user) @@ -308,6 +315,10 @@ module Email end end + def auth_res_action + @auth_res_action ||= AuthenticationResults.new(@mail.header[:authentication_results]).action + end + def select_body text = nil html = nil diff --git a/spec/components/email/authentication_results_spec.rb b/spec/components/email/authentication_results_spec.rb new file mode 100644 index 00000000000..8a103c89bdd --- /dev/null +++ b/spec/components/email/authentication_results_spec.rb @@ -0,0 +1,299 @@ +# frozen_string_literal: true + +require "rails_helper" +require "email/authentication_results" + +describe Email::AuthenticationResults do + describe "#results" do + it "parses 'Nearly Trivial Case: Service Provided, but No Authentication Done' correctly" do + # https://tools.ietf.org/html/rfc8601#appendix-B.2 + results = described_class.new(" example.org 1; none").results + expect(results[0][:authserv_id]).to eq "example.org" + expect(results[0][:resinfo]).to be nil + end + + it "parses 'Service Provided, Authentication Done' correctly" do + # https://tools.ietf.org/html/rfc8601#appendix-B.3 + results = described_class.new(<<~EOF + example.com; + spf=pass smtp.mailfrom=example.net + EOF + ).results + expect(results[0][:authserv_id]).to eq "example.com" + expect(results[0][:resinfo][0][:method]).to eq "spf" + expect(results[0][:resinfo][0][:result]).to eq "pass" + expect(results[0][:resinfo][0][:reason]).to be nil + expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "smtp" + expect(results[0][:resinfo][0][:props][0][:property]).to eq "mailfrom" + expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "example.net" + end + + it "parses 'Service Provided, Several Authentications Done, Single MTA' correctly" do + # https://tools.ietf.org/html/rfc8601#appendix-B.4 + results = described_class.new([<<~EOF , + example.com; + auth=pass (cram-md5) smtp.auth=sender@example.net; + spf=pass smtp.mailfrom=example.net + EOF + <<~EOF , + example.com; iprev=pass + policy.iprev=192.0.2.200 + EOF + ]).results + expect(results[0][:authserv_id]).to eq "example.com" + expect(results[0][:resinfo][0][:method]).to eq "auth" + expect(results[0][:resinfo][0][:result]).to eq "pass" + expect(results[0][:resinfo][0][:reason]).to be nil + expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "smtp" + expect(results[0][:resinfo][0][:props][0][:property]).to eq "auth" + expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "sender@example.net" + expect(results[0][:resinfo][1][:method]).to eq "spf" + expect(results[0][:resinfo][1][:result]).to eq "pass" + expect(results[0][:resinfo][1][:reason]).to be nil + expect(results[0][:resinfo][1][:props][0][:ptype]).to eq "smtp" + expect(results[0][:resinfo][1][:props][0][:property]).to eq "mailfrom" + expect(results[0][:resinfo][1][:props][0][:pvalue]).to eq "example.net" + expect(results[1][:authserv_id]).to eq "example.com" + expect(results[1][:resinfo][0][:method]).to eq "iprev" + expect(results[1][:resinfo][0][:result]).to eq "pass" + expect(results[1][:resinfo][0][:reason]).to be nil + expect(results[1][:resinfo][0][:props][0][:ptype]).to eq "policy" + expect(results[1][:resinfo][0][:props][0][:property]).to eq "iprev" + expect(results[1][:resinfo][0][:props][0][:pvalue]).to eq "192.0.2.200" + end + + it "parses 'Service Provided, Several Authentications Done, Different MTAs' correctly" do + # https://tools.ietf.org/html/rfc8601#appendix-B.5 + results = described_class.new([<<~EOF , + example.com; + dkim=pass (good signature) header.d=example.com + EOF + <<~EOF , + example.com; + auth=pass (cram-md5) smtp.auth=sender@example.com; + spf=fail smtp.mailfrom=example.com + EOF + ]).results + + expect(results[0][:authserv_id]).to eq "example.com" + expect(results[0][:resinfo][0][:method]).to eq "dkim" + expect(results[0][:resinfo][0][:result]).to eq "pass" + expect(results[0][:resinfo][0][:reason]).to be nil + expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "header" + expect(results[0][:resinfo][0][:props][0][:property]).to eq "d" + expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "example.com" + expect(results[1][:authserv_id]).to eq "example.com" + expect(results[1][:resinfo][0][:method]).to eq "auth" + expect(results[1][:resinfo][0][:result]).to eq "pass" + expect(results[1][:resinfo][0][:reason]).to be nil + expect(results[1][:resinfo][0][:props][0][:ptype]).to eq "smtp" + expect(results[1][:resinfo][0][:props][0][:property]).to eq "auth" + expect(results[1][:resinfo][0][:props][0][:pvalue]).to eq "sender@example.com" + expect(results[1][:resinfo][1][:method]).to eq "spf" + expect(results[1][:resinfo][1][:result]).to eq "fail" + expect(results[1][:resinfo][1][:reason]).to be nil + expect(results[1][:resinfo][1][:props][0][:ptype]).to eq "smtp" + expect(results[1][:resinfo][1][:props][0][:property]).to eq "mailfrom" + expect(results[1][:resinfo][1][:props][0][:pvalue]).to eq "example.com" + end + + it "parses 'Service Provided, Multi-tiered Authentication Done' correctly" do + # https://tools.ietf.org/html/rfc8601#appendix-B.6 + results = described_class.new([<<~EOF , + example.com; + dkim=pass reason="good signature" + header.i=@mail-router.example.net; + dkim=fail reason="bad signature" + header.i=@newyork.example.com + EOF + <<~EOF , + example.net; + dkim=pass (good signature) header.i=@newyork.example.com + EOF + ]).results + + expect(results[0][:authserv_id]).to eq "example.com" + expect(results[0][:resinfo][0][:method]).to eq "dkim" + expect(results[0][:resinfo][0][:result]).to eq "pass" + expect(results[0][:resinfo][0][:reason]).to eq "good signature" + expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "header" + expect(results[0][:resinfo][0][:props][0][:property]).to eq "i" + expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "@mail-router.example.net" + expect(results[0][:resinfo][1][:method]).to eq "dkim" + expect(results[0][:resinfo][1][:result]).to eq "fail" + expect(results[0][:resinfo][1][:reason]).to eq "bad signature" + expect(results[0][:resinfo][1][:props][0][:ptype]).to eq "header" + expect(results[0][:resinfo][1][:props][0][:property]).to eq "i" + expect(results[0][:resinfo][1][:props][0][:pvalue]).to eq "@newyork.example.com" + expect(results[1][:authserv_id]).to eq "example.net" + expect(results[1][:resinfo][0][:method]).to eq "dkim" + expect(results[1][:resinfo][0][:result]).to eq "pass" + expect(results[1][:resinfo][0][:reason]).to be nil + expect(results[1][:resinfo][0][:props][0][:ptype]).to eq "header" + expect(results[1][:resinfo][0][:props][0][:property]).to eq "i" + expect(results[1][:resinfo][0][:props][0][:pvalue]).to eq "@newyork.example.com" + end + + it "parses 'Comment-Heavy Example' correctly" do + # https://tools.ietf.org/html/rfc8601#appendix-B.7 + results = described_class.new(<<~EOF + foo.example.net (foobar) 1 (baz); + dkim (Because I like it) / 1 (One yay) = (wait for it) fail + policy (A dot can go here) . (like that) expired + (this surprised me) = (as I wasn't expecting it) 1362471462 + EOF + ).results + + expect(results[0][:authserv_id]).to eq "foo.example.net" + expect(results[0][:resinfo][0][:method]).to eq "dkim" + expect(results[0][:resinfo][0][:result]).to eq "fail" + expect(results[0][:resinfo][0][:reason]).to be nil + expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "policy" + expect(results[0][:resinfo][0][:props][0][:property]).to eq "expired" + expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "1362471462" + end + + it "parses header with no props correctly" do + results = described_class.new(" example.com; dmarc=pass").results + expect(results[0][:authserv_id]).to eq "example.com" + expect(results[0][:resinfo][0][:method]).to eq "dmarc" + expect(results[0][:resinfo][0][:result]).to eq "pass" + expect(results[0][:resinfo][0][:reason]).to be nil + expect(results[0][:resinfo][0][:props]).to eq [] + end + + it "parses header with multiple props correctly" do + results = described_class.new(<<~EOF + mx.google.com; + dkim=pass header.i=@email.example.com header.s=20111006 header.b=URn9MW+F; + spf=pass (google.com: domain of foo@b.email.example.com designates 1.2.3.4 as permitted sender) smtp.mailfrom=foo@b.email.example.com; + dmarc=pass (p=REJECT sp=REJECT dis=NONE) header.from=email.example.com + EOF + ).results + + expect(results[0][:authserv_id]).to eq "mx.google.com" + expect(results[0][:resinfo][0][:method]).to eq "dkim" + expect(results[0][:resinfo][0][:result]).to eq "pass" + expect(results[0][:resinfo][0][:reason]).to be nil + expect(results[0][:resinfo][0][:props][0][:ptype]).to eq "header" + expect(results[0][:resinfo][0][:props][0][:property]).to eq "i" + expect(results[0][:resinfo][0][:props][0][:pvalue]).to eq "@email.example.com" + expect(results[0][:resinfo][0][:props][1][:ptype]).to eq "header" + expect(results[0][:resinfo][0][:props][1][:property]).to eq "s" + expect(results[0][:resinfo][0][:props][1][:pvalue]).to eq "20111006" + expect(results[0][:resinfo][0][:props][2][:ptype]).to eq "header" + expect(results[0][:resinfo][0][:props][2][:property]).to eq "b" + expect(results[0][:resinfo][0][:props][2][:pvalue]).to eq "URn9MW+F" + expect(results[0][:resinfo][1][:method]).to eq "spf" + expect(results[0][:resinfo][1][:result]).to eq "pass" + expect(results[0][:resinfo][1][:reason]).to be nil + expect(results[0][:resinfo][1][:props][0][:ptype]).to eq "smtp" + expect(results[0][:resinfo][1][:props][0][:property]).to eq "mailfrom" + expect(results[0][:resinfo][1][:props][0][:pvalue]).to eq "foo@b.email.example.com" + expect(results[0][:resinfo][2][:method]).to eq "dmarc" + expect(results[0][:resinfo][2][:result]).to eq "pass" + expect(results[0][:resinfo][2][:reason]).to be nil + expect(results[0][:resinfo][2][:props][0][:ptype]).to eq "header" + expect(results[0][:resinfo][2][:props][0][:property]).to eq "from" + expect(results[0][:resinfo][2][:props][0][:pvalue]).to eq "email.example.com" + end + end + + describe "#verdict" do + before do + SiteSetting.email_in_authserv_id = "valid.com" + end + + shared_examples "is verdict" do |verdict| + it "is #{verdict}" do + expect(described_class.new(headers).verdict).to eq verdict + end + end + + context "with no authentication-results headers" do + let(:headers) { "" } + + it "is gray" do + expect(described_class.new(headers).verdict).to eq :gray + end + end + + context "with a single authentication-results header" do + context "with a valid fail" do + let(:headers) { "valid.com; dmarc=fail" } + include_examples "is verdict", :fail + end + + context "with a valid pass" do + let(:headers) { "valid.com; dmarc=pass" } + include_examples "is verdict", :pass + end + + context "with a valid error" do + let(:headers) { "valid.com; dmarc=error" } + include_examples "is verdict", :gray + end + + context "with no email_in_authserv_id set" do + before { SiteSetting.email_in_authserv_id = "" } + + context "with a fail" do + let(:headers) { "foobar.com; dmarc=fail" } + include_examples "is verdict", :fail + end + + context "with a pass" do + let(:headers) { "foobar.com; dmarc=pass" } + include_examples "is verdict", :gray + end + end + end + + context "with multiple authentication-results headers" do + context "with a valid fail, and an invalid pass" do + let(:headers) { ["valid.com; dmarc=fail", "invalid.com; dmarc=pass"] } + include_examples "is verdict", :fail + end + + context "with a valid fail, and a valid pass" do + let(:headers) { ["valid.com; dmarc=fail", "valid.com; dmarc=pass"] } + include_examples "is verdict", :fail + end + + context "with a valid error, and a valid pass" do + let(:headers) { ["valid.com; dmarc=foobar", "valid.com; dmarc=pass"] } + include_examples "is verdict", :pass + end + + context "with no email_in_authserv_id set" do + before { SiteSetting.email_in_authserv_id = "" } + + context "with an error, and a pass" do + let(:headers) { ["foobar.com; dmarc=foobar", "foobar.com; dmarc=pass"] } + include_examples "is verdict", :gray + end + end + end + end + + describe "#action" do + it "hides a fail verdict" do + results = described_class.new("") + results.expects(:verdict).returns(:fail) + expect(results.action).to eq (:hide) + end + + it "accepts a pass verdict" do + results = described_class.new("") + results.expects(:verdict).returns(:pass) + expect(results.action).to eq (:accept) + end + + it "accepts a gray verdict" do + results = described_class.new("") + results.expects(:verdict).returns(:gray) + expect(results.action).to eq (:accept) + end + end + +end diff --git a/spec/components/email/receiver_spec.rb b/spec/components/email/receiver_spec.rb index 811fc3839c8..400f72385c7 100644 --- a/spec/components/email/receiver_spec.rb +++ b/spec/components/email/receiver_spec.rb @@ -1021,6 +1021,19 @@ describe Email::Receiver do expect(post.hidden_reason_id).to eq(Post.hidden_reasons[:email_spam_header_found]) end + it "creates hidden topic for failed Authentication-Results header" do + Fabricate(:user, email: "existing@bar.com", trust_level: SiteSetting.email_in_min_trust) + expect { process(:dmarc_fail) }.to change { Topic.count }.by(1) # Topic created + + topic = Topic.last + expect(topic.visible).to eq(false) + + post = Post.last + expect(post.hidden).to eq(true) + expect(post.hidden_at).not_to eq(nil) + expect(post.hidden_reason_id).to eq(Post.hidden_reasons[:email_authentication_result_header]) + end + it "adds the 'elided' part of the original message when always_show_trimmed_content is enabled" do SiteSetting.always_show_trimmed_content = true diff --git a/spec/fixtures/emails/dmarc_fail.eml b/spec/fixtures/emails/dmarc_fail.eml new file mode 100644 index 00000000000..4fe78ef7ed9 --- /dev/null +++ b/spec/fixtures/emails/dmarc_fail.eml @@ -0,0 +1,12 @@ +Return-Path: +From: Foo Bar +To: category@bar.com +Subject: This is a topic from an existing user +Date: Fri, 15 Jan 2016 00:12:43 +0100 +Message-ID: <32@foo.bar.mail> +Mime-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable +Authentication-Results: example.com; dmarc=fail + +Hey, this is a topic from an existing user ;)