From c72bc27888df772200add0dbd3ebc9551ecfd1a3 Mon Sep 17 00:00:00 2001 From: Dan Ungureanu Date: Fri, 10 Jul 2020 12:05:55 +0300 Subject: [PATCH] FEATURE: Implement support for IMAP and SMTP email protocols. (#8301) Co-authored-by: Joffrey JAFFEUX --- .../components/groups-form-email-fields.js | 19 + .../discourse/app/controllers/group-manage.js | 5 + .../javascripts/discourse/app/models/group.js | 9 + .../discourse/app/routes/app-route-map.js | 1 + .../app/routes/group-manage-email.js | 10 + .../components/groups-form-email-fields.hbs | 67 ++++ .../app/templates/group/manage/email.hbs | 4 + .../stylesheets/common/base/groups.scss | 6 + app/controllers/groups_controller.rb | 9 + app/jobs/regular/group_smtp_email.rb | 29 ++ app/jobs/regular/process_email.rb | 2 +- app/mailers/group_smtp_mailer.rb | 121 +++++++ app/models/group.rb | 51 +++ app/models/group_archived_message.rb | 17 +- app/models/incoming_email.rb | 8 +- app/models/topic.rb | 4 +- app/serializers/basic_group_serializer.rb | 28 +- app/services/post_alerter.rb | 50 ++- config/locales/client.en.yml | 17 + config/locales/server.en.yml | 11 + config/routes.rb | 1 + config/site_settings.yml | 8 + config/unicorn.conf.rb | 158 ++++++--- ...81220115844_add_smtp_and_imap_to_groups.rb | 20 ++ ...3409_add_imap_fields_to_incoming_emails.rb | 11 + .../20200327164420_add_imap_stats_to_group.rb | 9 + lib/demon/email_sync.rb | 161 +++++++++ lib/email/message_builder.rb | 6 +- lib/email/processor.rb | 12 +- lib/email/receiver.rb | 86 +++-- lib/imap/providers/generic.rb | 116 ++++++ lib/imap/providers/gmail.rb | 145 ++++++++ lib/imap/sync.rb | 258 ++++++++++++++ lib/post_creator.rb | 6 +- lib/post_revisor.rb | 4 + spec/components/email/processor_spec.rb | 4 +- spec/components/email/receiver_spec.rb | 78 +++- spec/components/imap/imap_helper.rb | 28 ++ spec/components/imap/sync_spec.rb | 333 ++++++++++++++++++ spec/jobs/process_email_spec.rb | 2 +- spec/mailers/group_smtp_mailer_spec.rb | 76 ++++ .../basic_group_serializer_spec.rb | 22 ++ spec/services/post_alerter_spec.rb | 72 ++++ 43 files changed, 1956 insertions(+), 128 deletions(-) create mode 100644 app/assets/javascripts/discourse/app/components/groups-form-email-fields.js create mode 100644 app/assets/javascripts/discourse/app/routes/group-manage-email.js create mode 100644 app/assets/javascripts/discourse/app/templates/components/groups-form-email-fields.hbs create mode 100644 app/assets/javascripts/discourse/app/templates/group/manage/email.hbs create mode 100644 app/jobs/regular/group_smtp_email.rb create mode 100644 app/mailers/group_smtp_mailer.rb create mode 100644 db/migrate/20181220115844_add_smtp_and_imap_to_groups.rb create mode 100644 db/migrate/20190111183409_add_imap_fields_to_incoming_emails.rb create mode 100644 db/migrate/20200327164420_add_imap_stats_to_group.rb create mode 100644 lib/demon/email_sync.rb create mode 100644 lib/imap/providers/generic.rb create mode 100644 lib/imap/providers/gmail.rb create mode 100644 lib/imap/sync.rb create mode 100644 spec/components/imap/imap_helper.rb create mode 100644 spec/components/imap/sync_spec.rb create mode 100644 spec/mailers/group_smtp_mailer_spec.rb diff --git a/app/assets/javascripts/discourse/app/components/groups-form-email-fields.js b/app/assets/javascripts/discourse/app/components/groups-form-email-fields.js new file mode 100644 index 00000000000..faebb6310f0 --- /dev/null +++ b/app/assets/javascripts/discourse/app/components/groups-form-email-fields.js @@ -0,0 +1,19 @@ +import Component from "@ember/component"; +import discourseComputed from "discourse-common/utils/decorators"; + +export default Component.extend({ + @discourseComputed("model.imap_mailboxes") + mailboxes(imapMailboxes) { + return imapMailboxes.map(mailbox => ({ name: mailbox, value: mailbox })); + }, + + @discourseComputed("model.imap_old_emails") + oldEmails(oldEmails) { + return oldEmails || 0; + }, + + @discourseComputed("model.imap_old_emails", "model.imap_new_emails") + totalEmails(oldEmails, newEmails) { + return (oldEmails || 0) + (newEmails || 0); + } +}); diff --git a/app/assets/javascripts/discourse/app/controllers/group-manage.js b/app/assets/javascripts/discourse/app/controllers/group-manage.js index 7a9e7859eb0..cc5bd11a5c0 100644 --- a/app/assets/javascripts/discourse/app/controllers/group-manage.js +++ b/app/assets/javascripts/discourse/app/controllers/group-manage.js @@ -18,6 +18,11 @@ export default Controller.extend({ ]; if (!automatic) { + defaultTabs.splice(2, 0, { + route: "group.manage.email", + title: "groups.manage.email.title" + }); + defaultTabs.splice(1, 0, { route: "group.manage.membership", title: "groups.manage.membership.title" diff --git a/app/assets/javascripts/discourse/app/models/group.js b/app/assets/javascripts/discourse/app/models/group.js index 142341499ea..af514bda0a4 100644 --- a/app/assets/javascripts/discourse/app/models/group.js +++ b/app/assets/javascripts/discourse/app/models/group.js @@ -188,6 +188,15 @@ const Group = RestModel.extend({ primary_group: !!this.primary_group, grant_trust_level: this.grant_trust_level, incoming_email: this.incoming_email, + smtp_server: this.smtp_server, + smtp_port: this.smtp_port, + smtp_ssl: this.smtp_ssl, + imap_server: this.imap_server, + imap_port: this.imap_port, + imap_ssl: this.imap_ssl, + imap_mailbox_name: this.imap_mailbox_name, + email_username: this.email_username, + email_password: this.email_password, flair_icon: null, flair_upload_id: null, flair_bg_color: this.flairBackgroundHexColor, diff --git a/app/assets/javascripts/discourse/app/routes/app-route-map.js b/app/assets/javascripts/discourse/app/routes/app-route-map.js index 9057ff2fe50..23aa5e23a85 100644 --- a/app/assets/javascripts/discourse/app/routes/app-route-map.js +++ b/app/assets/javascripts/discourse/app/routes/app-route-map.js @@ -92,6 +92,7 @@ export default function() { this.route("profile"); this.route("membership"); this.route("interaction"); + this.route("email"); this.route("members"); this.route("logs"); }); diff --git a/app/assets/javascripts/discourse/app/routes/group-manage-email.js b/app/assets/javascripts/discourse/app/routes/group-manage-email.js new file mode 100644 index 00000000000..58ce783b48b --- /dev/null +++ b/app/assets/javascripts/discourse/app/routes/group-manage-email.js @@ -0,0 +1,10 @@ +import DiscourseRoute from "discourse/routes/discourse"; +import I18n from "I18n"; + +export default DiscourseRoute.extend({ + showFooter: true, + + titleToken() { + return I18n.t("groups.manage.email.title"); + } +}); diff --git a/app/assets/javascripts/discourse/app/templates/components/groups-form-email-fields.hbs b/app/assets/javascripts/discourse/app/templates/components/groups-form-email-fields.hbs new file mode 100644 index 00000000000..0a8e08f0a3b --- /dev/null +++ b/app/assets/javascripts/discourse/app/templates/components/groups-form-email-fields.hbs @@ -0,0 +1,67 @@ +{{#if currentUser.admin}} +
+ +
+ + {{#if model.imap_last_error}} +
{{model.imap_last_error}}
+ {{else}} +
+ {{i18n "groups.manage.email.status" old_emails=oldEmails total_emails=totalEmails}} +
+ {{/if}} + +
+ + {{input type="text" name="smtp_server" value=model.smtp_server}} +
+ +
+ + {{input type="text" name="smtp_port" value=model.smtp_port}} +
+ +
+ {{input type="checkbox" name="smtp_ssl" checked=model.smtp_ssl}} + +
+ +
+ + {{input type="text" name="imap_server" value=model.imap_server}} +
+ +
+ + {{input type="text" name="imap_port" value=model.imap_port}} +
+ +
+ {{input type="checkbox" name="imap_ssl" checked=model.imap_ssl}} + +
+ +
+ + {{input type="text" name="username" value=model.email_username}} +
+ +
+ + {{input type="password" name="password" value=model.email_password}} +
+ +
+ {{#if mailboxes}} + + {{combo-box name="imap_mailbox_name" + value=model.imap_mailbox_name + valueProperty="value" + content=mailboxes + none="groups.manage.email.mailboxes.disabled" + onChange=(action (mut model.imap_mailbox_name))}} + {{else}} + {{i18n "groups.manage.email.mailboxes.none_found"}} + {{/if}} +
+{{/if}} diff --git a/app/assets/javascripts/discourse/app/templates/group/manage/email.hbs b/app/assets/javascripts/discourse/app/templates/group/manage/email.hbs new file mode 100644 index 00000000000..6ef5aada789 --- /dev/null +++ b/app/assets/javascripts/discourse/app/templates/group/manage/email.hbs @@ -0,0 +1,4 @@ +
+ {{groups-form-email-fields model=model}} + {{group-manage-save-button model=model}} +
diff --git a/app/assets/stylesheets/common/base/groups.scss b/app/assets/stylesheets/common/base/groups.scss index 0cc6fcd8429..3419cb53b46 100644 --- a/app/assets/stylesheets/common/base/groups.scss +++ b/app/assets/stylesheets/common/base/groups.scss @@ -141,3 +141,9 @@ } } } + +.groups-form { + .control-group-inline { + display: inline; + } +} diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb index 2e112edd049..08205d06635 100644 --- a/app/controllers/groups_controller.rb +++ b/app/controllers/groups_controller.rb @@ -547,6 +547,15 @@ class GroupsController < ApplicationController if current_user.admin default_params.push(*[ :incoming_email, + :smtp_server, + :smtp_port, + :smtp_ssl, + :imap_server, + :imap_port, + :imap_ssl, + :imap_mailbox_name, + :email_username, + :email_password, :primary_group, :visibility_level, :members_visibility_level, diff --git a/app/jobs/regular/group_smtp_email.rb b/app/jobs/regular/group_smtp_email.rb new file mode 100644 index 00000000000..6ff4301b3f2 --- /dev/null +++ b/app/jobs/regular/group_smtp_email.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require_dependency 'email/sender' + +module Jobs + class GroupSmtpEmail < ::Jobs::Base + sidekiq_options queue: 'critical' + + def execute(args) + group = Group.find_by(id: args[:group_id]) + post = Post.find_by(id: args[:post_id]) + email = args[:email] + + Rails.logger.debug("[IMAP] Sending email for group #{group.name} and post #{post.id}") + message = GroupSmtpMailer.send_mail(group, email, post) + Email::Sender.new(message, :group_smtp).send + + # Create an incoming email record to avoid importing again from IMAP + # server. + IncomingEmail.create!( + user_id: post.user_id, + topic_id: post.topic_id, + post_id: post.id, + raw: message.to_s, + message_id: message.message_id + ) + end + end +end diff --git a/app/jobs/regular/process_email.rb b/app/jobs/regular/process_email.rb index f738fc5be3f..1a6f9d1db80 100644 --- a/app/jobs/regular/process_email.rb +++ b/app/jobs/regular/process_email.rb @@ -6,7 +6,7 @@ module Jobs sidekiq_options retry: 3 def execute(args) - Email::Processor.process!(args[:mail], args[:retry_on_rate_limit] || false) + Email::Processor.process!(args[:mail], retry_on_rate_limit: args[:retry_on_rate_limit] || false) end sidekiq_retries_exhausted do |msg| diff --git a/app/mailers/group_smtp_mailer.rb b/app/mailers/group_smtp_mailer.rb new file mode 100644 index 00000000000..15b9051b6e6 --- /dev/null +++ b/app/mailers/group_smtp_mailer.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +require_dependency 'email/message_builder' + +class GroupSmtpMailer < ActionMailer::Base + include Email::BuildEmailHelper + + def send_mail(from_group, to_address, post) + raise 'SMTP is disabled' if !SiteSetting.enable_smtp + + incoming_email = IncomingEmail.joins(:post) + .where('imap_uid IS NOT NULL') + .where(topic_id: post.topic_id, posts: { post_number: 1 }) + .limit(1).first + + context_posts = Post + .where(topic_id: post.topic_id) + .where("post_number < ?", post.post_number) + .where(user_deleted: false) + .where(hidden: false) + .where(post_type: Post.types[:regular]) + .order(created_at: :desc) + .limit(SiteSetting.email_posts_context) + .to_a + + delivery_options = { + address: from_group.smtp_server, + port: from_group.smtp_port, + domain: from_group.email_username.split('@').last, + user_name: from_group.email_username, + password: from_group.email_password, + authentication: GlobalSetting.smtp_authentication, + enable_starttls_auto: from_group.smtp_ssl + } + + user_name = post.user.username + if SiteSetting.enable_names && SiteSetting.display_name_on_email_from + user_name = post.user.name unless post.user.name.blank? + end + + build_email(to_address, + message: post.raw, + url: post.url(without_slug: SiteSetting.private_email?), + post_id: post.id, + topic_id: post.topic_id, + context: context(context_posts), + username: post.user.username, + group_name: from_group.name, + allow_reply_by_email: true, + only_reply_by_email: true, + private_reply: post.topic.private_message?, + participants: participants(post), + include_respond_instructions: true, + template: 'user_notifications.user_posted_pm', + use_topic_title_subject: true, + topic_title: incoming_email&.subject || post.topic.title, + add_re_to_subject: true, + locale: SiteSetting.default_locale, + delivery_method_options: delivery_options, + from: from_group.email_username, + from_alias: I18n.t('email_from', user_name: user_name, site_name: Email.site_title), + html_override: html_override(post, context_posts: context_posts) + ) + end + + private + + def context(context_posts) + return "" if SiteSetting.private_email? + + context = +"" + + if context_posts.size > 0 + context << +"-- \n*#{I18n.t('user_notifications.previous_discussion')}*\n" + context_posts.each { |post| context << email_post_markdown(post, true) } + end + + context + end + + def email_post_markdown(post, add_posted_by = false) + result = +"#{post.with_secure_media? ? strip_secure_urls(post.raw) : post.raw}\n\n" + if add_posted_by + result << "#{I18n.t('user_notifications.posted_by', username: post.username, post_date: post.created_at.strftime("%m/%d/%Y"))}\n\n" + end + result + end + + def html_override(post, context_posts: nil) + UserNotificationRenderer.render( + template: 'email/notification', + format: :html, + locals: { + context_posts: context_posts, + reached_limit: nil, + post: post, + in_reply_to_post: post.reply_to_post, + classes: Rtl.new(nil).css_class, + first_footer_classes: '' + } + ) + end + + def participants(post) + list = [] + + post.topic.allowed_groups.each do |g| + list.push("[#{g.name} (#{g.users.count})](#{Discourse.base_url}/groups/#{g.name})") + end + + post.topic.allowed_users.each do |u| + if SiteSetting.prioritize_username_in_ux? + list.push("[#{u.username}](#{Discourse.base_url}/u/#{u.username_lower})") + else + list.push("[#{u.name.blank? ? u.username : u.name}](#{Discourse.base_url}/u/#{u.username_lower})") + end + end + + list.join(', ') + end +end diff --git a/app/models/group.rb b/app/models/group.rb index 3d15993b9c3..34a91cb7f59 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'net/imap' + class Group < ActiveRecord::Base # TODO(2021-05-26): remove self.ignored_columns = %w{ @@ -53,6 +55,7 @@ class Group < ActiveRecord::Base def expire_cache ApplicationSerializer.expire_cache_fragment!("group_names") SvgSprite.expire_cache + Discourse.cache.delete("group_imap_mailboxes_#{self.id}") end def remove_review_groups @@ -752,6 +755,40 @@ class Group < ActiveRecord::Base flair_icon.presence || flair_upload&.short_path end + def imap_mailboxes + return [] if self.imap_server.blank? || + self.email_username.blank? || + self.email_password.blank? + + Discourse.cache.fetch("group_imap_mailboxes_#{self.id}", expires_in: 30.minutes) do + Rails.logger.info("[IMAP] Refreshing mailboxes list for group #{self.name}") + mailboxes = [] + + begin + @imap = Net::IMAP.new(self.imap_server, self.imap_port, self.imap_ssl) + @imap.login(self.email_username, self.email_password) + + @imap.list('', '*').each do |m| + next if m.attr.include?(:Noselect) + mailboxes << m.name + end + + update_columns(imap_last_error: nil) + rescue => ex + update_columns(imap_last_error: ex.message) + end + + mailboxes + end + end + + def email_username_regex + user, domain = email_username.split('@') + if user.present? && domain.present? + /^#{Regexp.escape(user)}(\+[^@]*)?@#{Regexp.escape(domain)}$/i + end + end + protected def name_format_validator @@ -935,10 +972,24 @@ end # membership_request_template :text # messageable_level :integer default(0) # mentionable_level :integer default(0) +# smtp_server :string +# smtp_port :integer +# smtp_ssl :boolean +# imap_server :string +# imap_port :integer +# imap_ssl :boolean +# imap_mailbox_name :string default(""), not null +# imap_uid_validity :integer default(0), not null +# imap_last_uid :integer default(0), not null +# email_username :string +# email_password :string # publish_read_state :boolean default(FALSE), not null # members_visibility_level :integer default(0), not null # flair_icon :string # flair_upload_id :integer +# imap_last_error :text +# imap_old_emails :integer +# imap_new_emails :integer # # Indexes # diff --git a/app/models/group_archived_message.rb b/app/models/group_archived_message.rb index 8199a53cf62..6d31b94d206 100644 --- a/app/models/group_archived_message.rb +++ b/app/models/group_archived_message.rb @@ -4,21 +4,23 @@ class GroupArchivedMessage < ActiveRecord::Base belongs_to :group belongs_to :topic - def self.move_to_inbox!(group_id, topic) + def self.move_to_inbox!(group_id, topic, opts = {}) topic_id = topic.id - GroupArchivedMessage.where(group_id: group_id, topic_id: topic_id).destroy_all + destroyed = GroupArchivedMessage.where(group_id: group_id, topic_id: topic_id).destroy_all trigger(:move_to_inbox, group_id, topic_id) MessageBus.publish("/topic/#{topic_id}", { type: "move_to_inbox" }, group_ids: [group_id]) publish_topic_tracking_state(topic) + set_imap_sync(topic_id) if !opts[:skip_imap_sync] && destroyed.present? end - def self.archive!(group_id, topic) + def self.archive!(group_id, topic, opts = {}) topic_id = topic.id - GroupArchivedMessage.where(group_id: group_id, topic_id: topic_id).destroy_all + destroyed = GroupArchivedMessage.where(group_id: group_id, topic_id: topic_id).destroy_all GroupArchivedMessage.create!(group_id: group_id, topic_id: topic_id) trigger(:archive_message, group_id, topic_id) MessageBus.publish("/topic/#{topic_id}", { type: "archived" }, group_ids: [group_id]) publish_topic_tracking_state(topic) + set_imap_sync(topic_id) if !opts[:skip_imap_sync] && destroyed.blank? end def self.trigger(event, group_id, topic_id) @@ -36,6 +38,13 @@ class GroupArchivedMessage < ActiveRecord::Base topic, group_archive: true ) end + + def self.set_imap_sync(topic_id) + IncomingEmail.joins(:post) + .where.not(imap_uid: nil) + .where(topic_id: topic_id, posts: { post_number: 1 }) + .update_all(imap_sync: true) + end end # == Schema Information diff --git a/app/models/incoming_email.rb b/app/models/incoming_email.rb index aa73e0e5ddd..e608abbbf88 100644 --- a/app/models/incoming_email.rb +++ b/app/models/incoming_email.rb @@ -9,6 +9,7 @@ class IncomingEmail < ActiveRecord::Base scope :addressed_to, -> (email) do where(<<~SQL, email: "%#{email}%") + incoming_emails.from_address = :email OR incoming_emails.to_addresses ILIKE :email OR incoming_emails.cc_addresses ILIKE :email SQL @@ -20,7 +21,8 @@ class IncomingEmail < ActiveRecord::Base SELECT 1 FROM user_emails WHERE user_emails.user_id = :user_id AND - (incoming_emails.to_addresses ILIKE '%' || user_emails.email || '%' OR + (incoming_emails.from_address = user_emails.email OR + incoming_emails.to_addresses ILIKE '%' || user_emails.email || '%' OR incoming_emails.cc_addresses ILIKE '%' || user_emails.email || '%') ) SQL @@ -47,11 +49,15 @@ end # rejection_message :text # is_auto_generated :boolean default(FALSE) # is_bounce :boolean default(FALSE), not null +# imap_uid_validity :integer +# imap_uid :integer +# imap_sync :boolean # # Indexes # # index_incoming_emails_on_created_at (created_at) # index_incoming_emails_on_error (error) +# index_incoming_emails_on_imap_sync (imap_sync) # index_incoming_emails_on_message_id (message_id) # index_incoming_emails_on_post_id (post_id) # index_incoming_emails_on_user_id (user_id) WHERE (user_id IS NOT NULL) diff --git a/app/models/topic.rb b/app/models/topic.rb index 06446ca588b..643c48b258c 100644 --- a/app/models/topic.rb +++ b/app/models/topic.rb @@ -200,6 +200,7 @@ class Topic < ActiveRecord::Base has_many :ordered_posts, -> { order(post_number: :asc) }, class_name: "Post" has_many :topic_allowed_users has_many :topic_allowed_groups + has_many :incoming_email has_many :group_archived_messages, dependent: :destroy has_many :user_archived_messages, dependent: :destroy @@ -866,7 +867,8 @@ class Topic < ActiveRecord::Base no_bump: opts[:bump].blank?, topic_id: self.id, skip_validations: true, - custom_fields: opts[:custom_fields]) + custom_fields: opts[:custom_fields], + import_mode: opts[:import_mode]) if (new_post = creator.create) && new_post.present? increment!(:moderator_posts_count) if new_post.persisted? diff --git a/app/serializers/basic_group_serializer.rb b/app/serializers/basic_group_serializer.rb index 1b68420dd3c..ef999031bd6 100644 --- a/app/serializers/basic_group_serializer.rb +++ b/app/serializers/basic_group_serializer.rb @@ -33,6 +33,30 @@ class BasicGroupSerializer < ApplicationSerializer :can_see_members, :publish_read_state + def self.admin_attributes(*attrs) + attributes(*attrs) + attrs.each do |attr| + define_method "include_#{attr}?" do + scope.is_admin? + end + end + end + + admin_attributes :automatic_membership_email_domains, + :smtp_server, + :smtp_port, + :smtp_ssl, + :imap_server, + :imap_port, + :imap_ssl, + :imap_mailbox_name, + :imap_mailboxes, + :email_username, + :email_password, + :imap_last_error, + :imap_old_emails, + :imap_new_emails + def include_display_name? object.automatic end @@ -51,10 +75,6 @@ class BasicGroupSerializer < ApplicationSerializer staff? end - def include_automatic_membership_email_domains? - scope.is_admin? - end - def include_has_messages? staff? || scope.can_see_group_messages?(object) end diff --git a/app/services/post_alerter.rb b/app/services/post_alerter.rb index 23025a58603..a1647dc6f59 100644 --- a/app/services/post_alerter.rb +++ b/app/services/post_alerter.rb @@ -404,7 +404,9 @@ class PostAlerter notification_data[:group_name] = group.name end - if original_post.via_email && (incoming_email = original_post.incoming_email) + if opts[:skip_send_email_to]&.include?(user.email) + skip_send_email = true + elsif original_post.via_email && (incoming_email = original_post.incoming_email) skip_send_email = contains_email_address?(incoming_email.to_addresses, user) || contains_email_address?(incoming_email.cc_addresses, user) else @@ -542,29 +544,69 @@ class PostAlerter users end + def group_notifying_via_smtp(post) + return nil if !SiteSetting.enable_smtp || + post.post_type != Post.types[:regular] || + post.incoming_email + + post.topic.allowed_groups + .where.not(smtp_server: nil) + .where.not(smtp_port: nil) + .where.not(email_username: nil) + .where.not(email_password: nil) + .first + end + def notify_pm_users(post, reply_to_user, notified) return unless post.topic warn_if_not_sidekiq + # users who interacted with the post by _directly_ emailing the group + if group = group_notifying_via_smtp(post) + group_email_regex = group.email_username_regex + email_addresses = Set[group.email_username] + + post.topic.incoming_email.each do |incoming_email| + to_addresses = incoming_email.to_addresses&.split(';') + cc_addresses = incoming_email.cc_addresses&.split(';') + + next if to_addresses&.none? { |address| address =~ group_email_regex } && + cc_addresses&.none? { |address| address =~ group_email_regex } + + email_addresses.add(incoming_email.from_address) + email_addresses.merge(to_addresses) if to_addresses.present? + email_addresses.merge(cc_addresses) if cc_addresses.present? + end + + email_addresses.subtract([nil, '']) + + if email_addresses.size > 1 + Jobs.enqueue(:group_smtp_email, + group_id: group.id, + post_id: post.id, + email: email_addresses.to_a - [group.email_username]) + end + end + # users that aren't part of any mentioned groups users = directly_targeted_users(post).reject { |u| notified.include?(u) } DiscourseEvent.trigger(:before_create_notifications_for_users, users, post) users.each do |user| notification_level = TopicUser.get(post.topic, user)&.notification_level if reply_to_user == user || notification_level == TopicUser.notification_levels[:watching] || user.staged? - create_notification(user, Notification.types[:private_message], post) + create_notification(user, Notification.types[:private_message], post, skip_send_email_to: email_addresses) end end - # users that are part of all mentionned groups + # users that are part of all mentioned groups users = indirectly_targeted_users(post).reject { |u| notified.include?(u) } DiscourseEvent.trigger(:before_create_notifications_for_users, users, post) users.each do |user| case TopicUser.get(post.topic, user)&.notification_level when TopicUser.notification_levels[:watching] # only create a notification when watching the group - create_notification(user, Notification.types[:private_message], post) + create_notification(user, Notification.types[:private_message], post, skip_send_email_to: email_addresses) when TopicUser.notification_levels[:tracking] notify_group_summary(user, post) end diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index 21805a12464..c77ef6dd27c 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -643,6 +643,23 @@ en: title: Interaction posting: Posting notification: Notification + email: + title: "Email" + status: "Synchronized %{old_emails} / %{total_emails} emails via IMAP." + credentials: + title: "Credentials" + smtp_server: "SMTP Server" + smtp_port: "SMTP Port" + smtp_ssl: "Use SSL for SMTP" + imap_server: "IMAP Server" + imap_port: "IMAP Port" + imap_ssl: "Use SSL for IMAP" + username: "Username" + password: "Password" + mailboxes: + synchronized: "Synchronized Mailbox" + none_found: "No mailboxes were found in this email account." + disabled: "disabled" membership: title: Membership access: Access diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 8a9c42ff48b..e685a5ec02a 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -1966,6 +1966,17 @@ en: email_in_min_trust: "The minimum trust level a user needs to have to be allowed to post new topics via email." email_in_authserv_id: "The identifier of the service doing authentication checks on incoming emails. See https://meta.discourse.org/t/134358 for instructions on how to configure this." email_in_spam_header: "The email header to detect spam." + + enable_imap: "Enable IMAP for synchronizing group messages." + enable_imap_write: "Enable two-way IMAP synchronization. If disabled, all write-operations on IMAP accounts are disabled." + enable_imap_idle: "Use IMAP IDLE mechanism to wait for new emails." + enable_smtp: "Enable SMTP for sending notifications for group messages." + + imap_polling_period_mins: "The period in minutes between checking the IMAP accounts for emails." + imap_polling_old_emails: "The maximum number of old emails (processed) to be updated every time an IMAP box is polled (0 for all)." + imap_polling_new_emails: "The maximum number of new emails (unprocessed) to be updated every time an IMAP box is polled ." + imap_batch_import_email: "The minimum number of new emails that trigger import mode (disables post alerts)." + email_prefix: "The [label] used in the subject of emails. It will default to 'title' if not set." email_site_title: "The title of the site used as the sender of emails from the site. Default to 'title' if not set. If your 'title' contains characters that are not allowed in email sender strings, use this setting." diff --git a/config/routes.rb b/config/routes.rb index f34b9123d42..a64c15e82b1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -572,6 +572,7 @@ Discourse::Application.routes.draw do manage/members manage/membership manage/interaction + manage/email manage/logs }.each do |path| get path => 'groups#show' diff --git a/config/site_settings.yml b/config/site_settings.yml index 57a2c509644..5b6938ee9f4 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -1011,6 +1011,14 @@ email: - X-Spam-Flag - X-Spam-Status - X-SES-Spam-Verdict + enable_imap: false + enable_imap_write: false + enable_imap_idle: false + enable_smtp: false + imap_polling_period_mins: 5 + imap_polling_old_emails: 1000 + imap_polling_new_emails: 250 + imap_batch_import_email: 100 email_prefix: "" email_site_title: "" disable_emails: diff --git a/config/unicorn.conf.rb b/config/unicorn.conf.rb index 1d1d0135abb..3a6cf236246 100644 --- a/config/unicorn.conf.rb +++ b/config/unicorn.conf.rb @@ -101,77 +101,131 @@ before_fork do |server, worker| Demon::Sidekiq.kill("USR1") old_handler.call end + end - class ::Unicorn::HttpServer - alias :master_sleep_orig :master_sleep + puts "Starting up email sync" + Demon::EmailSync.start + Signal.trap("SIGTSTP") do + STDERR.puts "#{Time.now}: Issuing stop to email_sync" + Demon::EmailSync.stop + end - def max_rss - rss = `ps -eo rss,args | grep sidekiq | grep -v grep | awk '{print $1}'` - .split("\n") - .map(&:to_i) - .max + class ::Unicorn::HttpServer + alias :master_sleep_orig :master_sleep - rss ||= 0 + def max_sidekiq_rss + rss = `ps -eo rss,args | grep sidekiq | grep -v grep | awk '{print $1}'` + .split("\n") + .map(&:to_i) + .max - rss * 1024 - end + rss ||= 0 - def max_allowed_size - [ENV['UNICORN_SIDEKIQ_MAX_RSS'].to_i, 500].max.megabytes - end + rss * 1024 + end - def out_of_memory? - max_rss > max_allowed_size - end + def max_allowed_sidekiq_rss + [ENV['UNICORN_SIDEKIQ_MAX_RSS'].to_i, 500].max.megabytes + end - def force_kill_rogue_sidekiq - info = `ps -eo pid,rss,args | grep sidekiq | grep -v grep | awk '{print $1,$2}'` - info.split("\n").each do |row| - pid, mem = row.split(" ").map(&:to_i) - if pid > 0 && (mem * 1024) > max_allowed_size - Rails.logger.warn "Detected rogue Sidekiq pid #{pid} mem #{mem * 1024}, killing" - Process.kill("KILL", pid) rescue nil - end + def force_kill_rogue_sidekiq + info = `ps -eo pid,rss,args | grep sidekiq | grep -v grep | awk '{print $1,$2}'` + info.split("\n").each do |row| + pid, mem = row.split(" ").map(&:to_i) + if pid > 0 && (mem * 1024) > max_allowed_sidekiq_rss + Rails.logger.warn "Detected rogue Sidekiq pid #{pid} mem #{mem * 1024}, killing" + Process.kill("KILL", pid) rescue nil end end + end - def check_sidekiq_heartbeat - @sidekiq_heartbeat_interval ||= 30.minutes - @sidekiq_next_heartbeat_check ||= Time.now.to_i + @sidekiq_heartbeat_interval + def check_sidekiq_heartbeat + @sidekiq_heartbeat_interval ||= 30.minutes + @sidekiq_next_heartbeat_check ||= Time.now.to_i + @sidekiq_heartbeat_interval - if @sidekiq_next_heartbeat_check < Time.now.to_i + if @sidekiq_next_heartbeat_check < Time.now.to_i - last_heartbeat = Jobs::RunHeartbeat.last_heartbeat - restart = false + last_heartbeat = Jobs::RunHeartbeat.last_heartbeat + restart = false - if out_of_memory? - Rails.logger.warn("Sidekiq is consuming too much memory (using: %0.2fM) for '%s', restarting" % [(max_rss.to_f / 1.megabyte), ENV["DISCOURSE_HOSTNAME"]]) - restart = true - end - - if last_heartbeat < Time.now.to_i - @sidekiq_heartbeat_interval - STDERR.puts "Sidekiq heartbeat test failed, restarting" - Rails.logger.warn "Sidekiq heartbeat test failed, restarting" - - restart = true - end - @sidekiq_next_heartbeat_check = Time.now.to_i + @sidekiq_heartbeat_interval - - if restart - Demon::Sidekiq.restart - sleep 10 - force_kill_rogue_sidekiq - end - Discourse.redis.close + sidekiq_rss = max_sidekiq_rss + if sidekiq_rss > max_allowed_sidekiq_rss + Rails.logger.warn("Sidekiq is consuming too much memory (using: %0.2fM) for '%s', restarting" % [(sidekiq_rss.to_f / 1.megabyte), ENV["DISCOURSE_HOSTNAME"]]) + restart = true end + + if last_heartbeat < Time.now.to_i - @sidekiq_heartbeat_interval + STDERR.puts "Sidekiq heartbeat test failed, restarting" + Rails.logger.warn "Sidekiq heartbeat test failed, restarting" + + restart = true + end + @sidekiq_next_heartbeat_check = Time.now.to_i + @sidekiq_heartbeat_interval + + if restart + Demon::Sidekiq.restart + sleep 10 + force_kill_rogue_sidekiq + end + Discourse.redis.close + end + end + + def max_email_sync_rss + return 0 if Demon::EmailSync.demons.empty? + + email_sync_pids = Demon::EmailSync.demons.map { |uid, demon| demon.pid } + return 0 if email_sync_pids.empty? + + rss = `ps -eo pid,rss,args | grep '#{email_sync_pids.join('|')}' | grep -v grep | awk '{print $2}'` + .split("\n") + .map(&:to_i) + .max + + (rss || 0) * 1024 + end + + def max_allowed_email_sync_rss + [ENV['UNICORN_EMAIL_SYNC_MAX_RSS'].to_i, 500].max.megabytes + end + + def check_email_sync_heartbeat + # Skip first check to let process warm up + @email_sync_next_heartbeat_check ||= (Time.now + Demon::EmailSync::HEARTBEAT_INTERVAL).to_i + + return if @email_sync_next_heartbeat_check > Time.now.to_i + @email_sync_next_heartbeat_check = (Time.now + Demon::EmailSync::HEARTBEAT_INTERVAL).to_i + + restart = false + + # Restart process if it does not respond anymore + last_heartbeat_ago = Time.now.to_i - Discourse.redis.get(Demon::EmailSync::HEARTBEAT_KEY).to_i + if last_heartbeat_ago > Demon::EmailSync::HEARTBEAT_INTERVAL.to_i + STDERR.puts("EmailSync heartbeat test failed (last heartbeat was #{last_heartbeat_ago}s ago), restarting") + restart = true end - def master_sleep(sec) + # Restart process if memory usage is too high + email_sync_rss = max_email_sync_rss + if email_sync_rss > max_allowed_email_sync_rss + STDERR.puts("EmailSync is consuming too much memory (using: %0.2fM) for '%s', restarting" % [(email_sync_rss.to_f / 1.megabyte), ENV["DISCOURSE_HOSTNAME"]]) + restart = true + end + + Demon::EmailSync.restart if restart + end + + def master_sleep(sec) + sidekiqs = ENV['UNICORN_SIDEKIQS'].to_i + if sidekiqs > 0 Demon::Sidekiq.ensure_running check_sidekiq_heartbeat - - master_sleep_orig(sec) end + + Demon::EmailSync.ensure_running + check_email_sync_heartbeat + + master_sleep_orig(sec) end end diff --git a/db/migrate/20181220115844_add_smtp_and_imap_to_groups.rb b/db/migrate/20181220115844_add_smtp_and_imap_to_groups.rb new file mode 100644 index 00000000000..42f79a4b52e --- /dev/null +++ b/db/migrate/20181220115844_add_smtp_and_imap_to_groups.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddSmtpAndImapToGroups < ActiveRecord::Migration[5.2] + def change + add_column :groups, :smtp_server, :string + add_column :groups, :smtp_port, :integer + add_column :groups, :smtp_ssl, :boolean + + add_column :groups, :imap_server, :string + add_column :groups, :imap_port, :integer + add_column :groups, :imap_ssl, :boolean + + add_column :groups, :imap_mailbox_name, :string, default: '', null: false + add_column :groups, :imap_uid_validity, :integer, default: 0, null: false + add_column :groups, :imap_last_uid, :integer, default: 0, null: false + + add_column :groups, :email_username, :string + add_column :groups, :email_password, :string + end +end diff --git a/db/migrate/20190111183409_add_imap_fields_to_incoming_emails.rb b/db/migrate/20190111183409_add_imap_fields_to_incoming_emails.rb new file mode 100644 index 00000000000..682a8f8e89f --- /dev/null +++ b/db/migrate/20190111183409_add_imap_fields_to_incoming_emails.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddImapFieldsToIncomingEmails < ActiveRecord::Migration[5.2] + def change + add_column :incoming_emails, :imap_uid_validity, :integer + add_column :incoming_emails, :imap_uid, :integer + add_column :incoming_emails, :imap_sync, :boolean + + add_index :incoming_emails, :imap_sync + end +end diff --git a/db/migrate/20200327164420_add_imap_stats_to_group.rb b/db/migrate/20200327164420_add_imap_stats_to_group.rb new file mode 100644 index 00000000000..cc2e4b94843 --- /dev/null +++ b/db/migrate/20200327164420_add_imap_stats_to_group.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddImapStatsToGroup < ActiveRecord::Migration[6.0] + def change + add_column :groups, :imap_last_error, :text + add_column :groups, :imap_old_emails, :integer + add_column :groups, :imap_new_emails, :integer + end +end diff --git a/lib/demon/email_sync.rb b/lib/demon/email_sync.rb new file mode 100644 index 00000000000..015dabaa872 --- /dev/null +++ b/lib/demon/email_sync.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +require "demon/base" + +class Demon::EmailSync < ::Demon::Base + HEARTBEAT_KEY ||= "email_sync_heartbeat" + HEARTBEAT_INTERVAL ||= 60.seconds + + def self.prefix + "email_sync" + end + + private + + def suppress_stdout + false + end + + def suppress_stderr + false + end + + def start_thread(db, group) + Thread.new do + RailsMultisite::ConnectionManagement.with_connection(db) do + begin + obj = Imap::Sync.for_group(group) + rescue Net::IMAP::NoResponseError => e + group.update(imap_last_error: e.message) + Thread.exit + end + + @sync_lock.synchronize { @sync_data[db][group.id][:obj] = obj } + + status = nil + idle = false + + while @running && group.reload.imap_mailbox_name.present? do + status = obj.process( + idle: obj.can_idle? && status && status[:remaining] == 0, + old_emails_limit: status && status[:remaining] > 0 ? 0 : nil, + ) + + if !obj.can_idle? && status[:remaining] == 0 + # Thread goes into sleep for a bit so it is better to return any + # connection back to the pool. + ActiveRecord::Base.connection_handler.clear_active_connections! + + sleep SiteSetting.imap_polling_period_mins.minutes + end + end + + obj.disconnect! + end + end + end + + def kill_threads + # This is not really safe so the caller should ensure it happens in a + # thread-safe context. + # It should be safe when called from within a `trap` (there are no + # synchronization primitives available anyway). + @running = false + + @sync_data.each do |db, sync_data| + sync_data.each do |_, data| + data[:thread].kill + data[:thread].join + data[:obj]&.disconnect! rescue nil + end + end + + exit 0 + end + + def after_fork + puts "Loading EmailSync in process id #{Process.pid}" + + loop do + break if Discourse.redis.set(HEARTBEAT_KEY, Time.now.to_i, ex: HEARTBEAT_INTERVAL, nx: true) + sleep HEARTBEAT_INTERVAL + end + + puts "Starting EmailSync main thread" + + @running = true + @sync_data = {} + @sync_lock = Mutex.new + + trap('INT') { kill_threads } + trap('TERM') { kill_threads } + trap('HUP') { kill_threads } + + while @running + Discourse.redis.set(HEARTBEAT_KEY, Time.now.to_i, ex: HEARTBEAT_INTERVAL) + + # Kill all threads for databases that no longer exist + all_dbs = Set.new(RailsMultisite::ConnectionManagement.all_dbs) + @sync_data.filter! do |db, sync_data| + next true if all_dbs.include?(db) + + sync_data.each do |_, data| + data[:thread].kill + data[:thread].join + data[:obj]&.disconnect! + end + + false + end + + RailsMultisite::ConnectionManagement.each_connection do |db| + if SiteSetting.enable_imap + groups = Group.where.not(imap_mailbox_name: '').map { |group| [group.id, group] }.to_h + + @sync_lock.synchronize do + @sync_data[db] ||= {} + + # Kill threads for group's mailbox that are no longer synchronized. + @sync_data[db].filter! do |group_id, data| + next true if groups[group_id] && data[:thread]&.alive? && !data[:obj]&.disconnected? + + if !groups[group_id] + puts("[EmailSync] Killing thread for group (id = #{group_id}) because mailbox is no longer synced") + else + puts("[EmailSync] Thread for group #{groups[group_id].name} is dead") + end + + data[:thread].kill + data[:thread].join + data[:obj]&.disconnect! + + false + end + + # Spawn new threads for groups that are now synchronized. + groups.each do |group_id, group| + if !@sync_data[db][group_id] + puts("[EmailSync] Starting thread for group #{group.name} and mailbox #{group.imap_mailbox_name}") + @sync_data[db][group_id] = { thread: start_thread(db, group), obj: nil } + end + end + end + end + end + + # Thread goes into sleep for a bit so it is better to return any + # connection back to the pool. + ActiveRecord::Base.connection_handler.clear_active_connections! + + sleep 5 + end + + @sync_lock.synchronize { kill_threads } + Discourse.redis.del(HEARTBEAT_KEY) + exit 0 + rescue => e + STDERR.puts e.message + STDERR.puts e.backtrace.join("\n") + exit 1 + end +end diff --git a/lib/email/message_builder.rb b/lib/email/message_builder.rb index 611cdca115d..efb0abf01f3 100644 --- a/lib/email/message_builder.rb +++ b/lib/email/message_builder.rb @@ -127,13 +127,17 @@ module Email end def build_args - { + args = { to: @to, subject: subject, body: body, charset: 'UTF-8', from: from_value } + + args[:delivery_method_options] = @opts[:delivery_method_options] if @opts[:delivery_method_options] + + args end def header_args diff --git a/lib/email/processor.rb b/lib/email/processor.rb index e9b8857234f..6fd8602a89a 100644 --- a/lib/email/processor.rb +++ b/lib/email/processor.rb @@ -5,21 +5,21 @@ module Email class Processor attr_reader :receiver - def initialize(mail, retry_on_rate_limit = true) + def initialize(mail, opts = {}) @mail = mail - @retry_on_rate_limit = retry_on_rate_limit + @opts = opts end - def self.process!(mail, retry_on_rate_limit = true) - Email::Processor.new(mail, retry_on_rate_limit).process! + def self.process!(mail, opts = {}) + Email::Processor.new(mail, opts).process! end def process! begin - @receiver = Email::Receiver.new(@mail) + @receiver = Email::Receiver.new(@mail, @opts) @receiver.process! rescue RateLimiter::LimitExceeded - @retry_on_rate_limit ? Jobs.enqueue(:process_email, mail: @mail) : raise + @opts[:retry_on_rate_limit] ? Jobs.enqueue(:process_email, mail: @mail) : raise rescue => e return handle_bounce(e) if @receiver.is_bounce? diff --git a/lib/email/receiver.rb b/lib/email/receiver.rb index 60eacb16ee5..78e0dcb3c87 100644 --- a/lib/email/receiver.rb +++ b/lib/email/receiver.rb @@ -58,6 +58,7 @@ module Email @mail = Mail.new(@raw_email) @message_id = @mail.message_id.presence || Digest::MD5.hexdigest(mail_string) @opts = opts + @destinations ||= opts[:destinations] end def process! @@ -65,14 +66,19 @@ module Email id_hash = Digest::SHA1.hexdigest(@message_id) DistributedMutex.synchronize("process_email_#{id_hash}") do begin - return if IncomingEmail.exists?(message_id: @message_id) + @incoming_email = IncomingEmail.find_by(message_id: @message_id) + if @incoming_email + @incoming_email.update(imap_uid_validity: @opts[:uid_validity], imap_uid: @opts[:uid], imap_sync: false) + return + end ensure_valid_address_lists ensure_valid_date @from_email, @from_display_name = parse_from_field @from_user = User.find_by_email(@from_email) @incoming_email = create_incoming_email - process_internal + post = process_internal raise BouncedEmailError if is_bounce? + return post rescue Exception => e error = e.to_s error = e.class.name if error.blank? @@ -112,6 +118,9 @@ module Email from_address: @from_email, to_addresses: @mail.to&.map(&:downcase)&.join(";"), cc_addresses: @mail.cc&.map(&:downcase)&.join(";"), + imap_uid_validity: @opts[:uid_validity], + imap_uid: @opts[:uid], + imap_sync: false ) end @@ -137,7 +146,7 @@ module Email if is_auto_generated? && !sent_to_mailinglist_mirror? @incoming_email.update_columns(is_auto_generated: true) - if SiteSetting.block_auto_generated_emails? && !is_bounce? + if SiteSetting.block_auto_generated_emails? && !is_bounce? && !@opts[:allow_auto_generated] raise AutoGeneratedEmailError end end @@ -566,7 +575,7 @@ module Email def subject @subject ||= if mail_subject = @mail.subject - mail_subject.delete("\u0000") + mail_subject.delete("\u0000")[0..254] else I18n.t("emails.incoming.default_subject", email: @from_email) end @@ -621,10 +630,7 @@ module Email def sent_to_mailinglist_mirror? @sent_to_mailinglist_mirror ||= begin destinations.each do |destination| - next unless destination[:type] == :category - - category = destination[:obj] - return true if category.mailinglist_mirror? + return true if destination.is_a?(Category) && destination.mailinglist_mirror? end false @@ -635,10 +641,10 @@ module Email # only check for a group/category when 'email_in' is enabled if SiteSetting.email_in group = Group.find_by_email(address) - return { type: :group, obj: group } if group + return group if group category = Category.find_by_email(address) - return { type: :category, obj: category } if category + return category if category end # reply @@ -647,7 +653,7 @@ module Email match.captures.each do |c| next if c.blank? post_reply_key = PostReplyKey.find_by(reply_key: c) - return { type: :reply, obj: post_reply_key } if post_reply_key + return post_reply_key if post_reply_key end end nil @@ -658,19 +664,13 @@ module Email has_been_forwarded? && process_forwarded_email(destination, user) - return if is_bounce? && destination[:type] != :reply + return if is_bounce? && !destination.is_a?(PostReplyKey) - case destination[:type] - when :group + if destination.is_a?(Group) user ||= stage_from_user - - group = destination[:obj] - create_group_post(group, user, body, elided) - - when :category - category = destination[:obj] - - raise StrangersNotAllowedError if (user.nil? || user.staged?) && !category.email_in_allow_strangers + create_group_post(destination, user, body, elided) + elsif destination.is_a?(Category) + raise StrangersNotAllowedError if (user.nil? || user.staged?) && !destination.email_in_allow_strangers user ||= stage_from_user @@ -680,19 +680,18 @@ module Email raw: body, elided: elided, title: subject, - category: category.id, + category: destination.id, skip_validations: user.staged?) - when :reply + elsif destination.is_a?(PostReplyKey) # We don't stage new users for emails to reply addresses, exit if user is nil raise BadDestinationAddress if user.blank? - post_reply_key = destination[:obj] - post = Post.with_deleted.find(post_reply_key.post_id) + post = Post.with_deleted.find(destination.post_id) raise ReplyNotAllowedError if !Guardian.new(user).can_create_post?(post&.topic) - if post_reply_key.user_id != user.id && !forwarded_reply_key?(post_reply_key, user) - raise ReplyUserNotMatchingError, "post_reply_key.user_id => #{post_reply_key.user_id.inspect}, user.id => #{user.id.inspect}" + if destination.user_id != user.id && !forwarded_reply_key?(destination, user) + raise ReplyUserNotMatchingError, "post_reply_key.user_id => #{destination.user_id.inspect}, user.id => #{user.id.inspect}" end create_reply(user: user, @@ -712,11 +711,12 @@ module Email incoming_emails = IncomingEmail .where(message_id: message_ids) .addressed_to_user(user) - .pluck(:post_id, :to_addresses, :cc_addresses) + .pluck(:post_id, :from_address, :to_addresses, :cc_addresses) - incoming_emails.each do |post_id, to_addresses, cc_addresses| - post_ids << post_id if contains_email_address_of_user?(to_addresses, user) || - contains_email_address_of_user?(cc_addresses, user) + incoming_emails.each do |post_id, from_address, to_addresses, cc_addresses| + post_ids << post_id if contains_email_address_of_user?(from_address, user) || + contains_email_address_of_user?(to_addresses, user) || + contains_email_address_of_user?(cc_addresses, user) end if post_ids.any? && post = Post.where(id: post_ids).order(:created_at).last @@ -801,31 +801,27 @@ module Email end def forwarded_email_create_topic(destination: , user: , raw: , title: , date: nil, embedded_user: nil) - case destination[:type] - when :group - group = destination[:obj] + if destination.is_a?(Group) topic_user = embedded_user&.call || user create_topic(user: topic_user, raw: raw, title: title, archetype: Archetype.private_message, target_usernames: [user.username], - target_group_names: [group.name], + target_group_names: [destination.name], is_group_message: true, skip_validations: true, created_at: date) - when :category - category = destination[:obj] - - return false if user.staged? && !category.email_in_allow_strangers + elsif destination.is_a?(Category) + return false if user.staged? && !destination.email_in_allow_strangers return false if !user.has_trust_level?(SiteSetting.email_in_min_trust) topic_user = embedded_user&.call || user create_topic(user: topic_user, raw: raw, title: title, - category: category.id, + category: destination.id, skip_validations: topic_user.staged?, created_at: date) else @@ -854,7 +850,7 @@ module Email # create reply when available if @before_embedded.present? post_type = Post.types[:regular] - post_type = Post.types[:whisper] if post.topic.private_message? && destination[:obj].usernames[user.username] + post_type = Post.types[:whisper] if post.topic.private_message? && destination.usernames[user.username] create_reply(user: user, raw: @before_embedded, @@ -1114,6 +1110,8 @@ module Email end def create_post(options = {}) + options[:import_mode] = @opts[:import_mode] + options[:via_email] = true options[:raw_email] = @raw_email @@ -1185,11 +1183,11 @@ module Email if user && can_invite?(post.topic, user) post.topic.topic_allowed_users.create!(user_id: user.id) TopicUser.auto_notification_for_staging(user.id, post.topic_id, TopicUser.notification_reasons[:auto_watch]) - post.topic.add_small_action(sender, "invited_user", user.username) + post.topic.add_small_action(sender, "invited_user", user.username, import_mode: @opts[:import_mode]) end # cap number of staged users created per email if @staged_users.count > SiteSetting.maximum_staged_users_per_email - post.topic.add_moderator_post(sender, I18n.t("emails.incoming.maximum_staged_user_per_email_reached")) + post.topic.add_moderator_post(sender, I18n.t("emails.incoming.maximum_staged_user_per_email_reached"), import_mode: @opts[:import_mode]) return end end diff --git a/lib/imap/providers/generic.rb b/lib/imap/providers/generic.rb new file mode 100644 index 00000000000..98781ca9073 --- /dev/null +++ b/lib/imap/providers/generic.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +require 'net/imap' + +module Imap + module Providers + class Generic + + def initialize(server, options = {}) + @server = server + @port = options[:port] || 993 + @ssl = options[:ssl] || true + @username = options[:username] + @password = options[:password] + @timeout = options[:timeout] || 10 + end + + def imap + @imap ||= Net::IMAP.new(@server, port: @port, ssl: @ssl, open_timeout: @timeout) + end + + def disconnected? + @imap && @imap.disconnected? + end + + def connect! + imap.login(@username, @password) + end + + def disconnect! + imap.logout rescue nil + imap.disconnect + end + + def can?(capability) + @capabilities ||= imap.responses['CAPABILITY'][-1] || imap.capability + @capabilities.include?(capability) + end + + def uids(opts = {}) + if opts[:from] && opts[:to] + imap.uid_search("UID #{opts[:from]}:#{opts[:to]}") + elsif opts[:from] + imap.uid_search("UID #{opts[:from]}:*") + elsif opts[:to] + imap.uid_search("UID 1:#{opts[:to]}") + else + imap.uid_search('ALL') + end + end + + def labels + @labels ||= begin + labels = {} + + list_mailboxes.each do |name| + if tag = to_tag(name) + labels[tag] = name + end + end + + labels + end + end + + def open_mailbox(mailbox_name, write: false) + if write + raise 'two-way IMAP sync is disabled' if !SiteSetting.enable_imap_write + imap.select(mailbox_name) + else + imap.examine(mailbox_name) + end + + { + uid_validity: imap.responses['UIDVALIDITY'][-1] + } + end + + def emails(uids, fields, opts = {}) + imap.uid_fetch(uids, fields).map do |email| + attributes = {} + + fields.each do |field| + attributes[field] = email.attr[field] + end + + attributes + end + end + + def store(uid, attribute, old_set, new_set) + additions = new_set.reject { |val| old_set.include?(val) } + imap.uid_store(uid, "+#{attribute}", additions) if additions.length > 0 + removals = old_set.reject { |val| new_set.include?(val) } + imap.uid_store(uid, "-#{attribute}", removals) if removals.length > 0 + end + + def to_tag(label) + label = DiscourseTagging.clean_tag(label.to_s) + label if label != 'inbox' && label != 'sent' + end + + def tag_to_flag(tag) + :Seen if tag == 'seen' + end + + def tag_to_label(tag) + labels[tag] + end + + def list_mailboxes + imap.list('', '*').map(&:name) + end + end + end +end diff --git a/lib/imap/providers/gmail.rb b/lib/imap/providers/gmail.rb new file mode 100644 index 00000000000..97db64a53e9 --- /dev/null +++ b/lib/imap/providers/gmail.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +module Imap + module Providers + class Gmail < Generic + X_GM_LABELS = 'X-GM-LABELS' + + def imap + @imap ||= super.tap { |imap| apply_gmail_patch(imap) } + end + + def emails(uids, fields, opts = {}) + fields[fields.index('LABELS')] = X_GM_LABELS + + emails = super(uids, fields, opts) + + emails.each do |email| + email['LABELS'] = Array(email['LABELS']) + + if email[X_GM_LABELS] + email['LABELS'] << Array(email.delete(X_GM_LABELS)) + email['LABELS'].flatten! + end + + email['LABELS'] << '\\Inbox' if opts[:mailbox] == 'INBOX' + + email['LABELS'].uniq! + end + + emails + end + + def store(uid, attribute, old_set, new_set) + attribute = X_GM_LABELS if attribute == 'LABELS' + super(uid, attribute, old_set, new_set) + end + + def to_tag(label) + # Label `\\Starred` is Gmail equivalent of :Flagged (both present) + return 'starred' if label == :Flagged + return if label == '[Gmail]/All Mail' + + label = label.to_s.gsub('[Gmail]/', '') + super(label) + end + + def tag_to_flag(tag) + return :Flagged if tag == 'starred' + + super(tag) + end + + def tag_to_label(tag) + return '\\Important' if tag == 'important' + return '\\Starred' if tag == 'starred' + + super(tag) + end + + private + + def apply_gmail_patch(imap) + class << imap.instance_variable_get('@parser') + + # Modified version of the original `msg_att` from here: + # https://github.com/ruby/ruby/blob/1cc8ff001da217d0e98d13fe61fbc9f5547ef722/lib/net/imap.rb#L2346 + # rubocop:disable Style/RedundantReturn + def msg_att(n) + match(T_LPAR) + attr = {} + while true + token = lookahead + case token.symbol + when T_RPAR + shift_token + break + when T_SPACE + shift_token + next + end + case token.value + when /\A(?:ENVELOPE)\z/ni + name, val = envelope_data + when /\A(?:FLAGS)\z/ni + name, val = flags_data + when /\A(?:INTERNALDATE)\z/ni + name, val = internaldate_data + when /\A(?:RFC822(?:\.HEADER|\.TEXT)?)\z/ni + name, val = rfc822_text + when /\A(?:RFC822\.SIZE)\z/ni + name, val = rfc822_size + when /\A(?:BODY(?:STRUCTURE)?)\z/ni + name, val = body_data + when /\A(?:UID)\z/ni + name, val = uid_data + when /\A(?:MODSEQ)\z/ni + name, val = modseq_data + # Adding support for GMail extended attributes. + when /\A(?:X-GM-LABELS)\z/ni + name, val = label_data + when /\A(?:X-GM-MSGID)\z/ni + name, val = uid_data + when /\A(?:X-GM-THRID)\z/ni + name, val = uid_data + else + parse_error("unknown attribute `%s' for {%d}", token.value, n) + end + attr[name] = val + end + return attr + end + + def label_data + token = match(T_ATOM) + name = token.value.upcase + + match(T_SPACE) + match(T_LPAR) + + result = [] + while true + token = lookahead + case token.symbol + when T_RPAR + shift_token + break + when T_SPACE + shift_token + end + + token = lookahead + if string_token?(token) + result.push(string) + else + result.push(atom) + end + end + return name, result + end + # rubocop:enable Style/RedundantReturn + end + end + end + end +end diff --git a/lib/imap/sync.rb b/lib/imap/sync.rb new file mode 100644 index 00000000000..e1a27e712df --- /dev/null +++ b/lib/imap/sync.rb @@ -0,0 +1,258 @@ +# frozen_string_literal: true + +require 'net/imap' + +module Imap + class Sync + def self.for_group(group, opts = {}) + if group.imap_server == 'imap.gmail.com' + opts[:provider] ||= Imap::Providers::Gmail + end + + Imap::Sync.new(group, opts) + end + + def initialize(group, opts = {}) + @group = group + + provider_klass ||= opts[:provider] || Imap::Providers::Generic + @provider = provider_klass.new(@group.imap_server, + port: @group.imap_port, + ssl: @group.imap_ssl, + username: @group.email_username, + password: @group.email_password + ) + + connect! + end + + def connect! + @provider.connect! + end + + def disconnect! + @provider.disconnect! + end + + def disconnected? + @provider.disconnected? + end + + def can_idle? + SiteSetting.enable_imap_idle && @provider.can?('IDLE') + end + + def process(idle: false, import_limit: nil, old_emails_limit: nil, new_emails_limit: nil) + raise 'disconnected' if disconnected? + + import_limit ||= SiteSetting.imap_batch_import_email + old_emails_limit ||= SiteSetting.imap_polling_old_emails + new_emails_limit ||= SiteSetting.imap_polling_new_emails + + # IMAP server -> Discourse (download): discovers updates to old emails + # (synced emails) and fetches new emails. + + # TODO: Use `Net::IMAP.encode_utf7(@group.imap_mailbox_name)`? + @status = @provider.open_mailbox(@group.imap_mailbox_name) + + if @status[:uid_validity] != @group.imap_uid_validity + # If UID validity changes, the whole mailbox must be synchronized (all + # emails are considered new and will be associated to existent topics + # in Email::Reciever by matching Message-Ids). + Rails.logger.warn("[IMAP] UIDVALIDITY = #{@status[:uid_validity]} does not match expected #{@group.imap_uid_validity}, invalidating IMAP cache and resyncing emails for group #{@group.name} and mailbox #{@group.imap_mailbox_name}") + @group.imap_last_uid = 0 + end + + if idle && !can_idle? + Rails.logger.warn("[IMAP] IMAP server for group #{@group.name} cannot IDLE") + idle = false + end + + if idle + raise 'IMAP IDLE is disabled' if !SiteSetting.enable_imap_idle + + # Thread goes into sleep and it is better to return any connection + # back to the pool. + ActiveRecord::Base.connection_handler.clear_active_connections! + + @provider.imap.idle(SiteSetting.imap_polling_period_mins.minutes.to_i) do |resp| + if resp.kind_of?(Net::IMAP::UntaggedResponse) && resp.name == 'EXISTS' + @provider.imap.idle_done + end + end + end + + # Fetching UIDs of old (already imported into Discourse, but might need + # update) and new (not downloaded yet) emails. + if @group.imap_last_uid == 0 + old_uids = [] + new_uids = @provider.uids + else + old_uids = @provider.uids(to: @group.imap_last_uid) # 1 .. seen + new_uids = @provider.uids(from: @group.imap_last_uid + 1) # seen+1 .. inf + end + + # Sometimes, new_uids contains elements from old_uids. + new_uids = new_uids - old_uids + + Rails.logger.debug("[IMAP] Remote email server has #{old_uids.size} old emails and #{new_uids.size} new emails") + + all_old_uids_size = old_uids.size + all_new_uids_size = new_uids.size + + @group.update_columns( + imap_last_error: nil, + imap_old_emails: all_old_uids_size, + imap_new_emails: all_new_uids_size + ) + + import_mode = import_limit > -1 && new_uids.size > import_limit + old_uids = old_uids.sample(old_emails_limit).sort! if old_emails_limit > -1 + new_uids = new_uids[0..new_emails_limit - 1] if new_emails_limit > 0 + + if old_uids.present? + Rails.logger.debug("[IMAP] Syncing #{old_uids.size} randomly-selected old emails") + emails = @provider.emails(old_uids, ['UID', 'FLAGS', 'LABELS'], mailbox: @group.imap_mailbox_name) + emails.each do |email| + incoming_email = IncomingEmail.find_by( + imap_uid_validity: @status[:uid_validity], + imap_uid: email['UID'] + ) + + if incoming_email.present? + update_topic(email, incoming_email, mailbox_name: @group.imap_mailbox_name) + else + Rails.logger.warn("[IMAP] Could not find old email (UIDVALIDITY = #{@status[:uid_validity]}, UID = #{email['UID']})") + end + end + end + + if new_uids.present? + Rails.logger.debug("[IMAP] Syncing #{new_uids.size} new emails (oldest first)") + + emails = @provider.emails(new_uids, ['UID', 'FLAGS', 'LABELS', 'RFC822'], mailbox: @group.imap_mailbox_name) + processed = 0 + + emails.each do |email| + # Synchronously process emails because the order of emails matter + # (for example replies must be processed after the original email + # to have a topic where the reply can be posted). + begin + receiver = Email::Receiver.new(email['RFC822'], + allow_auto_generated: true, + import_mode: import_mode, + destinations: [@group], + uid_validity: @status[:uid_validity], + uid: email['UID'] + ) + receiver.process! + update_topic(email, receiver.incoming_email, mailbox_name: @group.imap_mailbox_name) + rescue Email::Receiver::ProcessingError => e + Rails.logger.warn("[IMAP] Could not process (UIDVALIDITY = #{@status[:uid_validity]}, UID = #{email['UID']}): #{e.message}") + end + + processed += 1 + @group.update_columns( + imap_uid_validity: @status[:uid_validity], + imap_last_uid: email['UID'], + imap_old_emails: all_old_uids_size + processed, + imap_new_emails: all_new_uids_size - processed + ) + end + end + + # Discourse -> IMAP server (upload): syncs updated flags and labels. + if SiteSetting.enable_imap_write + to_sync = IncomingEmail.where(imap_sync: true) + if to_sync.size > 0 + @provider.open_mailbox(@group.imap_mailbox_name, write: true) + to_sync.each do |incoming_email| + Rails.logger.debug("[IMAP] Updating email for #{@group.name} and incoming email ID = #{incoming_email.id}") + update_email(@group.imap_mailbox_name, incoming_email) + end + end + end + + { remaining: all_new_uids_size - new_uids.size } + end + + def update_topic(email, incoming_email, opts = {}) + return if !incoming_email || + incoming_email.imap_sync || + !incoming_email.topic || + incoming_email.post&.post_number != 1 + + update_topic_archived_state(email, incoming_email, opts) + update_topic_tags(email, incoming_email, opts) + end + + private + + def update_topic_archived_state(email, incoming_email, opts = {}) + topic = incoming_email.topic + + topic_is_archived = topic.group_archived_messages.size > 0 + email_is_archived = !email['LABELS'].include?('\\Inbox') && !email['LABELS'].include?('INBOX') + + if topic_is_archived && !email_is_archived + GroupArchivedMessage.move_to_inbox!(@group.id, topic, skip_imap_sync: true) + elsif !topic_is_archived && email_is_archived + GroupArchivedMessage.archive!(@group.id, topic, skip_imap_sync: true) + end + end + + def update_topic_tags(email, incoming_email, opts = {}) + group_email_regex = @group.email_username_regex + topic = incoming_email.topic + + tags = Set.new + + # "Plus" part from the destination email address + to_addresses = incoming_email.to_addresses&.split(";") || [] + cc_addresses = incoming_email.cc_addresses&.split(";") || [] + (to_addresses + cc_addresses).each do |address| + if plus_part = address&.scan(group_email_regex)&.first&.first + tags.add("plus:#{plus_part[1..-1]}") if plus_part.length > 0 + end + end + + # Mailbox name + tags.add(@provider.to_tag(opts[:mailbox_name])) if opts[:mailbox_name] + + # Flags and labels + email['FLAGS'].each { |flag| tags.add(@provider.to_tag(flag)) } + email['LABELS'].each { |label| tags.add(@provider.to_tag(label)) } + + tags.subtract([nil, '']) + + # TODO: Optimize tagging. + # `DiscourseTagging.tag_topic_by_names` does a lot of lookups in the + # database and some of them could be cached in this context. + DiscourseTagging.tag_topic_by_names(topic, Guardian.new(Discourse.system_user), tags.to_a) + end + + def update_email(mailbox_name, incoming_email) + return if !SiteSetting.tagging_enabled || !SiteSetting.allow_staff_to_tag_pms + return if incoming_email&.post&.post_number != 1 || !incoming_email.imap_sync + return unless email = @provider.emails(incoming_email.imap_uid, ['FLAGS', 'LABELS'], mailbox: mailbox_name).first + incoming_email.update(imap_sync: false) + + labels = email['LABELS'] + flags = email['FLAGS'] + topic = incoming_email.topic + + # TODO: Delete remote email if topic no longer exists + # new_flags << Net::IMAP::DELETED if !incoming_email.topic + return if !topic + + # Sync topic status and labels with email flags and labels. + tags = topic.tags.pluck(:name) + new_flags = tags.map { |tag| @provider.tag_to_flag(tag) }.reject(&:blank?) + # new_flags << Net::IMAP::DELETED if !incoming_email.topic + new_labels = tags.map { |tag| @provider.tag_to_label(tag) }.reject(&:blank?) + new_labels << '\\Inbox' if topic.group_archived_messages.length == 0 + @provider.store(incoming_email.imap_uid, 'FLAGS', flags, new_flags) + @provider.store(incoming_email.imap_uid, 'LABELS', labels, new_labels) + end + end +end diff --git a/lib/post_creator.rb b/lib/post_creator.rb index 6a9e4ca8735..b5b13a2961c 100644 --- a/lib/post_creator.rb +++ b/lib/post_creator.rb @@ -186,10 +186,8 @@ class PostCreator @post.link_post_uploads update_uploads_secure_status ensure_in_allowed_users if guardian.is_staff? - unarchive_message - if !@opts[:import_mode] - DraftSequence.next!(@user, draft_key) - end + unarchive_message if !@opts[:import_mode] + DraftSequence.next!(@user, draft_key) if !@opts[:import_mode] @post.save_reply_relationships end end diff --git a/lib/post_revisor.rb b/lib/post_revisor.rb index d610fccf375..fc4abeeb2c3 100644 --- a/lib/post_revisor.rb +++ b/lib/post_revisor.rb @@ -158,6 +158,10 @@ class PostRevisor @skip_revision = false @skip_revision = @opts[:skip_revision] if @opts.has_key?(:skip_revision) + if @post.incoming_email&.imap_uid + @post.incoming_email&.update(imap_sync: true) + end + old_raw = @post.raw Post.transaction do diff --git a/spec/components/email/processor_spec.rb b/spec/components/email/processor_spec.rb index 606955551a1..8a4a5ea355b 100644 --- a/spec/components/email/processor_spec.rb +++ b/spec/components/email/processor_spec.rb @@ -60,12 +60,12 @@ describe Email::Processor do it "enqueues a background job by default" do Jobs.expects(:enqueue).with(:process_email, mail: mail) - Email::Processor.process!(mail) + Email::Processor.process!(mail, retry_on_rate_limit: true) end it "doesn't enqueue a background job when retry is disabled" do Jobs.expects(:enqueue).with(:process_email, mail: mail).never - expect { Email::Processor.process!(mail, false) }.to raise_error(limit_exceeded) + expect { Email::Processor.process!(mail, retry_on_rate_limit: false) }.to raise_error(limit_exceeded) end end diff --git a/spec/components/email/receiver_spec.rb b/spec/components/email/receiver_spec.rb index 553ce77c9e4..44dae0fd8da 100644 --- a/spec/components/email/receiver_spec.rb +++ b/spec/components/email/receiver_spec.rb @@ -1153,9 +1153,7 @@ describe Email::Receiver do dest = Email::Receiver.check_address('foo+4f97315cc828096c9cb34c6f1a0d6fe8@bar.com') - expect(dest).to be_present - expect(dest[:type]).to eq(:reply) - expect(dest[:obj]).to eq(post_reply_key) + expect(dest).to eq(post_reply_key) end end end @@ -1587,4 +1585,78 @@ describe Email::Receiver do expect { Email::Receiver.new(email).process! }.to raise_error(Email::Receiver::ReplyToDigestError) end end + + context "find_related_post" do + + let(:user) { Fabricate(:user) } + let(:group) { Fabricate(:group, users: [user]) } + + let (:email_1) { <<~EOF + MIME-Version: 1.0 + Date: Wed, 01 Jan 2019 12:00:00 +0200 + Message-ID: <7aN1uwcokt2xkfG3iYrpKmiuVhy4w9b5@mail.gmail.com> + Subject: Lorem ipsum dolor sit amet + From: Dan Ungureanu + To: team-test@discourse.org + Content-Type: text/plain; charset="UTF-8" + + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas + semper, erat tempor sodales commodo, mi diam tempus lorem, in vehicula + leo libero quis lacus. Nullam justo nunc, sagittis nec metus placerat, + auctor condimentum neque. Sed risus purus, fermentum eget purus + porttitor, finibus efficitur orci. Integer tempus mi nec odio + elementum pulvinar. Pellentesque sed fringilla nulla, ac mollis quam. + Vivamus semper lacinia scelerisque. Cras urna magna, porttitor nec + libero quis, congue viverra sapien. Nulla sodales ac tellus a + suscipit. + EOF + } + + let (:post_2) { + incoming_email = IncomingEmail.find_by(message_id: "7aN1uwcokt2xkfG3iYrpKmiuVhy4w9b5@mail.gmail.com") + + PostCreator.create(user, + raw: "Vestibulum rutrum tortor vitae arcu varius, non vestibulum ipsum tempor. Integer nibh libero, dignissim eu velit vel, interdum posuere mi. Aliquam erat volutpat. Pellentesque id nulla ultricies, eleifend ipsum non, fringilla purus. Aliquam pretium dolor lobortis urna volutpat, vel consectetur arcu porta. In non erat quis nibh gravida pharetra consequat vel risus. Aliquam rutrum consectetur est ac posuere. Praesent mattis nunc risus, a molestie lectus accumsan porta.", + topic_id: incoming_email.topic_id + ) + } + + let (:email_3) { <<~EOF + MIME-Version: 1.0 + Date: Wed, 01 Jan 2019 12:00:00 +0200 + References: <7aN1uwcokt2xkfG3iYrpKmiuVhy4w9b5@mail.gmail.com> + In-Reply-To: + Message-ID: + Subject: Re: Lorem ipsum dolor sit amet + From: Dan Ungureanu + To: team-test@discourse.org + Content-Type: text/plain; charset="UTF-8" + + Integer mattis suscipit facilisis. Ut ullamcorper libero at faucibus + sodales. Ut suscipit elit ac dui porta consequat. Suspendisse potenti. + Nam ut accumsan dui, eget commodo sapien. Etiam ultrices elementum + cursus. Vivamus et diam et orci lobortis porttitor. Aliquam + scelerisque ex a imperdiet ornare. Donec interdum laoreet posuere. + Nulla sagittis, velit id posuere sollicitudin, elit nunc laoreet + libero, vitae aliquet tortor eros at est. Donec vitae massa vehicula, + aliquet libero non, porttitor ipsum. Phasellus pellentesque sodales + lacus eu sagittis. Aliquam ut condimentum nisi. Nulla in placerat + felis. Sed pellentesque, massa auctor venenatis gravida, risus lorem + iaculis mi, at hendrerit nisi turpis sit amet metus. Nulla egestas + ante eget nisi luctus consectetur. + EOF + } + + def receive(email_string) + Email::Receiver.new(email_string, + destinations: [group] + ).process! + end + + it "makes all posts in same topic" do + expect { receive(email_1) }.to change { Topic.count }.by(1).and change { Post.where(post_type: Post.types[:regular]).count }.by(1) + expect { post_2 }.to change { Topic.count }.by(0).and change { Post.where(post_type: Post.types[:regular]).count }.by(1) + expect { receive(email_3) }.to change { Topic.count }.by(0).and change { Post.where(post_type: Post.types[:regular]).count }.by(1) + end + end end diff --git a/spec/components/imap/imap_helper.rb b/spec/components/imap/imap_helper.rb new file mode 100644 index 00000000000..99bcc412240 --- /dev/null +++ b/spec/components/imap/imap_helper.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class MockedImapProvider < Imap::Providers::Gmail + def connect!; end + def disconnect!; end + def open_mailbox(mailbox_name, write: false); end + + def labels + ['INBOX'] + end +end + +def EmailFabricator(options) + email = +'' + email += "Date: Sat, 31 Mar 2018 17:50:19 -0700\n" + email += "From: #{options[:from] || "Dan "}\n" + email += "To: #{options[:to] || "Joffrey "}\n" + email += "Cc: #{options[:cc]}\n" if options[:cc] + email += "In-Reply-To: #{options[:in_reply_to]}\n" if options[:in_reply_to] + email += "References: #{options[:in_reply_to]}\n" if options[:in_reply_to] + email += "Message-ID: #{options[:message_id]}\n" if options[:message_id] + email += "Subject: #{options[:subject] || "This is a test email subhect"}\n" + email += "Mime-Version: 1.0\n" + email += "Content-Type: #{options[:content_type] || "text/plain;\n charset=UTF-8"}\n" + email += "Content-Transfer-Encoding: 7bit\n" + email += "\n#{options[:body] || "This is an email *body*. :smile:"}" + email +end diff --git a/spec/components/imap/sync_spec.rb b/spec/components/imap/sync_spec.rb new file mode 100644 index 00000000000..152b5db24d9 --- /dev/null +++ b/spec/components/imap/sync_spec.rb @@ -0,0 +1,333 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'imap/sync' +require_relative 'imap_helper' + +describe Imap::Sync do + + before do + SiteSetting.tagging_enabled = true + SiteSetting.allow_staff_to_tag_pms = true + + SiteSetting.enable_imap = true + + Jobs.run_immediately! + end + + let(:group) do + Fabricate( + :group, + imap_server: 'imap.gmail.com', + imap_port: 993, + email_username: 'discourse@example.com', + email_password: 'discourse@example.com', + imap_mailbox_name: '[Gmail]/All Mail' + ) + end + + let(:sync_handler) { Imap::Sync.new(group, provider: MockedImapProvider) } + + context 'no previous sync' do + let(:from) { 'john@free.fr' } + let(:subject) { 'Testing email post' } + let(:message_id) { "#{SecureRandom.hex}@example.com" } + + let(:email) do + EmailFabricator( + from: from, + to: group.email_username, + subject: subject, + message_id: message_id) + end + + before do + provider = MockedImapProvider.any_instance + provider.stubs(:open_mailbox).returns(uid_validity: 1) + provider.stubs(:uids).with.returns([100]) + provider.stubs(:uids).with(to: 100).returns([100]) + provider.stubs(:uids).with(from: 101).returns([]) + provider.stubs(:emails).returns( + [ + { + 'UID' => 100, + 'LABELS' => %w[\\Important test-label], + 'FLAGS' => %i[Seen], + 'RFC822' => email + } + ] + ) + end + + it 'creates a topic from an incoming email' do + expect { sync_handler.process } + .to change { Topic.count }.by(1) + .and change { Post.where(post_type: Post.types[:regular]).count }.by(1) + .and change { IncomingEmail.count }.by(1) + + expect(group.imap_uid_validity).to eq(1) + expect(group.imap_last_uid).to eq(100) + + topic = Topic.last + expect(topic.title).to eq(subject) + expect(topic.user.email).to eq(from) + expect(topic.tags.pluck(:name)).to eq(%w[seen important test-label]) + + post = topic.first_post + expect(post.raw).to eq('This is an email *body*. :smile:') + + incoming_email = post.incoming_email + expect(incoming_email.raw.lines.map(&:strip)).to eq(email.lines.map(&:strip)) + expect(incoming_email.message_id).to eq(message_id) + expect(incoming_email.from_address).to eq(from) + expect(incoming_email.to_addresses).to eq(group.email_username) + expect(incoming_email.imap_uid_validity).to eq(1) + expect(incoming_email.imap_uid).to eq(100) + expect(incoming_email.imap_sync).to eq(false) + end + + it 'does not duplicate topics' do + expect { sync_handler.process } + .to change { Topic.count }.by(1) + .and change { Post.where(post_type: Post.types[:regular]).count }.by(1) + .and change { IncomingEmail.count }.by(1) + + expect { sync_handler.process } + .to change { Topic.count }.by(0) + .and change { Post.where(post_type: Post.types[:regular]).count }.by(0) + .and change { IncomingEmail.count }.by(0) + end + + it 'does not duplicate incoming emails' do + incoming_email = Fabricate(:incoming_email, message_id: message_id) + + expect { sync_handler.process } + .to change { Topic.count }.by(0) + .and change { Post.where(post_type: Post.types[:regular]).count }.by(0) + .and change { IncomingEmail.count }.by(0) + + incoming_email.reload + expect(incoming_email.message_id).to eq(message_id) + expect(incoming_email.imap_uid_validity).to eq(1) + expect(incoming_email.imap_uid).to eq(100) + expect(incoming_email.imap_sync).to eq(false) + end + end + + context 'previous sync' do + let(:subject) { 'Testing email post' } + + let(:first_from) { 'john@free.fr' } + let(:first_message_id) { SecureRandom.hex } + let(:first_body) { 'This is the first message of this exchange.' } + + let(:second_from) { 'sam@free.fr' } + let(:second_message_id) { SecureRandom.hex } + let(:second_body) { '

This is an answer to this message.

' } + + it 'continues with new emails' do + provider = MockedImapProvider.any_instance + provider.stubs(:open_mailbox).returns(uid_validity: 1) + + provider.stubs(:uids).with.returns([100]) + provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns( + [ + { + 'UID' => 100, + 'LABELS' => %w[\\Inbox], + 'FLAGS' => %i[Seen], + 'RFC822' => EmailFabricator( + message_id: first_message_id, + from: first_from, + to: group.email_username, + cc: second_from, + subject: subject, + body: first_body + ) + } + ] + ) + + expect { sync_handler.process } + .to change { Topic.count }.by(1) + .and change { Post.where(post_type: Post.types[:regular]).count }.by(1) + .and change { IncomingEmail.count }.by(1) + + topic = Topic.last + expect(topic.title).to eq(subject) + expect(GroupArchivedMessage.where(topic_id: topic.id).exists?).to eq(false) + + post = Post.where(post_type: Post.types[:regular]).last + expect(post.user.email).to eq(first_from) + expect(post.raw).to eq(first_body) + expect(group.imap_uid_validity).to eq(1) + expect(group.imap_last_uid).to eq(100) + + provider.stubs(:uids).with(to: 100).returns([100]) + provider.stubs(:uids).with(from: 101).returns([200]) + provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS'], anything).returns( + [ + { + 'UID' => 100, + 'LABELS' => %w[\\Inbox], + 'FLAGS' => %i[Seen] + } + ] + ) + provider.stubs(:emails).with([200], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns( + [ + { + 'UID' => 200, + 'LABELS' => %w[\\Inbox], + 'FLAGS' => %i[Recent], + 'RFC822' => EmailFabricator( + message_id: SecureRandom.hex, + in_reply_to: first_message_id, + from: second_from, + to: group.email_username, + subject: "Re: #{subject}", + body: second_body + ) + } + ] + ) + + expect { sync_handler.process } + .to change { Topic.count }.by(0) + .and change { Post.where(post_type: Post.types[:regular]).count }.by(1) + .and change { IncomingEmail.count }.by(1) + + post = Post.where(post_type: Post.types[:regular]).last + expect(post.user.email).to eq(second_from) + expect(post.raw).to eq(second_body) + expect(group.imap_uid_validity).to eq(1) + expect(group.imap_last_uid).to eq(200) + + provider.stubs(:uids).with(to: 200).returns([100, 200]) + provider.stubs(:uids).with(from: 201).returns([]) + provider.stubs(:emails).with([100, 200], ['UID', 'FLAGS', 'LABELS'], anything).returns( + [ + { + 'UID' => 100, + 'LABELS' => %w[], + 'FLAGS' => %i[Seen] + }, + { + 'UID' => 200, + 'LABELS' => %w[], + 'FLAGS' => %i[Recent], + } + ] + ) + + expect { sync_handler.process } + .to change { Topic.count }.by(0) + .and change { Post.where(post_type: Post.types[:regular]).count }.by(0) + .and change { IncomingEmail.count }.by(0) + + topic = Topic.last + expect(topic.title).to eq(subject) + expect(GroupArchivedMessage.where(topic_id: topic.id).exists?).to eq(true) + + expect(Topic.last.posts.where(post_type: Post.types[:regular]).count).to eq(2) + end + end + + context 'invaidated previous sync' do + let(:subject) { 'Testing email post' } + + let(:first_from) { 'john@free.fr' } + let(:first_message_id) { SecureRandom.hex } + let(:first_body) { 'This is the first message of this exchange.' } + + let(:second_from) { 'sam@free.fr' } + let(:second_message_id) { SecureRandom.hex } + let(:second_body) { '

This is an answer to this message.

' } + + it 'is updated' do + provider = MockedImapProvider.any_instance + + provider.stubs(:open_mailbox).returns(uid_validity: 1) + provider.stubs(:uids).with.returns([100, 200]) + provider.stubs(:emails).with([100, 200], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns( + [ + { + 'UID' => 100, + 'LABELS' => %w[\\Inbox], + 'FLAGS' => %i[Seen], + 'RFC822' => EmailFabricator( + message_id: first_message_id, + from: first_from, + to: group.email_username, + cc: second_from, + subject: subject, + body: first_body + ) + }, + { + 'UID' => 200, + 'LABELS' => %w[\\Inbox], + 'FLAGS' => %i[Recent], + 'RFC822' => EmailFabricator( + message_id: second_message_id, + in_reply_to: first_message_id, + from: second_from, + to: group.email_username, + subject: "Re: #{subject}", + body: second_body + ) + } + ] + ) + + expect { sync_handler.process } + .to change { Topic.count }.by(1) + .and change { Post.where(post_type: Post.types[:regular]).count }.by(2) + .and change { IncomingEmail.count }.by(2) + + imap_data = Topic.last.incoming_email.pluck(:imap_uid_validity, :imap_uid) + expect(imap_data).to contain_exactly([1, 100], [1, 200]) + + provider.stubs(:open_mailbox).returns(uid_validity: 2) + provider.stubs(:uids).with.returns([111, 222]) + provider.stubs(:emails).with([111, 222], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns( + [ + { + 'UID' => 111, + 'LABELS' => %w[\\Inbox], + 'FLAGS' => %i[Seen], + 'RFC822' => EmailFabricator( + message_id: first_message_id, + from: first_from, + to: group.email_username, + cc: second_from, + subject: subject, + body: first_body + ) + }, + { + 'UID' => 222, + 'LABELS' => %w[\\Inbox], + 'FLAGS' => %i[Recent], + 'RFC822' => EmailFabricator( + message_id: second_message_id, + in_reply_to: first_message_id, + from: second_from, + to: group.email_username, + subject: "Re: #{subject}", + body: second_body + ) + } + ] + ) + + expect { sync_handler.process } + .to change { Topic.count }.by(0) + .and change { Post.where(post_type: Post.types[:regular]).count }.by(0) + .and change { IncomingEmail.count }.by(0) + + imap_data = Topic.last.incoming_email.pluck(:imap_uid_validity, :imap_uid) + expect(imap_data).to contain_exactly([2, 111], [2, 222]) + end + end +end diff --git a/spec/jobs/process_email_spec.rb b/spec/jobs/process_email_spec.rb index bd828f7f794..54cd12ef703 100644 --- a/spec/jobs/process_email_spec.rb +++ b/spec/jobs/process_email_spec.rb @@ -7,7 +7,7 @@ describe Jobs::ProcessEmail do let(:mail) { "From: foo@bar.com\nTo: bar@foo.com\nSubject: FOO BAR\n\nFoo foo bar bar?" } it "process an email without retry" do - Email::Processor.expects(:process!).with(mail, false) + Email::Processor.expects(:process!).with(mail, retry_on_rate_limit: false) Jobs::ProcessEmail.new.execute(mail: mail) end diff --git a/spec/mailers/group_smtp_mailer_spec.rb b/spec/mailers/group_smtp_mailer_spec.rb new file mode 100644 index 00000000000..6ecf9bbb102 --- /dev/null +++ b/spec/mailers/group_smtp_mailer_spec.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true +require 'rails_helper' +require 'email/receiver' + +describe GroupSmtpMailer do + let(:group) { + Fabricate(:group, + name: 'Testers', + title: 'Tester', + full_name: 'Testers Group', + smtp_server: 'smtp.gmail.com', + smtp_port: 587, + smtp_ssl: true, + imap_server: 'imap.gmail.com', + imap_port: 993, + imap_ssl: true, + email_username: 'bugs@gmail.com', + email_password: 'super$secret$password' + ) + } + + let(:user) { + user = Fabricate(:user) + group.add_owner(user) + user + } + + let(:email) { + <<~EOF + Delivered-To: bugs@gmail.com + MIME-Version: 1.0 + From: John Doe + Date: Tue, 01 Jan 2019 12:00:00 +0200 + Message-ID: + Subject: Hello from John + To: "bugs@gmail.com" + Content-Type: text/plain; charset="UTF-8" + + Hello, + + How are you doing? + EOF + } + + let(:receiver) { + receiver = Email::Receiver.new(email, + destinations: [group], + uid_validity: 1, + uid: 10000 + ) + receiver.process! + receiver + } + + let(:raw) { 'hello, how are you doing?' } + + before do + SiteSetting.enable_smtp = true + Jobs.run_immediately! + end + + it 'sends an email as reply' do + post = PostCreator.create(user, + topic_id: receiver.incoming_email.topic.id, + raw: raw + ) + + expect(ActionMailer::Base.deliveries.size).to eq(1) + + sent_mail = ActionMailer::Base.deliveries[0] + expect(sent_mail.to).to contain_exactly('john@doe.com') + expect(sent_mail.subject).to eq('Re: Hello from John') + expect(sent_mail.to_s).to include(raw) + end + +end diff --git a/spec/serializers/basic_group_serializer_spec.rb b/spec/serializers/basic_group_serializer_spec.rb index 7231e5d29ae..4000751696c 100644 --- a/spec/serializers/basic_group_serializer_spec.rb +++ b/spec/serializers/basic_group_serializer_spec.rb @@ -113,4 +113,26 @@ describe BasicGroupSerializer do end end end + + describe 'admin only fields' do + fab!(:group) { Fabricate(:group, email_username: 'foo@bar.com', email_password: 'pa$$w0rd') } + + describe 'for a user' do + let(:guardian) { Guardian.new(Fabricate(:user)) } + + it 'are not visible' do + expect(subject.as_json[:email_username]).to be_nil + expect(subject.as_json[:email_password]).to be_nil + end + end + + describe 'for an admin' do + let(:guardian) { Guardian.new(Fabricate(:admin)) } + + it 'are visible' do + expect(subject.as_json[:email_username]).to eq('foo@bar.com') + expect(subject.as_json[:email_password]).to eq('pa$$w0rd') + end + end + end end diff --git a/spec/services/post_alerter_spec.rb b/spec/services/post_alerter_spec.rb index 35c4b866ffd..0fc64e1416c 100644 --- a/spec/services/post_alerter_spec.rb +++ b/spec/services/post_alerter_spec.rb @@ -1165,4 +1165,76 @@ describe PostAlerter do expect(Notification.last.notification_type).to eq(Notification.types[:posted]) end end + + context "SMTP" do + before do + SiteSetting.enable_smtp = true + Jobs.run_immediately! + end + + fab!(:group) do + Fabricate( + :group, + smtp_server: "imap.gmail.com", + smtp_port: 587, + email_username: "discourse@example.com", + email_password: "discourse@example.com" + ) + end + + fab!(:topic) do + Fabricate(:private_message_topic, + topic_allowed_groups: [ + Fabricate.build(:topic_allowed_group, group: group) + ] + ) + end + + it "sends notifications for new posts in topic" do + post = Fabricate( + :post, + topic: topic, + incoming_email: + Fabricate( + :incoming_email, + topic: topic, + from_address: "foo@discourse.org", + to_addresses: group.email_username, + cc_addresses: "bar@discourse.org" + ) + ) + expect { PostAlerter.new.after_save_post(post, true) }.to change { ActionMailer::Base.deliveries.size }.by(0) + + post = Fabricate(:post, topic: topic) + expect { PostAlerter.new.after_save_post(post, true) }.to change { ActionMailer::Base.deliveries.size }.by(1) + email = ActionMailer::Base.deliveries.last + expect(email.from).to include(group.email_username) + expect(email.to).to contain_exactly("foo@discourse.org", "bar@discourse.org") + expect(email.subject).to eq("Re: #{topic.title}") + + post = Fabricate( + :post, + topic: topic, + incoming_email: + Fabricate( + :incoming_email, + topic: topic, + from_address: "bar@discourse.org", + to_addresses: group.email_username, + cc_addresses: "baz@discourse.org" + ) + ) + expect { PostAlerter.new.after_save_post(post, true) }.to change { ActionMailer::Base.deliveries.size }.by(0) + + post = Fabricate(:post, topic: topic.reload) + expect { PostAlerter.new.after_save_post(post, true) }.to change { ActionMailer::Base.deliveries.size }.by(1) + email = ActionMailer::Base.deliveries.last + expect(email.from).to eq([group.email_username]) + expect(email.to).to contain_exactly("foo@discourse.org", "bar@discourse.org", "baz@discourse.org") + expect(email.subject).to eq("Re: #{topic.title}") + + post = Fabricate(:post, topic: topic, post_type: Post.types[:whisper]) + expect { PostAlerter.new.after_save_post(post, true) }.to change { ActionMailer::Base.deliveries.size }.by(0) + end + end end