mirror of
https://github.com/discourse/discourse.git
synced 2025-02-09 21:04:48 +00:00
FEATURE: Implement support for IMAP and SMTP email protocols. (#8301)
Co-authored-by: Joffrey JAFFEUX <j.jaffeux@gmail.com>
This commit is contained in:
Dan Ungureanu
GitHub
parent
e88b17c044
commit
c72bc27888
@ -0,0 +1,19 @@
|
||||
import Component from "@ember/component";
|
||||
import discourseComputed from "discourse-common/utils/decorators";
|
||||
|
||||
export default Component.extend({
|
||||
@discourseComputed("model.imap_mailboxes")
|
||||
mailboxes(imapMailboxes) {
|
||||
return imapMailboxes.map(mailbox => ({ name: mailbox, value: mailbox }));
|
||||
},
|
||||
|
||||
@discourseComputed("model.imap_old_emails")
|
||||
oldEmails(oldEmails) {
|
||||
return oldEmails || 0;
|
||||
},
|
||||
|
||||
@discourseComputed("model.imap_old_emails", "model.imap_new_emails")
|
||||
totalEmails(oldEmails, newEmails) {
|
||||
return (oldEmails || 0) + (newEmails || 0);
|
||||
}
|
||||
});
|
||||
@ -18,6 +18,11 @@ export default Controller.extend({
|
||||
];
|
||||
|
||||
if (!automatic) {
|
||||
defaultTabs.splice(2, 0, {
|
||||
route: "group.manage.email",
|
||||
title: "groups.manage.email.title"
|
||||
});
|
||||
|
||||
defaultTabs.splice(1, 0, {
|
||||
route: "group.manage.membership",
|
||||
title: "groups.manage.membership.title"
|
||||
|
||||
@ -188,6 +188,15 @@ const Group = RestModel.extend({
|
||||
primary_group: !!this.primary_group,
|
||||
grant_trust_level: this.grant_trust_level,
|
||||
incoming_email: this.incoming_email,
|
||||
smtp_server: this.smtp_server,
|
||||
smtp_port: this.smtp_port,
|
||||
smtp_ssl: this.smtp_ssl,
|
||||
imap_server: this.imap_server,
|
||||
imap_port: this.imap_port,
|
||||
imap_ssl: this.imap_ssl,
|
||||
imap_mailbox_name: this.imap_mailbox_name,
|
||||
email_username: this.email_username,
|
||||
email_password: this.email_password,
|
||||
flair_icon: null,
|
||||
flair_upload_id: null,
|
||||
flair_bg_color: this.flairBackgroundHexColor,
|
||||
|
||||
@ -92,6 +92,7 @@ export default function() {
|
||||
this.route("profile");
|
||||
this.route("membership");
|
||||
this.route("interaction");
|
||||
this.route("email");
|
||||
this.route("members");
|
||||
this.route("logs");
|
||||
});
|
||||
|
||||
@ -0,0 +1,10 @@
|
||||
import DiscourseRoute from "discourse/routes/discourse";
|
||||
import I18n from "I18n";
|
||||
|
||||
export default DiscourseRoute.extend({
|
||||
showFooter: true,
|
||||
|
||||
titleToken() {
|
||||
return I18n.t("groups.manage.email.title");
|
||||
}
|
||||
});
|
||||
@ -0,0 +1,67 @@
|
||||
{{#if currentUser.admin}}
|
||||
<div class="control-group">
|
||||
<label class="control-label">{{i18n "groups.manage.email.credentials.title"}}</label>
|
||||
</div>
|
||||
|
||||
{{#if model.imap_last_error}}
|
||||
<div class="alert alert-error">{{model.imap_last_error}}</div>
|
||||
{{else}}
|
||||
<div class="alert alert-info">
|
||||
{{i18n "groups.manage.email.status" old_emails=oldEmails total_emails=totalEmails}}
|
||||
</div>
|
||||
{{/if}}
|
||||
|
||||
<div class="control-group">
|
||||
<label for="smtp_server">{{i18n "groups.manage.email.credentials.smtp_server"}}</label>
|
||||
{{input type="text" name="smtp_server" value=model.smtp_server}}
|
||||
</div>
|
||||
|
||||
<div class="control-group">
|
||||
<label for="smtp_port">{{i18n "groups.manage.email.credentials.smtp_port"}}</label>
|
||||
{{input type="text" name="smtp_port" value=model.smtp_port}}
|
||||
</div>
|
||||
|
||||
<div class="control-group">
|
||||
{{input type="checkbox" name="smtp_ssl" checked=model.smtp_ssl}}
|
||||
<label class="control-group-inline" for="smtp_ssl">{{i18n "groups.manage.email.credentials.smtp_ssl"}}</label>
|
||||
</div>
|
||||
|
||||
<div class="control-group">
|
||||
<label for="imap_server">{{i18n "groups.manage.email.credentials.imap_server"}}</label>
|
||||
{{input type="text" name="imap_server" value=model.imap_server}}
|
||||
</div>
|
||||
|
||||
<div class="control-group">
|
||||
<label for="imap_port">{{i18n "groups.manage.email.credentials.imap_port"}}</label>
|
||||
{{input type="text" name="imap_port" value=model.imap_port}}
|
||||
</div>
|
||||
|
||||
<div class="control-group">
|
||||
{{input type="checkbox" name="imap_ssl" checked=model.imap_ssl}}
|
||||
<label class="control-group-inline" for="imap_ssl">{{i18n "groups.manage.email.credentials.imap_ssl"}}</label>
|
||||
</div>
|
||||
|
||||
<div class="control-group">
|
||||
<label for="username">{{i18n "groups.manage.email.credentials.username"}}</label>
|
||||
{{input type="text" name="username" value=model.email_username}}
|
||||
</div>
|
||||
|
||||
<div class="control-group">
|
||||
<label for="password">{{i18n "groups.manage.email.credentials.password"}}</label>
|
||||
{{input type="password" name="password" value=model.email_password}}
|
||||
</div>
|
||||
|
||||
<div class="control-group">
|
||||
{{#if mailboxes}}
|
||||
<label for="imap_mailbox_name">{{i18n "groups.manage.email.mailboxes.synchronized"}}</label>
|
||||
{{combo-box name="imap_mailbox_name"
|
||||
value=model.imap_mailbox_name
|
||||
valueProperty="value"
|
||||
content=mailboxes
|
||||
none="groups.manage.email.mailboxes.disabled"
|
||||
onChange=(action (mut model.imap_mailbox_name))}}
|
||||
{{else}}
|
||||
{{i18n "groups.manage.email.mailboxes.none_found"}}
|
||||
{{/if}}
|
||||
</div>
|
||||
{{/if}}
|
||||
@ -0,0 +1,4 @@
|
||||
<form class="groups-form form-vertical">
|
||||
{{groups-form-email-fields model=model}}
|
||||
{{group-manage-save-button model=model}}
|
||||
</form>
|
||||
@ -141,3 +141,9 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.groups-form {
|
||||
.control-group-inline {
|
||||
display: inline;
|
||||
}
|
||||
}
|
||||
|
||||
@ -547,6 +547,15 @@ class GroupsController < ApplicationController
|
||||
if current_user.admin
|
||||
default_params.push(*[
|
||||
:incoming_email,
|
||||
:smtp_server,
|
||||
:smtp_port,
|
||||
:smtp_ssl,
|
||||
:imap_server,
|
||||
:imap_port,
|
||||
:imap_ssl,
|
||||
:imap_mailbox_name,
|
||||
:email_username,
|
||||
:email_password,
|
||||
:primary_group,
|
||||
:visibility_level,
|
||||
:members_visibility_level,
|
||||
|
||||
29
app/jobs/regular/group_smtp_email.rb
Normal file
29
app/jobs/regular/group_smtp_email.rb
Normal file
@ -0,0 +1,29 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_dependency 'email/sender'
|
||||
|
||||
module Jobs
|
||||
class GroupSmtpEmail < ::Jobs::Base
|
||||
sidekiq_options queue: 'critical'
|
||||
|
||||
def execute(args)
|
||||
group = Group.find_by(id: args[:group_id])
|
||||
post = Post.find_by(id: args[:post_id])
|
||||
email = args[:email]
|
||||
|
||||
Rails.logger.debug("[IMAP] Sending email for group #{group.name} and post #{post.id}")
|
||||
message = GroupSmtpMailer.send_mail(group, email, post)
|
||||
Email::Sender.new(message, :group_smtp).send
|
||||
|
||||
# Create an incoming email record to avoid importing again from IMAP
|
||||
# server.
|
||||
IncomingEmail.create!(
|
||||
user_id: post.user_id,
|
||||
topic_id: post.topic_id,
|
||||
post_id: post.id,
|
||||
raw: message.to_s,
|
||||
message_id: message.message_id
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -6,7 +6,7 @@ module Jobs
|
||||
sidekiq_options retry: 3
|
||||
|
||||
def execute(args)
|
||||
Email::Processor.process!(args[:mail], args[:retry_on_rate_limit] || false)
|
||||
Email::Processor.process!(args[:mail], retry_on_rate_limit: args[:retry_on_rate_limit] || false)
|
||||
end
|
||||
|
||||
sidekiq_retries_exhausted do |msg|
|
||||
|
||||
121
app/mailers/group_smtp_mailer.rb
Normal file
121
app/mailers/group_smtp_mailer.rb
Normal file
@ -0,0 +1,121 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_dependency 'email/message_builder'
|
||||
|
||||
class GroupSmtpMailer < ActionMailer::Base
|
||||
include Email::BuildEmailHelper
|
||||
|
||||
def send_mail(from_group, to_address, post)
|
||||
raise 'SMTP is disabled' if !SiteSetting.enable_smtp
|
||||
|
||||
incoming_email = IncomingEmail.joins(:post)
|
||||
.where('imap_uid IS NOT NULL')
|
||||
.where(topic_id: post.topic_id, posts: { post_number: 1 })
|
||||
.limit(1).first
|
||||
|
||||
context_posts = Post
|
||||
.where(topic_id: post.topic_id)
|
||||
.where("post_number < ?", post.post_number)
|
||||
.where(user_deleted: false)
|
||||
.where(hidden: false)
|
||||
.where(post_type: Post.types[:regular])
|
||||
.order(created_at: :desc)
|
||||
.limit(SiteSetting.email_posts_context)
|
||||
.to_a
|
||||
|
||||
delivery_options = {
|
||||
address: from_group.smtp_server,
|
||||
port: from_group.smtp_port,
|
||||
domain: from_group.email_username.split('@').last,
|
||||
user_name: from_group.email_username,
|
||||
password: from_group.email_password,
|
||||
authentication: GlobalSetting.smtp_authentication,
|
||||
enable_starttls_auto: from_group.smtp_ssl
|
||||
}
|
||||
|
||||
user_name = post.user.username
|
||||
if SiteSetting.enable_names && SiteSetting.display_name_on_email_from
|
||||
user_name = post.user.name unless post.user.name.blank?
|
||||
end
|
||||
|
||||
build_email(to_address,
|
||||
message: post.raw,
|
||||
url: post.url(without_slug: SiteSetting.private_email?),
|
||||
post_id: post.id,
|
||||
topic_id: post.topic_id,
|
||||
context: context(context_posts),
|
||||
username: post.user.username,
|
||||
group_name: from_group.name,
|
||||
allow_reply_by_email: true,
|
||||
only_reply_by_email: true,
|
||||
private_reply: post.topic.private_message?,
|
||||
participants: participants(post),
|
||||
include_respond_instructions: true,
|
||||
template: 'user_notifications.user_posted_pm',
|
||||
use_topic_title_subject: true,
|
||||
topic_title: incoming_email&.subject || post.topic.title,
|
||||
add_re_to_subject: true,
|
||||
locale: SiteSetting.default_locale,
|
||||
delivery_method_options: delivery_options,
|
||||
from: from_group.email_username,
|
||||
from_alias: I18n.t('email_from', user_name: user_name, site_name: Email.site_title),
|
||||
html_override: html_override(post, context_posts: context_posts)
|
||||
)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def context(context_posts)
|
||||
return "" if SiteSetting.private_email?
|
||||
|
||||
context = +""
|
||||
|
||||
if context_posts.size > 0
|
||||
context << +"-- \n*#{I18n.t('user_notifications.previous_discussion')}*\n"
|
||||
context_posts.each { |post| context << email_post_markdown(post, true) }
|
||||
end
|
||||
|
||||
context
|
||||
end
|
||||
|
||||
def email_post_markdown(post, add_posted_by = false)
|
||||
result = +"#{post.with_secure_media? ? strip_secure_urls(post.raw) : post.raw}\n\n"
|
||||
if add_posted_by
|
||||
result << "#{I18n.t('user_notifications.posted_by', username: post.username, post_date: post.created_at.strftime("%m/%d/%Y"))}\n\n"
|
||||
end
|
||||
result
|
||||
end
|
||||
|
||||
def html_override(post, context_posts: nil)
|
||||
UserNotificationRenderer.render(
|
||||
template: 'email/notification',
|
||||
format: :html,
|
||||
locals: {
|
||||
context_posts: context_posts,
|
||||
reached_limit: nil,
|
||||
post: post,
|
||||
in_reply_to_post: post.reply_to_post,
|
||||
classes: Rtl.new(nil).css_class,
|
||||
first_footer_classes: ''
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
def participants(post)
|
||||
list = []
|
||||
|
||||
post.topic.allowed_groups.each do |g|
|
||||
list.push("[#{g.name} (#{g.users.count})](#{Discourse.base_url}/groups/#{g.name})")
|
||||
end
|
||||
|
||||
post.topic.allowed_users.each do |u|
|
||||
if SiteSetting.prioritize_username_in_ux?
|
||||
list.push("[#{u.username}](#{Discourse.base_url}/u/#{u.username_lower})")
|
||||
else
|
||||
list.push("[#{u.name.blank? ? u.username : u.name}](#{Discourse.base_url}/u/#{u.username_lower})")
|
||||
end
|
||||
end
|
||||
|
||||
list.join(', ')
|
||||
end
|
||||
end
|
||||
@ -1,5 +1,7 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'net/imap'
|
||||
|
||||
class Group < ActiveRecord::Base
|
||||
# TODO(2021-05-26): remove
|
||||
self.ignored_columns = %w{
|
||||
@ -53,6 +55,7 @@ class Group < ActiveRecord::Base
|
||||
def expire_cache
|
||||
ApplicationSerializer.expire_cache_fragment!("group_names")
|
||||
SvgSprite.expire_cache
|
||||
Discourse.cache.delete("group_imap_mailboxes_#{self.id}")
|
||||
end
|
||||
|
||||
def remove_review_groups
|
||||
@ -752,6 +755,40 @@ class Group < ActiveRecord::Base
|
||||
flair_icon.presence || flair_upload&.short_path
|
||||
end
|
||||
|
||||
def imap_mailboxes
|
||||
return [] if self.imap_server.blank? ||
|
||||
self.email_username.blank? ||
|
||||
self.email_password.blank?
|
||||
|
||||
Discourse.cache.fetch("group_imap_mailboxes_#{self.id}", expires_in: 30.minutes) do
|
||||
Rails.logger.info("[IMAP] Refreshing mailboxes list for group #{self.name}")
|
||||
mailboxes = []
|
||||
|
||||
begin
|
||||
@imap = Net::IMAP.new(self.imap_server, self.imap_port, self.imap_ssl)
|
||||
@imap.login(self.email_username, self.email_password)
|
||||
|
||||
@imap.list('', '*').each do |m|
|
||||
next if m.attr.include?(:Noselect)
|
||||
mailboxes << m.name
|
||||
end
|
||||
|
||||
update_columns(imap_last_error: nil)
|
||||
rescue => ex
|
||||
update_columns(imap_last_error: ex.message)
|
||||
end
|
||||
|
||||
mailboxes
|
||||
end
|
||||
end
|
||||
|
||||
def email_username_regex
|
||||
user, domain = email_username.split('@')
|
||||
if user.present? && domain.present?
|
||||
/^#{Regexp.escape(user)}(\+[^@]*)?@#{Regexp.escape(domain)}$/i
|
||||
end
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def name_format_validator
|
||||
@ -935,10 +972,24 @@ end
|
||||
# membership_request_template :text
|
||||
# messageable_level :integer default(0)
|
||||
# mentionable_level :integer default(0)
|
||||
# smtp_server :string
|
||||
# smtp_port :integer
|
||||
# smtp_ssl :boolean
|
||||
# imap_server :string
|
||||
# imap_port :integer
|
||||
# imap_ssl :boolean
|
||||
# imap_mailbox_name :string default(""), not null
|
||||
# imap_uid_validity :integer default(0), not null
|
||||
# imap_last_uid :integer default(0), not null
|
||||
# email_username :string
|
||||
# email_password :string
|
||||
# publish_read_state :boolean default(FALSE), not null
|
||||
# members_visibility_level :integer default(0), not null
|
||||
# flair_icon :string
|
||||
# flair_upload_id :integer
|
||||
# imap_last_error :text
|
||||
# imap_old_emails :integer
|
||||
# imap_new_emails :integer
|
||||
#
|
||||
# Indexes
|
||||
#
|
||||
|
||||
@ -4,21 +4,23 @@ class GroupArchivedMessage < ActiveRecord::Base
|
||||
belongs_to :group
|
||||
belongs_to :topic
|
||||
|
||||
def self.move_to_inbox!(group_id, topic)
|
||||
def self.move_to_inbox!(group_id, topic, opts = {})
|
||||
topic_id = topic.id
|
||||
GroupArchivedMessage.where(group_id: group_id, topic_id: topic_id).destroy_all
|
||||
destroyed = GroupArchivedMessage.where(group_id: group_id, topic_id: topic_id).destroy_all
|
||||
trigger(:move_to_inbox, group_id, topic_id)
|
||||
MessageBus.publish("/topic/#{topic_id}", { type: "move_to_inbox" }, group_ids: [group_id])
|
||||
publish_topic_tracking_state(topic)
|
||||
set_imap_sync(topic_id) if !opts[:skip_imap_sync] && destroyed.present?
|
||||
end
|
||||
|
||||
def self.archive!(group_id, topic)
|
||||
def self.archive!(group_id, topic, opts = {})
|
||||
topic_id = topic.id
|
||||
GroupArchivedMessage.where(group_id: group_id, topic_id: topic_id).destroy_all
|
||||
destroyed = GroupArchivedMessage.where(group_id: group_id, topic_id: topic_id).destroy_all
|
||||
GroupArchivedMessage.create!(group_id: group_id, topic_id: topic_id)
|
||||
trigger(:archive_message, group_id, topic_id)
|
||||
MessageBus.publish("/topic/#{topic_id}", { type: "archived" }, group_ids: [group_id])
|
||||
publish_topic_tracking_state(topic)
|
||||
set_imap_sync(topic_id) if !opts[:skip_imap_sync] && destroyed.blank?
|
||||
end
|
||||
|
||||
def self.trigger(event, group_id, topic_id)
|
||||
@ -36,6 +38,13 @@ class GroupArchivedMessage < ActiveRecord::Base
|
||||
topic, group_archive: true
|
||||
)
|
||||
end
|
||||
|
||||
def self.set_imap_sync(topic_id)
|
||||
IncomingEmail.joins(:post)
|
||||
.where.not(imap_uid: nil)
|
||||
.where(topic_id: topic_id, posts: { post_number: 1 })
|
||||
.update_all(imap_sync: true)
|
||||
end
|
||||
end
|
||||
|
||||
# == Schema Information
|
||||
|
||||
@ -9,6 +9,7 @@ class IncomingEmail < ActiveRecord::Base
|
||||
|
||||
scope :addressed_to, -> (email) do
|
||||
where(<<~SQL, email: "%#{email}%")
|
||||
incoming_emails.from_address = :email OR
|
||||
incoming_emails.to_addresses ILIKE :email OR
|
||||
incoming_emails.cc_addresses ILIKE :email
|
||||
SQL
|
||||
@ -20,7 +21,8 @@ class IncomingEmail < ActiveRecord::Base
|
||||
SELECT 1
|
||||
FROM user_emails
|
||||
WHERE user_emails.user_id = :user_id AND
|
||||
(incoming_emails.to_addresses ILIKE '%' || user_emails.email || '%' OR
|
||||
(incoming_emails.from_address = user_emails.email OR
|
||||
incoming_emails.to_addresses ILIKE '%' || user_emails.email || '%' OR
|
||||
incoming_emails.cc_addresses ILIKE '%' || user_emails.email || '%')
|
||||
)
|
||||
SQL
|
||||
@ -47,11 +49,15 @@ end
|
||||
# rejection_message :text
|
||||
# is_auto_generated :boolean default(FALSE)
|
||||
# is_bounce :boolean default(FALSE), not null
|
||||
# imap_uid_validity :integer
|
||||
# imap_uid :integer
|
||||
# imap_sync :boolean
|
||||
#
|
||||
# Indexes
|
||||
#
|
||||
# index_incoming_emails_on_created_at (created_at)
|
||||
# index_incoming_emails_on_error (error)
|
||||
# index_incoming_emails_on_imap_sync (imap_sync)
|
||||
# index_incoming_emails_on_message_id (message_id)
|
||||
# index_incoming_emails_on_post_id (post_id)
|
||||
# index_incoming_emails_on_user_id (user_id) WHERE (user_id IS NOT NULL)
|
||||
|
||||
@ -200,6 +200,7 @@ class Topic < ActiveRecord::Base
|
||||
has_many :ordered_posts, -> { order(post_number: :asc) }, class_name: "Post"
|
||||
has_many :topic_allowed_users
|
||||
has_many :topic_allowed_groups
|
||||
has_many :incoming_email
|
||||
|
||||
has_many :group_archived_messages, dependent: :destroy
|
||||
has_many :user_archived_messages, dependent: :destroy
|
||||
@ -866,7 +867,8 @@ class Topic < ActiveRecord::Base
|
||||
no_bump: opts[:bump].blank?,
|
||||
topic_id: self.id,
|
||||
skip_validations: true,
|
||||
custom_fields: opts[:custom_fields])
|
||||
custom_fields: opts[:custom_fields],
|
||||
import_mode: opts[:import_mode])
|
||||
|
||||
if (new_post = creator.create) && new_post.present?
|
||||
increment!(:moderator_posts_count) if new_post.persisted?
|
||||
|
||||
@ -33,6 +33,30 @@ class BasicGroupSerializer < ApplicationSerializer
|
||||
:can_see_members,
|
||||
:publish_read_state
|
||||
|
||||
def self.admin_attributes(*attrs)
|
||||
attributes(*attrs)
|
||||
attrs.each do |attr|
|
||||
define_method "include_#{attr}?" do
|
||||
scope.is_admin?
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
admin_attributes :automatic_membership_email_domains,
|
||||
:smtp_server,
|
||||
:smtp_port,
|
||||
:smtp_ssl,
|
||||
:imap_server,
|
||||
:imap_port,
|
||||
:imap_ssl,
|
||||
:imap_mailbox_name,
|
||||
:imap_mailboxes,
|
||||
:email_username,
|
||||
:email_password,
|
||||
:imap_last_error,
|
||||
:imap_old_emails,
|
||||
:imap_new_emails
|
||||
|
||||
def include_display_name?
|
||||
object.automatic
|
||||
end
|
||||
@ -51,10 +75,6 @@ class BasicGroupSerializer < ApplicationSerializer
|
||||
staff?
|
||||
end
|
||||
|
||||
def include_automatic_membership_email_domains?
|
||||
scope.is_admin?
|
||||
end
|
||||
|
||||
def include_has_messages?
|
||||
staff? || scope.can_see_group_messages?(object)
|
||||
end
|
||||
|
||||
@ -404,7 +404,9 @@ class PostAlerter
|
||||
notification_data[:group_name] = group.name
|
||||
end
|
||||
|
||||
if original_post.via_email && (incoming_email = original_post.incoming_email)
|
||||
if opts[:skip_send_email_to]&.include?(user.email)
|
||||
skip_send_email = true
|
||||
elsif original_post.via_email && (incoming_email = original_post.incoming_email)
|
||||
skip_send_email = contains_email_address?(incoming_email.to_addresses, user) ||
|
||||
contains_email_address?(incoming_email.cc_addresses, user)
|
||||
else
|
||||
@ -542,29 +544,69 @@ class PostAlerter
|
||||
users
|
||||
end
|
||||
|
||||
def group_notifying_via_smtp(post)
|
||||
return nil if !SiteSetting.enable_smtp ||
|
||||
post.post_type != Post.types[:regular] ||
|
||||
post.incoming_email
|
||||
|
||||
post.topic.allowed_groups
|
||||
.where.not(smtp_server: nil)
|
||||
.where.not(smtp_port: nil)
|
||||
.where.not(email_username: nil)
|
||||
.where.not(email_password: nil)
|
||||
.first
|
||||
end
|
||||
|
||||
def notify_pm_users(post, reply_to_user, notified)
|
||||
return unless post.topic
|
||||
|
||||
warn_if_not_sidekiq
|
||||
|
||||
# users who interacted with the post by _directly_ emailing the group
|
||||
if group = group_notifying_via_smtp(post)
|
||||
group_email_regex = group.email_username_regex
|
||||
email_addresses = Set[group.email_username]
|
||||
|
||||
post.topic.incoming_email.each do |incoming_email|
|
||||
to_addresses = incoming_email.to_addresses&.split(';')
|
||||
cc_addresses = incoming_email.cc_addresses&.split(';')
|
||||
|
||||
next if to_addresses&.none? { |address| address =~ group_email_regex } &&
|
||||
cc_addresses&.none? { |address| address =~ group_email_regex }
|
||||
|
||||
email_addresses.add(incoming_email.from_address)
|
||||
email_addresses.merge(to_addresses) if to_addresses.present?
|
||||
email_addresses.merge(cc_addresses) if cc_addresses.present?
|
||||
end
|
||||
|
||||
email_addresses.subtract([nil, ''])
|
||||
|
||||
if email_addresses.size > 1
|
||||
Jobs.enqueue(:group_smtp_email,
|
||||
group_id: group.id,
|
||||
post_id: post.id,
|
||||
email: email_addresses.to_a - [group.email_username])
|
||||
end
|
||||
end
|
||||
|
||||
# users that aren't part of any mentioned groups
|
||||
users = directly_targeted_users(post).reject { |u| notified.include?(u) }
|
||||
DiscourseEvent.trigger(:before_create_notifications_for_users, users, post)
|
||||
users.each do |user|
|
||||
notification_level = TopicUser.get(post.topic, user)&.notification_level
|
||||
if reply_to_user == user || notification_level == TopicUser.notification_levels[:watching] || user.staged?
|
||||
create_notification(user, Notification.types[:private_message], post)
|
||||
create_notification(user, Notification.types[:private_message], post, skip_send_email_to: email_addresses)
|
||||
end
|
||||
end
|
||||
|
||||
# users that are part of all mentionned groups
|
||||
# users that are part of all mentioned groups
|
||||
users = indirectly_targeted_users(post).reject { |u| notified.include?(u) }
|
||||
DiscourseEvent.trigger(:before_create_notifications_for_users, users, post)
|
||||
users.each do |user|
|
||||
case TopicUser.get(post.topic, user)&.notification_level
|
||||
when TopicUser.notification_levels[:watching]
|
||||
# only create a notification when watching the group
|
||||
create_notification(user, Notification.types[:private_message], post)
|
||||
create_notification(user, Notification.types[:private_message], post, skip_send_email_to: email_addresses)
|
||||
when TopicUser.notification_levels[:tracking]
|
||||
notify_group_summary(user, post)
|
||||
end
|
||||
|
||||
@ -643,6 +643,23 @@ en:
|
||||
title: Interaction
|
||||
posting: Posting
|
||||
notification: Notification
|
||||
email:
|
||||
title: "Email"
|
||||
status: "Synchronized %{old_emails} / %{total_emails} emails via IMAP."
|
||||
credentials:
|
||||
title: "Credentials"
|
||||
smtp_server: "SMTP Server"
|
||||
smtp_port: "SMTP Port"
|
||||
smtp_ssl: "Use SSL for SMTP"
|
||||
imap_server: "IMAP Server"
|
||||
imap_port: "IMAP Port"
|
||||
imap_ssl: "Use SSL for IMAP"
|
||||
username: "Username"
|
||||
password: "Password"
|
||||
mailboxes:
|
||||
synchronized: "Synchronized Mailbox"
|
||||
none_found: "No mailboxes were found in this email account."
|
||||
disabled: "disabled"
|
||||
membership:
|
||||
title: Membership
|
||||
access: Access
|
||||
|
||||
@ -1966,6 +1966,17 @@ en:
|
||||
email_in_min_trust: "The minimum trust level a user needs to have to be allowed to post new topics via email."
|
||||
email_in_authserv_id: "The identifier of the service doing authentication checks on incoming emails. See <a href='https://meta.discourse.org/t/134358'>https://meta.discourse.org/t/134358</a> for instructions on how to configure this."
|
||||
email_in_spam_header: "The email header to detect spam."
|
||||
|
||||
enable_imap: "Enable IMAP for synchronizing group messages."
|
||||
enable_imap_write: "Enable two-way IMAP synchronization. If disabled, all write-operations on IMAP accounts are disabled."
|
||||
enable_imap_idle: "Use IMAP IDLE mechanism to wait for new emails."
|
||||
enable_smtp: "Enable SMTP for sending notifications for group messages."
|
||||
|
||||
imap_polling_period_mins: "The period in minutes between checking the IMAP accounts for emails."
|
||||
imap_polling_old_emails: "The maximum number of old emails (processed) to be updated every time an IMAP box is polled (0 for all)."
|
||||
imap_polling_new_emails: "The maximum number of new emails (unprocessed) to be updated every time an IMAP box is polled ."
|
||||
imap_batch_import_email: "The minimum number of new emails that trigger import mode (disables post alerts)."
|
||||
|
||||
email_prefix: "The [label] used in the subject of emails. It will default to 'title' if not set."
|
||||
email_site_title: "The title of the site used as the sender of emails from the site. Default to 'title' if not set. If your 'title' contains characters that are not allowed in email sender strings, use this setting."
|
||||
|
||||
|
||||
@ -572,6 +572,7 @@ Discourse::Application.routes.draw do
|
||||
manage/members
|
||||
manage/membership
|
||||
manage/interaction
|
||||
manage/email
|
||||
manage/logs
|
||||
}.each do |path|
|
||||
get path => 'groups#show'
|
||||
|
||||
@ -1011,6 +1011,14 @@ email:
|
||||
- X-Spam-Flag
|
||||
- X-Spam-Status
|
||||
- X-SES-Spam-Verdict
|
||||
enable_imap: false
|
||||
enable_imap_write: false
|
||||
enable_imap_idle: false
|
||||
enable_smtp: false
|
||||
imap_polling_period_mins: 5
|
||||
imap_polling_old_emails: 1000
|
||||
imap_polling_new_emails: 250
|
||||
imap_batch_import_email: 100
|
||||
email_prefix: ""
|
||||
email_site_title: ""
|
||||
disable_emails:
|
||||
|
||||
@ -101,77 +101,131 @@ before_fork do |server, worker|
|
||||
Demon::Sidekiq.kill("USR1")
|
||||
old_handler.call
|
||||
end
|
||||
end
|
||||
|
||||
class ::Unicorn::HttpServer
|
||||
alias :master_sleep_orig :master_sleep
|
||||
puts "Starting up email sync"
|
||||
Demon::EmailSync.start
|
||||
Signal.trap("SIGTSTP") do
|
||||
STDERR.puts "#{Time.now}: Issuing stop to email_sync"
|
||||
Demon::EmailSync.stop
|
||||
end
|
||||
|
||||
def max_rss
|
||||
rss = `ps -eo rss,args | grep sidekiq | grep -v grep | awk '{print $1}'`
|
||||
.split("\n")
|
||||
.map(&:to_i)
|
||||
.max
|
||||
class ::Unicorn::HttpServer
|
||||
alias :master_sleep_orig :master_sleep
|
||||
|
||||
rss ||= 0
|
||||
def max_sidekiq_rss
|
||||
rss = `ps -eo rss,args | grep sidekiq | grep -v grep | awk '{print $1}'`
|
||||
.split("\n")
|
||||
.map(&:to_i)
|
||||
.max
|
||||
|
||||
rss * 1024
|
||||
end
|
||||
rss ||= 0
|
||||
|
||||
def max_allowed_size
|
||||
[ENV['UNICORN_SIDEKIQ_MAX_RSS'].to_i, 500].max.megabytes
|
||||
end
|
||||
rss * 1024
|
||||
end
|
||||
|
||||
def out_of_memory?
|
||||
max_rss > max_allowed_size
|
||||
end
|
||||
def max_allowed_sidekiq_rss
|
||||
[ENV['UNICORN_SIDEKIQ_MAX_RSS'].to_i, 500].max.megabytes
|
||||
end
|
||||
|
||||
def force_kill_rogue_sidekiq
|
||||
info = `ps -eo pid,rss,args | grep sidekiq | grep -v grep | awk '{print $1,$2}'`
|
||||
info.split("\n").each do |row|
|
||||
pid, mem = row.split(" ").map(&:to_i)
|
||||
if pid > 0 && (mem * 1024) > max_allowed_size
|
||||
Rails.logger.warn "Detected rogue Sidekiq pid #{pid} mem #{mem * 1024}, killing"
|
||||
Process.kill("KILL", pid) rescue nil
|
||||
end
|
||||
def force_kill_rogue_sidekiq
|
||||
info = `ps -eo pid,rss,args | grep sidekiq | grep -v grep | awk '{print $1,$2}'`
|
||||
info.split("\n").each do |row|
|
||||
pid, mem = row.split(" ").map(&:to_i)
|
||||
if pid > 0 && (mem * 1024) > max_allowed_sidekiq_rss
|
||||
Rails.logger.warn "Detected rogue Sidekiq pid #{pid} mem #{mem * 1024}, killing"
|
||||
Process.kill("KILL", pid) rescue nil
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def check_sidekiq_heartbeat
|
||||
@sidekiq_heartbeat_interval ||= 30.minutes
|
||||
@sidekiq_next_heartbeat_check ||= Time.now.to_i + @sidekiq_heartbeat_interval
|
||||
def check_sidekiq_heartbeat
|
||||
@sidekiq_heartbeat_interval ||= 30.minutes
|
||||
@sidekiq_next_heartbeat_check ||= Time.now.to_i + @sidekiq_heartbeat_interval
|
||||
|
||||
if @sidekiq_next_heartbeat_check < Time.now.to_i
|
||||
if @sidekiq_next_heartbeat_check < Time.now.to_i
|
||||
|
||||
last_heartbeat = Jobs::RunHeartbeat.last_heartbeat
|
||||
restart = false
|
||||
last_heartbeat = Jobs::RunHeartbeat.last_heartbeat
|
||||
restart = false
|
||||
|
||||
if out_of_memory?
|
||||
Rails.logger.warn("Sidekiq is consuming too much memory (using: %0.2fM) for '%s', restarting" % [(max_rss.to_f / 1.megabyte), ENV["DISCOURSE_HOSTNAME"]])
|
||||
restart = true
|
||||
end
|
||||
|
||||
if last_heartbeat < Time.now.to_i - @sidekiq_heartbeat_interval
|
||||
STDERR.puts "Sidekiq heartbeat test failed, restarting"
|
||||
Rails.logger.warn "Sidekiq heartbeat test failed, restarting"
|
||||
|
||||
restart = true
|
||||
end
|
||||
@sidekiq_next_heartbeat_check = Time.now.to_i + @sidekiq_heartbeat_interval
|
||||
|
||||
if restart
|
||||
Demon::Sidekiq.restart
|
||||
sleep 10
|
||||
force_kill_rogue_sidekiq
|
||||
end
|
||||
Discourse.redis.close
|
||||
sidekiq_rss = max_sidekiq_rss
|
||||
if sidekiq_rss > max_allowed_sidekiq_rss
|
||||
Rails.logger.warn("Sidekiq is consuming too much memory (using: %0.2fM) for '%s', restarting" % [(sidekiq_rss.to_f / 1.megabyte), ENV["DISCOURSE_HOSTNAME"]])
|
||||
restart = true
|
||||
end
|
||||
|
||||
if last_heartbeat < Time.now.to_i - @sidekiq_heartbeat_interval
|
||||
STDERR.puts "Sidekiq heartbeat test failed, restarting"
|
||||
Rails.logger.warn "Sidekiq heartbeat test failed, restarting"
|
||||
|
||||
restart = true
|
||||
end
|
||||
@sidekiq_next_heartbeat_check = Time.now.to_i + @sidekiq_heartbeat_interval
|
||||
|
||||
if restart
|
||||
Demon::Sidekiq.restart
|
||||
sleep 10
|
||||
force_kill_rogue_sidekiq
|
||||
end
|
||||
Discourse.redis.close
|
||||
end
|
||||
end
|
||||
|
||||
def max_email_sync_rss
|
||||
return 0 if Demon::EmailSync.demons.empty?
|
||||
|
||||
email_sync_pids = Demon::EmailSync.demons.map { |uid, demon| demon.pid }
|
||||
return 0 if email_sync_pids.empty?
|
||||
|
||||
rss = `ps -eo pid,rss,args | grep '#{email_sync_pids.join('|')}' | grep -v grep | awk '{print $2}'`
|
||||
.split("\n")
|
||||
.map(&:to_i)
|
||||
.max
|
||||
|
||||
(rss || 0) * 1024
|
||||
end
|
||||
|
||||
def max_allowed_email_sync_rss
|
||||
[ENV['UNICORN_EMAIL_SYNC_MAX_RSS'].to_i, 500].max.megabytes
|
||||
end
|
||||
|
||||
def check_email_sync_heartbeat
|
||||
# Skip first check to let process warm up
|
||||
@email_sync_next_heartbeat_check ||= (Time.now + Demon::EmailSync::HEARTBEAT_INTERVAL).to_i
|
||||
|
||||
return if @email_sync_next_heartbeat_check > Time.now.to_i
|
||||
@email_sync_next_heartbeat_check = (Time.now + Demon::EmailSync::HEARTBEAT_INTERVAL).to_i
|
||||
|
||||
restart = false
|
||||
|
||||
# Restart process if it does not respond anymore
|
||||
last_heartbeat_ago = Time.now.to_i - Discourse.redis.get(Demon::EmailSync::HEARTBEAT_KEY).to_i
|
||||
if last_heartbeat_ago > Demon::EmailSync::HEARTBEAT_INTERVAL.to_i
|
||||
STDERR.puts("EmailSync heartbeat test failed (last heartbeat was #{last_heartbeat_ago}s ago), restarting")
|
||||
restart = true
|
||||
end
|
||||
|
||||
def master_sleep(sec)
|
||||
# Restart process if memory usage is too high
|
||||
email_sync_rss = max_email_sync_rss
|
||||
if email_sync_rss > max_allowed_email_sync_rss
|
||||
STDERR.puts("EmailSync is consuming too much memory (using: %0.2fM) for '%s', restarting" % [(email_sync_rss.to_f / 1.megabyte), ENV["DISCOURSE_HOSTNAME"]])
|
||||
restart = true
|
||||
end
|
||||
|
||||
Demon::EmailSync.restart if restart
|
||||
end
|
||||
|
||||
def master_sleep(sec)
|
||||
sidekiqs = ENV['UNICORN_SIDEKIQS'].to_i
|
||||
if sidekiqs > 0
|
||||
Demon::Sidekiq.ensure_running
|
||||
check_sidekiq_heartbeat
|
||||
|
||||
master_sleep_orig(sec)
|
||||
end
|
||||
|
||||
Demon::EmailSync.ensure_running
|
||||
check_email_sync_heartbeat
|
||||
|
||||
master_sleep_orig(sec)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
20
db/migrate/20181220115844_add_smtp_and_imap_to_groups.rb
Normal file
20
db/migrate/20181220115844_add_smtp_and_imap_to_groups.rb
Normal file
@ -0,0 +1,20 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class AddSmtpAndImapToGroups < ActiveRecord::Migration[5.2]
|
||||
def change
|
||||
add_column :groups, :smtp_server, :string
|
||||
add_column :groups, :smtp_port, :integer
|
||||
add_column :groups, :smtp_ssl, :boolean
|
||||
|
||||
add_column :groups, :imap_server, :string
|
||||
add_column :groups, :imap_port, :integer
|
||||
add_column :groups, :imap_ssl, :boolean
|
||||
|
||||
add_column :groups, :imap_mailbox_name, :string, default: '', null: false
|
||||
add_column :groups, :imap_uid_validity, :integer, default: 0, null: false
|
||||
add_column :groups, :imap_last_uid, :integer, default: 0, null: false
|
||||
|
||||
add_column :groups, :email_username, :string
|
||||
add_column :groups, :email_password, :string
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,11 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class AddImapFieldsToIncomingEmails < ActiveRecord::Migration[5.2]
|
||||
def change
|
||||
add_column :incoming_emails, :imap_uid_validity, :integer
|
||||
add_column :incoming_emails, :imap_uid, :integer
|
||||
add_column :incoming_emails, :imap_sync, :boolean
|
||||
|
||||
add_index :incoming_emails, :imap_sync
|
||||
end
|
||||
end
|
||||
9
db/migrate/20200327164420_add_imap_stats_to_group.rb
Normal file
9
db/migrate/20200327164420_add_imap_stats_to_group.rb
Normal file
@ -0,0 +1,9 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class AddImapStatsToGroup < ActiveRecord::Migration[6.0]
|
||||
def change
|
||||
add_column :groups, :imap_last_error, :text
|
||||
add_column :groups, :imap_old_emails, :integer
|
||||
add_column :groups, :imap_new_emails, :integer
|
||||
end
|
||||
end
|
||||
161
lib/demon/email_sync.rb
Normal file
161
lib/demon/email_sync.rb
Normal file
@ -0,0 +1,161 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "demon/base"
|
||||
|
||||
class Demon::EmailSync < ::Demon::Base
|
||||
HEARTBEAT_KEY ||= "email_sync_heartbeat"
|
||||
HEARTBEAT_INTERVAL ||= 60.seconds
|
||||
|
||||
def self.prefix
|
||||
"email_sync"
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def suppress_stdout
|
||||
false
|
||||
end
|
||||
|
||||
def suppress_stderr
|
||||
false
|
||||
end
|
||||
|
||||
def start_thread(db, group)
|
||||
Thread.new do
|
||||
RailsMultisite::ConnectionManagement.with_connection(db) do
|
||||
begin
|
||||
obj = Imap::Sync.for_group(group)
|
||||
rescue Net::IMAP::NoResponseError => e
|
||||
group.update(imap_last_error: e.message)
|
||||
Thread.exit
|
||||
end
|
||||
|
||||
@sync_lock.synchronize { @sync_data[db][group.id][:obj] = obj }
|
||||
|
||||
status = nil
|
||||
idle = false
|
||||
|
||||
while @running && group.reload.imap_mailbox_name.present? do
|
||||
status = obj.process(
|
||||
idle: obj.can_idle? && status && status[:remaining] == 0,
|
||||
old_emails_limit: status && status[:remaining] > 0 ? 0 : nil,
|
||||
)
|
||||
|
||||
if !obj.can_idle? && status[:remaining] == 0
|
||||
# Thread goes into sleep for a bit so it is better to return any
|
||||
# connection back to the pool.
|
||||
ActiveRecord::Base.connection_handler.clear_active_connections!
|
||||
|
||||
sleep SiteSetting.imap_polling_period_mins.minutes
|
||||
end
|
||||
end
|
||||
|
||||
obj.disconnect!
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def kill_threads
|
||||
# This is not really safe so the caller should ensure it happens in a
|
||||
# thread-safe context.
|
||||
# It should be safe when called from within a `trap` (there are no
|
||||
# synchronization primitives available anyway).
|
||||
@running = false
|
||||
|
||||
@sync_data.each do |db, sync_data|
|
||||
sync_data.each do |_, data|
|
||||
data[:thread].kill
|
||||
data[:thread].join
|
||||
data[:obj]&.disconnect! rescue nil
|
||||
end
|
||||
end
|
||||
|
||||
exit 0
|
||||
end
|
||||
|
||||
def after_fork
|
||||
puts "Loading EmailSync in process id #{Process.pid}"
|
||||
|
||||
loop do
|
||||
break if Discourse.redis.set(HEARTBEAT_KEY, Time.now.to_i, ex: HEARTBEAT_INTERVAL, nx: true)
|
||||
sleep HEARTBEAT_INTERVAL
|
||||
end
|
||||
|
||||
puts "Starting EmailSync main thread"
|
||||
|
||||
@running = true
|
||||
@sync_data = {}
|
||||
@sync_lock = Mutex.new
|
||||
|
||||
trap('INT') { kill_threads }
|
||||
trap('TERM') { kill_threads }
|
||||
trap('HUP') { kill_threads }
|
||||
|
||||
while @running
|
||||
Discourse.redis.set(HEARTBEAT_KEY, Time.now.to_i, ex: HEARTBEAT_INTERVAL)
|
||||
|
||||
# Kill all threads for databases that no longer exist
|
||||
all_dbs = Set.new(RailsMultisite::ConnectionManagement.all_dbs)
|
||||
@sync_data.filter! do |db, sync_data|
|
||||
next true if all_dbs.include?(db)
|
||||
|
||||
sync_data.each do |_, data|
|
||||
data[:thread].kill
|
||||
data[:thread].join
|
||||
data[:obj]&.disconnect!
|
||||
end
|
||||
|
||||
false
|
||||
end
|
||||
|
||||
RailsMultisite::ConnectionManagement.each_connection do |db|
|
||||
if SiteSetting.enable_imap
|
||||
groups = Group.where.not(imap_mailbox_name: '').map { |group| [group.id, group] }.to_h
|
||||
|
||||
@sync_lock.synchronize do
|
||||
@sync_data[db] ||= {}
|
||||
|
||||
# Kill threads for group's mailbox that are no longer synchronized.
|
||||
@sync_data[db].filter! do |group_id, data|
|
||||
next true if groups[group_id] && data[:thread]&.alive? && !data[:obj]&.disconnected?
|
||||
|
||||
if !groups[group_id]
|
||||
puts("[EmailSync] Killing thread for group (id = #{group_id}) because mailbox is no longer synced")
|
||||
else
|
||||
puts("[EmailSync] Thread for group #{groups[group_id].name} is dead")
|
||||
end
|
||||
|
||||
data[:thread].kill
|
||||
data[:thread].join
|
||||
data[:obj]&.disconnect!
|
||||
|
||||
false
|
||||
end
|
||||
|
||||
# Spawn new threads for groups that are now synchronized.
|
||||
groups.each do |group_id, group|
|
||||
if !@sync_data[db][group_id]
|
||||
puts("[EmailSync] Starting thread for group #{group.name} and mailbox #{group.imap_mailbox_name}")
|
||||
@sync_data[db][group_id] = { thread: start_thread(db, group), obj: nil }
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Thread goes into sleep for a bit so it is better to return any
|
||||
# connection back to the pool.
|
||||
ActiveRecord::Base.connection_handler.clear_active_connections!
|
||||
|
||||
sleep 5
|
||||
end
|
||||
|
||||
@sync_lock.synchronize { kill_threads }
|
||||
Discourse.redis.del(HEARTBEAT_KEY)
|
||||
exit 0
|
||||
rescue => e
|
||||
STDERR.puts e.message
|
||||
STDERR.puts e.backtrace.join("\n")
|
||||
exit 1
|
||||
end
|
||||
end
|
||||
@ -127,13 +127,17 @@ module Email
|
||||
end
|
||||
|
||||
def build_args
|
||||
{
|
||||
args = {
|
||||
to: @to,
|
||||
subject: subject,
|
||||
body: body,
|
||||
charset: 'UTF-8',
|
||||
from: from_value
|
||||
}
|
||||
|
||||
args[:delivery_method_options] = @opts[:delivery_method_options] if @opts[:delivery_method_options]
|
||||
|
||||
args
|
||||
end
|
||||
|
||||
def header_args
|
||||
|
||||
@ -5,21 +5,21 @@ module Email
|
||||
class Processor
|
||||
attr_reader :receiver
|
||||
|
||||
def initialize(mail, retry_on_rate_limit = true)
|
||||
def initialize(mail, opts = {})
|
||||
@mail = mail
|
||||
@retry_on_rate_limit = retry_on_rate_limit
|
||||
@opts = opts
|
||||
end
|
||||
|
||||
def self.process!(mail, retry_on_rate_limit = true)
|
||||
Email::Processor.new(mail, retry_on_rate_limit).process!
|
||||
def self.process!(mail, opts = {})
|
||||
Email::Processor.new(mail, opts).process!
|
||||
end
|
||||
|
||||
def process!
|
||||
begin
|
||||
@receiver = Email::Receiver.new(@mail)
|
||||
@receiver = Email::Receiver.new(@mail, @opts)
|
||||
@receiver.process!
|
||||
rescue RateLimiter::LimitExceeded
|
||||
@retry_on_rate_limit ? Jobs.enqueue(:process_email, mail: @mail) : raise
|
||||
@opts[:retry_on_rate_limit] ? Jobs.enqueue(:process_email, mail: @mail) : raise
|
||||
rescue => e
|
||||
return handle_bounce(e) if @receiver.is_bounce?
|
||||
|
||||
|
||||
@ -58,6 +58,7 @@ module Email
|
||||
@mail = Mail.new(@raw_email)
|
||||
@message_id = @mail.message_id.presence || Digest::MD5.hexdigest(mail_string)
|
||||
@opts = opts
|
||||
@destinations ||= opts[:destinations]
|
||||
end
|
||||
|
||||
def process!
|
||||
@ -65,14 +66,19 @@ module Email
|
||||
id_hash = Digest::SHA1.hexdigest(@message_id)
|
||||
DistributedMutex.synchronize("process_email_#{id_hash}") do
|
||||
begin
|
||||
return if IncomingEmail.exists?(message_id: @message_id)
|
||||
@incoming_email = IncomingEmail.find_by(message_id: @message_id)
|
||||
if @incoming_email
|
||||
@incoming_email.update(imap_uid_validity: @opts[:uid_validity], imap_uid: @opts[:uid], imap_sync: false)
|
||||
return
|
||||
end
|
||||
ensure_valid_address_lists
|
||||
ensure_valid_date
|
||||
@from_email, @from_display_name = parse_from_field
|
||||
@from_user = User.find_by_email(@from_email)
|
||||
@incoming_email = create_incoming_email
|
||||
process_internal
|
||||
post = process_internal
|
||||
raise BouncedEmailError if is_bounce?
|
||||
return post
|
||||
rescue Exception => e
|
||||
error = e.to_s
|
||||
error = e.class.name if error.blank?
|
||||
@ -112,6 +118,9 @@ module Email
|
||||
from_address: @from_email,
|
||||
to_addresses: @mail.to&.map(&:downcase)&.join(";"),
|
||||
cc_addresses: @mail.cc&.map(&:downcase)&.join(";"),
|
||||
imap_uid_validity: @opts[:uid_validity],
|
||||
imap_uid: @opts[:uid],
|
||||
imap_sync: false
|
||||
)
|
||||
end
|
||||
|
||||
@ -137,7 +146,7 @@ module Email
|
||||
if is_auto_generated? && !sent_to_mailinglist_mirror?
|
||||
@incoming_email.update_columns(is_auto_generated: true)
|
||||
|
||||
if SiteSetting.block_auto_generated_emails? && !is_bounce?
|
||||
if SiteSetting.block_auto_generated_emails? && !is_bounce? && !@opts[:allow_auto_generated]
|
||||
raise AutoGeneratedEmailError
|
||||
end
|
||||
end
|
||||
@ -566,7 +575,7 @@ module Email
|
||||
def subject
|
||||
@subject ||=
|
||||
if mail_subject = @mail.subject
|
||||
mail_subject.delete("\u0000")
|
||||
mail_subject.delete("\u0000")[0..254]
|
||||
else
|
||||
I18n.t("emails.incoming.default_subject", email: @from_email)
|
||||
end
|
||||
@ -621,10 +630,7 @@ module Email
|
||||
def sent_to_mailinglist_mirror?
|
||||
@sent_to_mailinglist_mirror ||= begin
|
||||
destinations.each do |destination|
|
||||
next unless destination[:type] == :category
|
||||
|
||||
category = destination[:obj]
|
||||
return true if category.mailinglist_mirror?
|
||||
return true if destination.is_a?(Category) && destination.mailinglist_mirror?
|
||||
end
|
||||
|
||||
false
|
||||
@ -635,10 +641,10 @@ module Email
|
||||
# only check for a group/category when 'email_in' is enabled
|
||||
if SiteSetting.email_in
|
||||
group = Group.find_by_email(address)
|
||||
return { type: :group, obj: group } if group
|
||||
return group if group
|
||||
|
||||
category = Category.find_by_email(address)
|
||||
return { type: :category, obj: category } if category
|
||||
return category if category
|
||||
end
|
||||
|
||||
# reply
|
||||
@ -647,7 +653,7 @@ module Email
|
||||
match.captures.each do |c|
|
||||
next if c.blank?
|
||||
post_reply_key = PostReplyKey.find_by(reply_key: c)
|
||||
return { type: :reply, obj: post_reply_key } if post_reply_key
|
||||
return post_reply_key if post_reply_key
|
||||
end
|
||||
end
|
||||
nil
|
||||
@ -658,19 +664,13 @@ module Email
|
||||
has_been_forwarded? &&
|
||||
process_forwarded_email(destination, user)
|
||||
|
||||
return if is_bounce? && destination[:type] != :reply
|
||||
return if is_bounce? && !destination.is_a?(PostReplyKey)
|
||||
|
||||
case destination[:type]
|
||||
when :group
|
||||
if destination.is_a?(Group)
|
||||
user ||= stage_from_user
|
||||
|
||||
group = destination[:obj]
|
||||
create_group_post(group, user, body, elided)
|
||||
|
||||
when :category
|
||||
category = destination[:obj]
|
||||
|
||||
raise StrangersNotAllowedError if (user.nil? || user.staged?) && !category.email_in_allow_strangers
|
||||
create_group_post(destination, user, body, elided)
|
||||
elsif destination.is_a?(Category)
|
||||
raise StrangersNotAllowedError if (user.nil? || user.staged?) && !destination.email_in_allow_strangers
|
||||
|
||||
user ||= stage_from_user
|
||||
|
||||
@ -680,19 +680,18 @@ module Email
|
||||
raw: body,
|
||||
elided: elided,
|
||||
title: subject,
|
||||
category: category.id,
|
||||
category: destination.id,
|
||||
skip_validations: user.staged?)
|
||||
|
||||
when :reply
|
||||
elsif destination.is_a?(PostReplyKey)
|
||||
# We don't stage new users for emails to reply addresses, exit if user is nil
|
||||
raise BadDestinationAddress if user.blank?
|
||||
|
||||
post_reply_key = destination[:obj]
|
||||
post = Post.with_deleted.find(post_reply_key.post_id)
|
||||
post = Post.with_deleted.find(destination.post_id)
|
||||
raise ReplyNotAllowedError if !Guardian.new(user).can_create_post?(post&.topic)
|
||||
|
||||
if post_reply_key.user_id != user.id && !forwarded_reply_key?(post_reply_key, user)
|
||||
raise ReplyUserNotMatchingError, "post_reply_key.user_id => #{post_reply_key.user_id.inspect}, user.id => #{user.id.inspect}"
|
||||
if destination.user_id != user.id && !forwarded_reply_key?(destination, user)
|
||||
raise ReplyUserNotMatchingError, "post_reply_key.user_id => #{destination.user_id.inspect}, user.id => #{user.id.inspect}"
|
||||
end
|
||||
|
||||
create_reply(user: user,
|
||||
@ -712,11 +711,12 @@ module Email
|
||||
incoming_emails = IncomingEmail
|
||||
.where(message_id: message_ids)
|
||||
.addressed_to_user(user)
|
||||
.pluck(:post_id, :to_addresses, :cc_addresses)
|
||||
.pluck(:post_id, :from_address, :to_addresses, :cc_addresses)
|
||||
|
||||
incoming_emails.each do |post_id, to_addresses, cc_addresses|
|
||||
post_ids << post_id if contains_email_address_of_user?(to_addresses, user) ||
|
||||
contains_email_address_of_user?(cc_addresses, user)
|
||||
incoming_emails.each do |post_id, from_address, to_addresses, cc_addresses|
|
||||
post_ids << post_id if contains_email_address_of_user?(from_address, user) ||
|
||||
contains_email_address_of_user?(to_addresses, user) ||
|
||||
contains_email_address_of_user?(cc_addresses, user)
|
||||
end
|
||||
|
||||
if post_ids.any? && post = Post.where(id: post_ids).order(:created_at).last
|
||||
@ -801,31 +801,27 @@ module Email
|
||||
end
|
||||
|
||||
def forwarded_email_create_topic(destination: , user: , raw: , title: , date: nil, embedded_user: nil)
|
||||
case destination[:type]
|
||||
when :group
|
||||
group = destination[:obj]
|
||||
if destination.is_a?(Group)
|
||||
topic_user = embedded_user&.call || user
|
||||
create_topic(user: topic_user,
|
||||
raw: raw,
|
||||
title: title,
|
||||
archetype: Archetype.private_message,
|
||||
target_usernames: [user.username],
|
||||
target_group_names: [group.name],
|
||||
target_group_names: [destination.name],
|
||||
is_group_message: true,
|
||||
skip_validations: true,
|
||||
created_at: date)
|
||||
|
||||
when :category
|
||||
category = destination[:obj]
|
||||
|
||||
return false if user.staged? && !category.email_in_allow_strangers
|
||||
elsif destination.is_a?(Category)
|
||||
return false if user.staged? && !destination.email_in_allow_strangers
|
||||
return false if !user.has_trust_level?(SiteSetting.email_in_min_trust)
|
||||
|
||||
topic_user = embedded_user&.call || user
|
||||
create_topic(user: topic_user,
|
||||
raw: raw,
|
||||
title: title,
|
||||
category: category.id,
|
||||
category: destination.id,
|
||||
skip_validations: topic_user.staged?,
|
||||
created_at: date)
|
||||
else
|
||||
@ -854,7 +850,7 @@ module Email
|
||||
# create reply when available
|
||||
if @before_embedded.present?
|
||||
post_type = Post.types[:regular]
|
||||
post_type = Post.types[:whisper] if post.topic.private_message? && destination[:obj].usernames[user.username]
|
||||
post_type = Post.types[:whisper] if post.topic.private_message? && destination.usernames[user.username]
|
||||
|
||||
create_reply(user: user,
|
||||
raw: @before_embedded,
|
||||
@ -1114,6 +1110,8 @@ module Email
|
||||
end
|
||||
|
||||
def create_post(options = {})
|
||||
options[:import_mode] = @opts[:import_mode]
|
||||
|
||||
options[:via_email] = true
|
||||
options[:raw_email] = @raw_email
|
||||
|
||||
@ -1185,11 +1183,11 @@ module Email
|
||||
if user && can_invite?(post.topic, user)
|
||||
post.topic.topic_allowed_users.create!(user_id: user.id)
|
||||
TopicUser.auto_notification_for_staging(user.id, post.topic_id, TopicUser.notification_reasons[:auto_watch])
|
||||
post.topic.add_small_action(sender, "invited_user", user.username)
|
||||
post.topic.add_small_action(sender, "invited_user", user.username, import_mode: @opts[:import_mode])
|
||||
end
|
||||
# cap number of staged users created per email
|
||||
if @staged_users.count > SiteSetting.maximum_staged_users_per_email
|
||||
post.topic.add_moderator_post(sender, I18n.t("emails.incoming.maximum_staged_user_per_email_reached"))
|
||||
post.topic.add_moderator_post(sender, I18n.t("emails.incoming.maximum_staged_user_per_email_reached"), import_mode: @opts[:import_mode])
|
||||
return
|
||||
end
|
||||
end
|
||||
|
||||
116
lib/imap/providers/generic.rb
Normal file
116
lib/imap/providers/generic.rb
Normal file
@ -0,0 +1,116 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'net/imap'
|
||||
|
||||
module Imap
|
||||
module Providers
|
||||
class Generic
|
||||
|
||||
def initialize(server, options = {})
|
||||
@server = server
|
||||
@port = options[:port] || 993
|
||||
@ssl = options[:ssl] || true
|
||||
@username = options[:username]
|
||||
@password = options[:password]
|
||||
@timeout = options[:timeout] || 10
|
||||
end
|
||||
|
||||
def imap
|
||||
@imap ||= Net::IMAP.new(@server, port: @port, ssl: @ssl, open_timeout: @timeout)
|
||||
end
|
||||
|
||||
def disconnected?
|
||||
@imap && @imap.disconnected?
|
||||
end
|
||||
|
||||
def connect!
|
||||
imap.login(@username, @password)
|
||||
end
|
||||
|
||||
def disconnect!
|
||||
imap.logout rescue nil
|
||||
imap.disconnect
|
||||
end
|
||||
|
||||
def can?(capability)
|
||||
@capabilities ||= imap.responses['CAPABILITY'][-1] || imap.capability
|
||||
@capabilities.include?(capability)
|
||||
end
|
||||
|
||||
def uids(opts = {})
|
||||
if opts[:from] && opts[:to]
|
||||
imap.uid_search("UID #{opts[:from]}:#{opts[:to]}")
|
||||
elsif opts[:from]
|
||||
imap.uid_search("UID #{opts[:from]}:*")
|
||||
elsif opts[:to]
|
||||
imap.uid_search("UID 1:#{opts[:to]}")
|
||||
else
|
||||
imap.uid_search('ALL')
|
||||
end
|
||||
end
|
||||
|
||||
def labels
|
||||
@labels ||= begin
|
||||
labels = {}
|
||||
|
||||
list_mailboxes.each do |name|
|
||||
if tag = to_tag(name)
|
||||
labels[tag] = name
|
||||
end
|
||||
end
|
||||
|
||||
labels
|
||||
end
|
||||
end
|
||||
|
||||
def open_mailbox(mailbox_name, write: false)
|
||||
if write
|
||||
raise 'two-way IMAP sync is disabled' if !SiteSetting.enable_imap_write
|
||||
imap.select(mailbox_name)
|
||||
else
|
||||
imap.examine(mailbox_name)
|
||||
end
|
||||
|
||||
{
|
||||
uid_validity: imap.responses['UIDVALIDITY'][-1]
|
||||
}
|
||||
end
|
||||
|
||||
def emails(uids, fields, opts = {})
|
||||
imap.uid_fetch(uids, fields).map do |email|
|
||||
attributes = {}
|
||||
|
||||
fields.each do |field|
|
||||
attributes[field] = email.attr[field]
|
||||
end
|
||||
|
||||
attributes
|
||||
end
|
||||
end
|
||||
|
||||
def store(uid, attribute, old_set, new_set)
|
||||
additions = new_set.reject { |val| old_set.include?(val) }
|
||||
imap.uid_store(uid, "+#{attribute}", additions) if additions.length > 0
|
||||
removals = old_set.reject { |val| new_set.include?(val) }
|
||||
imap.uid_store(uid, "-#{attribute}", removals) if removals.length > 0
|
||||
end
|
||||
|
||||
def to_tag(label)
|
||||
label = DiscourseTagging.clean_tag(label.to_s)
|
||||
label if label != 'inbox' && label != 'sent'
|
||||
end
|
||||
|
||||
def tag_to_flag(tag)
|
||||
:Seen if tag == 'seen'
|
||||
end
|
||||
|
||||
def tag_to_label(tag)
|
||||
labels[tag]
|
||||
end
|
||||
|
||||
def list_mailboxes
|
||||
imap.list('', '*').map(&:name)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
145
lib/imap/providers/gmail.rb
Normal file
145
lib/imap/providers/gmail.rb
Normal file
@ -0,0 +1,145 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Imap
|
||||
module Providers
|
||||
class Gmail < Generic
|
||||
X_GM_LABELS = 'X-GM-LABELS'
|
||||
|
||||
def imap
|
||||
@imap ||= super.tap { |imap| apply_gmail_patch(imap) }
|
||||
end
|
||||
|
||||
def emails(uids, fields, opts = {})
|
||||
fields[fields.index('LABELS')] = X_GM_LABELS
|
||||
|
||||
emails = super(uids, fields, opts)
|
||||
|
||||
emails.each do |email|
|
||||
email['LABELS'] = Array(email['LABELS'])
|
||||
|
||||
if email[X_GM_LABELS]
|
||||
email['LABELS'] << Array(email.delete(X_GM_LABELS))
|
||||
email['LABELS'].flatten!
|
||||
end
|
||||
|
||||
email['LABELS'] << '\\Inbox' if opts[:mailbox] == 'INBOX'
|
||||
|
||||
email['LABELS'].uniq!
|
||||
end
|
||||
|
||||
emails
|
||||
end
|
||||
|
||||
def store(uid, attribute, old_set, new_set)
|
||||
attribute = X_GM_LABELS if attribute == 'LABELS'
|
||||
super(uid, attribute, old_set, new_set)
|
||||
end
|
||||
|
||||
def to_tag(label)
|
||||
# Label `\\Starred` is Gmail equivalent of :Flagged (both present)
|
||||
return 'starred' if label == :Flagged
|
||||
return if label == '[Gmail]/All Mail'
|
||||
|
||||
label = label.to_s.gsub('[Gmail]/', '')
|
||||
super(label)
|
||||
end
|
||||
|
||||
def tag_to_flag(tag)
|
||||
return :Flagged if tag == 'starred'
|
||||
|
||||
super(tag)
|
||||
end
|
||||
|
||||
def tag_to_label(tag)
|
||||
return '\\Important' if tag == 'important'
|
||||
return '\\Starred' if tag == 'starred'
|
||||
|
||||
super(tag)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def apply_gmail_patch(imap)
|
||||
class << imap.instance_variable_get('@parser')
|
||||
|
||||
# Modified version of the original `msg_att` from here:
|
||||
# https://github.com/ruby/ruby/blob/1cc8ff001da217d0e98d13fe61fbc9f5547ef722/lib/net/imap.rb#L2346
|
||||
# rubocop:disable Style/RedundantReturn
|
||||
def msg_att(n)
|
||||
match(T_LPAR)
|
||||
attr = {}
|
||||
while true
|
||||
token = lookahead
|
||||
case token.symbol
|
||||
when T_RPAR
|
||||
shift_token
|
||||
break
|
||||
when T_SPACE
|
||||
shift_token
|
||||
next
|
||||
end
|
||||
case token.value
|
||||
when /\A(?:ENVELOPE)\z/ni
|
||||
name, val = envelope_data
|
||||
when /\A(?:FLAGS)\z/ni
|
||||
name, val = flags_data
|
||||
when /\A(?:INTERNALDATE)\z/ni
|
||||
name, val = internaldate_data
|
||||
when /\A(?:RFC822(?:\.HEADER|\.TEXT)?)\z/ni
|
||||
name, val = rfc822_text
|
||||
when /\A(?:RFC822\.SIZE)\z/ni
|
||||
name, val = rfc822_size
|
||||
when /\A(?:BODY(?:STRUCTURE)?)\z/ni
|
||||
name, val = body_data
|
||||
when /\A(?:UID)\z/ni
|
||||
name, val = uid_data
|
||||
when /\A(?:MODSEQ)\z/ni
|
||||
name, val = modseq_data
|
||||
# Adding support for GMail extended attributes.
|
||||
when /\A(?:X-GM-LABELS)\z/ni
|
||||
name, val = label_data
|
||||
when /\A(?:X-GM-MSGID)\z/ni
|
||||
name, val = uid_data
|
||||
when /\A(?:X-GM-THRID)\z/ni
|
||||
name, val = uid_data
|
||||
else
|
||||
parse_error("unknown attribute `%s' for {%d}", token.value, n)
|
||||
end
|
||||
attr[name] = val
|
||||
end
|
||||
return attr
|
||||
end
|
||||
|
||||
def label_data
|
||||
token = match(T_ATOM)
|
||||
name = token.value.upcase
|
||||
|
||||
match(T_SPACE)
|
||||
match(T_LPAR)
|
||||
|
||||
result = []
|
||||
while true
|
||||
token = lookahead
|
||||
case token.symbol
|
||||
when T_RPAR
|
||||
shift_token
|
||||
break
|
||||
when T_SPACE
|
||||
shift_token
|
||||
end
|
||||
|
||||
token = lookahead
|
||||
if string_token?(token)
|
||||
result.push(string)
|
||||
else
|
||||
result.push(atom)
|
||||
end
|
||||
end
|
||||
return name, result
|
||||
end
|
||||
# rubocop:enable Style/RedundantReturn
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
258
lib/imap/sync.rb
Normal file
258
lib/imap/sync.rb
Normal file
@ -0,0 +1,258 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'net/imap'
|
||||
|
||||
module Imap
|
||||
class Sync
|
||||
def self.for_group(group, opts = {})
|
||||
if group.imap_server == 'imap.gmail.com'
|
||||
opts[:provider] ||= Imap::Providers::Gmail
|
||||
end
|
||||
|
||||
Imap::Sync.new(group, opts)
|
||||
end
|
||||
|
||||
def initialize(group, opts = {})
|
||||
@group = group
|
||||
|
||||
provider_klass ||= opts[:provider] || Imap::Providers::Generic
|
||||
@provider = provider_klass.new(@group.imap_server,
|
||||
port: @group.imap_port,
|
||||
ssl: @group.imap_ssl,
|
||||
username: @group.email_username,
|
||||
password: @group.email_password
|
||||
)
|
||||
|
||||
connect!
|
||||
end
|
||||
|
||||
def connect!
|
||||
@provider.connect!
|
||||
end
|
||||
|
||||
def disconnect!
|
||||
@provider.disconnect!
|
||||
end
|
||||
|
||||
def disconnected?
|
||||
@provider.disconnected?
|
||||
end
|
||||
|
||||
def can_idle?
|
||||
SiteSetting.enable_imap_idle && @provider.can?('IDLE')
|
||||
end
|
||||
|
||||
def process(idle: false, import_limit: nil, old_emails_limit: nil, new_emails_limit: nil)
|
||||
raise 'disconnected' if disconnected?
|
||||
|
||||
import_limit ||= SiteSetting.imap_batch_import_email
|
||||
old_emails_limit ||= SiteSetting.imap_polling_old_emails
|
||||
new_emails_limit ||= SiteSetting.imap_polling_new_emails
|
||||
|
||||
# IMAP server -> Discourse (download): discovers updates to old emails
|
||||
# (synced emails) and fetches new emails.
|
||||
|
||||
# TODO: Use `Net::IMAP.encode_utf7(@group.imap_mailbox_name)`?
|
||||
@status = @provider.open_mailbox(@group.imap_mailbox_name)
|
||||
|
||||
if @status[:uid_validity] != @group.imap_uid_validity
|
||||
# If UID validity changes, the whole mailbox must be synchronized (all
|
||||
# emails are considered new and will be associated to existent topics
|
||||
# in Email::Reciever by matching Message-Ids).
|
||||
Rails.logger.warn("[IMAP] UIDVALIDITY = #{@status[:uid_validity]} does not match expected #{@group.imap_uid_validity}, invalidating IMAP cache and resyncing emails for group #{@group.name} and mailbox #{@group.imap_mailbox_name}")
|
||||
@group.imap_last_uid = 0
|
||||
end
|
||||
|
||||
if idle && !can_idle?
|
||||
Rails.logger.warn("[IMAP] IMAP server for group #{@group.name} cannot IDLE")
|
||||
idle = false
|
||||
end
|
||||
|
||||
if idle
|
||||
raise 'IMAP IDLE is disabled' if !SiteSetting.enable_imap_idle
|
||||
|
||||
# Thread goes into sleep and it is better to return any connection
|
||||
# back to the pool.
|
||||
ActiveRecord::Base.connection_handler.clear_active_connections!
|
||||
|
||||
@provider.imap.idle(SiteSetting.imap_polling_period_mins.minutes.to_i) do |resp|
|
||||
if resp.kind_of?(Net::IMAP::UntaggedResponse) && resp.name == 'EXISTS'
|
||||
@provider.imap.idle_done
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Fetching UIDs of old (already imported into Discourse, but might need
|
||||
# update) and new (not downloaded yet) emails.
|
||||
if @group.imap_last_uid == 0
|
||||
old_uids = []
|
||||
new_uids = @provider.uids
|
||||
else
|
||||
old_uids = @provider.uids(to: @group.imap_last_uid) # 1 .. seen
|
||||
new_uids = @provider.uids(from: @group.imap_last_uid + 1) # seen+1 .. inf
|
||||
end
|
||||
|
||||
# Sometimes, new_uids contains elements from old_uids.
|
||||
new_uids = new_uids - old_uids
|
||||
|
||||
Rails.logger.debug("[IMAP] Remote email server has #{old_uids.size} old emails and #{new_uids.size} new emails")
|
||||
|
||||
all_old_uids_size = old_uids.size
|
||||
all_new_uids_size = new_uids.size
|
||||
|
||||
@group.update_columns(
|
||||
imap_last_error: nil,
|
||||
imap_old_emails: all_old_uids_size,
|
||||
imap_new_emails: all_new_uids_size
|
||||
)
|
||||
|
||||
import_mode = import_limit > -1 && new_uids.size > import_limit
|
||||
old_uids = old_uids.sample(old_emails_limit).sort! if old_emails_limit > -1
|
||||
new_uids = new_uids[0..new_emails_limit - 1] if new_emails_limit > 0
|
||||
|
||||
if old_uids.present?
|
||||
Rails.logger.debug("[IMAP] Syncing #{old_uids.size} randomly-selected old emails")
|
||||
emails = @provider.emails(old_uids, ['UID', 'FLAGS', 'LABELS'], mailbox: @group.imap_mailbox_name)
|
||||
emails.each do |email|
|
||||
incoming_email = IncomingEmail.find_by(
|
||||
imap_uid_validity: @status[:uid_validity],
|
||||
imap_uid: email['UID']
|
||||
)
|
||||
|
||||
if incoming_email.present?
|
||||
update_topic(email, incoming_email, mailbox_name: @group.imap_mailbox_name)
|
||||
else
|
||||
Rails.logger.warn("[IMAP] Could not find old email (UIDVALIDITY = #{@status[:uid_validity]}, UID = #{email['UID']})")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if new_uids.present?
|
||||
Rails.logger.debug("[IMAP] Syncing #{new_uids.size} new emails (oldest first)")
|
||||
|
||||
emails = @provider.emails(new_uids, ['UID', 'FLAGS', 'LABELS', 'RFC822'], mailbox: @group.imap_mailbox_name)
|
||||
processed = 0
|
||||
|
||||
emails.each do |email|
|
||||
# Synchronously process emails because the order of emails matter
|
||||
# (for example replies must be processed after the original email
|
||||
# to have a topic where the reply can be posted).
|
||||
begin
|
||||
receiver = Email::Receiver.new(email['RFC822'],
|
||||
allow_auto_generated: true,
|
||||
import_mode: import_mode,
|
||||
destinations: [@group],
|
||||
uid_validity: @status[:uid_validity],
|
||||
uid: email['UID']
|
||||
)
|
||||
receiver.process!
|
||||
update_topic(email, receiver.incoming_email, mailbox_name: @group.imap_mailbox_name)
|
||||
rescue Email::Receiver::ProcessingError => e
|
||||
Rails.logger.warn("[IMAP] Could not process (UIDVALIDITY = #{@status[:uid_validity]}, UID = #{email['UID']}): #{e.message}")
|
||||
end
|
||||
|
||||
processed += 1
|
||||
@group.update_columns(
|
||||
imap_uid_validity: @status[:uid_validity],
|
||||
imap_last_uid: email['UID'],
|
||||
imap_old_emails: all_old_uids_size + processed,
|
||||
imap_new_emails: all_new_uids_size - processed
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
# Discourse -> IMAP server (upload): syncs updated flags and labels.
|
||||
if SiteSetting.enable_imap_write
|
||||
to_sync = IncomingEmail.where(imap_sync: true)
|
||||
if to_sync.size > 0
|
||||
@provider.open_mailbox(@group.imap_mailbox_name, write: true)
|
||||
to_sync.each do |incoming_email|
|
||||
Rails.logger.debug("[IMAP] Updating email for #{@group.name} and incoming email ID = #{incoming_email.id}")
|
||||
update_email(@group.imap_mailbox_name, incoming_email)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
{ remaining: all_new_uids_size - new_uids.size }
|
||||
end
|
||||
|
||||
def update_topic(email, incoming_email, opts = {})
|
||||
return if !incoming_email ||
|
||||
incoming_email.imap_sync ||
|
||||
!incoming_email.topic ||
|
||||
incoming_email.post&.post_number != 1
|
||||
|
||||
update_topic_archived_state(email, incoming_email, opts)
|
||||
update_topic_tags(email, incoming_email, opts)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def update_topic_archived_state(email, incoming_email, opts = {})
|
||||
topic = incoming_email.topic
|
||||
|
||||
topic_is_archived = topic.group_archived_messages.size > 0
|
||||
email_is_archived = !email['LABELS'].include?('\\Inbox') && !email['LABELS'].include?('INBOX')
|
||||
|
||||
if topic_is_archived && !email_is_archived
|
||||
GroupArchivedMessage.move_to_inbox!(@group.id, topic, skip_imap_sync: true)
|
||||
elsif !topic_is_archived && email_is_archived
|
||||
GroupArchivedMessage.archive!(@group.id, topic, skip_imap_sync: true)
|
||||
end
|
||||
end
|
||||
|
||||
def update_topic_tags(email, incoming_email, opts = {})
|
||||
group_email_regex = @group.email_username_regex
|
||||
topic = incoming_email.topic
|
||||
|
||||
tags = Set.new
|
||||
|
||||
# "Plus" part from the destination email address
|
||||
to_addresses = incoming_email.to_addresses&.split(";") || []
|
||||
cc_addresses = incoming_email.cc_addresses&.split(";") || []
|
||||
(to_addresses + cc_addresses).each do |address|
|
||||
if plus_part = address&.scan(group_email_regex)&.first&.first
|
||||
tags.add("plus:#{plus_part[1..-1]}") if plus_part.length > 0
|
||||
end
|
||||
end
|
||||
|
||||
# Mailbox name
|
||||
tags.add(@provider.to_tag(opts[:mailbox_name])) if opts[:mailbox_name]
|
||||
|
||||
# Flags and labels
|
||||
email['FLAGS'].each { |flag| tags.add(@provider.to_tag(flag)) }
|
||||
email['LABELS'].each { |label| tags.add(@provider.to_tag(label)) }
|
||||
|
||||
tags.subtract([nil, ''])
|
||||
|
||||
# TODO: Optimize tagging.
|
||||
# `DiscourseTagging.tag_topic_by_names` does a lot of lookups in the
|
||||
# database and some of them could be cached in this context.
|
||||
DiscourseTagging.tag_topic_by_names(topic, Guardian.new(Discourse.system_user), tags.to_a)
|
||||
end
|
||||
|
||||
def update_email(mailbox_name, incoming_email)
|
||||
return if !SiteSetting.tagging_enabled || !SiteSetting.allow_staff_to_tag_pms
|
||||
return if incoming_email&.post&.post_number != 1 || !incoming_email.imap_sync
|
||||
return unless email = @provider.emails(incoming_email.imap_uid, ['FLAGS', 'LABELS'], mailbox: mailbox_name).first
|
||||
incoming_email.update(imap_sync: false)
|
||||
|
||||
labels = email['LABELS']
|
||||
flags = email['FLAGS']
|
||||
topic = incoming_email.topic
|
||||
|
||||
# TODO: Delete remote email if topic no longer exists
|
||||
# new_flags << Net::IMAP::DELETED if !incoming_email.topic
|
||||
return if !topic
|
||||
|
||||
# Sync topic status and labels with email flags and labels.
|
||||
tags = topic.tags.pluck(:name)
|
||||
new_flags = tags.map { |tag| @provider.tag_to_flag(tag) }.reject(&:blank?)
|
||||
# new_flags << Net::IMAP::DELETED if !incoming_email.topic
|
||||
new_labels = tags.map { |tag| @provider.tag_to_label(tag) }.reject(&:blank?)
|
||||
new_labels << '\\Inbox' if topic.group_archived_messages.length == 0
|
||||
@provider.store(incoming_email.imap_uid, 'FLAGS', flags, new_flags)
|
||||
@provider.store(incoming_email.imap_uid, 'LABELS', labels, new_labels)
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -186,10 +186,8 @@ class PostCreator
|
||||
@post.link_post_uploads
|
||||
update_uploads_secure_status
|
||||
ensure_in_allowed_users if guardian.is_staff?
|
||||
unarchive_message
|
||||
if !@opts[:import_mode]
|
||||
DraftSequence.next!(@user, draft_key)
|
||||
end
|
||||
unarchive_message if !@opts[:import_mode]
|
||||
DraftSequence.next!(@user, draft_key) if !@opts[:import_mode]
|
||||
@post.save_reply_relationships
|
||||
end
|
||||
end
|
||||
|
||||
@ -158,6 +158,10 @@ class PostRevisor
|
||||
@skip_revision = false
|
||||
@skip_revision = @opts[:skip_revision] if @opts.has_key?(:skip_revision)
|
||||
|
||||
if @post.incoming_email&.imap_uid
|
||||
@post.incoming_email&.update(imap_sync: true)
|
||||
end
|
||||
|
||||
old_raw = @post.raw
|
||||
|
||||
Post.transaction do
|
||||
|
||||
@ -60,12 +60,12 @@ describe Email::Processor do
|
||||
|
||||
it "enqueues a background job by default" do
|
||||
Jobs.expects(:enqueue).with(:process_email, mail: mail)
|
||||
Email::Processor.process!(mail)
|
||||
Email::Processor.process!(mail, retry_on_rate_limit: true)
|
||||
end
|
||||
|
||||
it "doesn't enqueue a background job when retry is disabled" do
|
||||
Jobs.expects(:enqueue).with(:process_email, mail: mail).never
|
||||
expect { Email::Processor.process!(mail, false) }.to raise_error(limit_exceeded)
|
||||
expect { Email::Processor.process!(mail, retry_on_rate_limit: false) }.to raise_error(limit_exceeded)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
@ -1153,9 +1153,7 @@ describe Email::Receiver do
|
||||
|
||||
dest = Email::Receiver.check_address('foo+4f97315cc828096c9cb34c6f1a0d6fe8@bar.com')
|
||||
|
||||
expect(dest).to be_present
|
||||
expect(dest[:type]).to eq(:reply)
|
||||
expect(dest[:obj]).to eq(post_reply_key)
|
||||
expect(dest).to eq(post_reply_key)
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -1587,4 +1585,78 @@ describe Email::Receiver do
|
||||
expect { Email::Receiver.new(email).process! }.to raise_error(Email::Receiver::ReplyToDigestError)
|
||||
end
|
||||
end
|
||||
|
||||
context "find_related_post" do
|
||||
|
||||
let(:user) { Fabricate(:user) }
|
||||
let(:group) { Fabricate(:group, users: [user]) }
|
||||
|
||||
let (:email_1) { <<~EOF
|
||||
MIME-Version: 1.0
|
||||
Date: Wed, 01 Jan 2019 12:00:00 +0200
|
||||
Message-ID: <7aN1uwcokt2xkfG3iYrpKmiuVhy4w9b5@mail.gmail.com>
|
||||
Subject: Lorem ipsum dolor sit amet
|
||||
From: Dan Ungureanu <dan@discourse.org>
|
||||
To: team-test@discourse.org
|
||||
Content-Type: text/plain; charset="UTF-8"
|
||||
|
||||
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas
|
||||
semper, erat tempor sodales commodo, mi diam tempus lorem, in vehicula
|
||||
leo libero quis lacus. Nullam justo nunc, sagittis nec metus placerat,
|
||||
auctor condimentum neque. Sed risus purus, fermentum eget purus
|
||||
porttitor, finibus efficitur orci. Integer tempus mi nec odio
|
||||
elementum pulvinar. Pellentesque sed fringilla nulla, ac mollis quam.
|
||||
Vivamus semper lacinia scelerisque. Cras urna magna, porttitor nec
|
||||
libero quis, congue viverra sapien. Nulla sodales ac tellus a
|
||||
suscipit.
|
||||
EOF
|
||||
}
|
||||
|
||||
let (:post_2) {
|
||||
incoming_email = IncomingEmail.find_by(message_id: "7aN1uwcokt2xkfG3iYrpKmiuVhy4w9b5@mail.gmail.com")
|
||||
|
||||
PostCreator.create(user,
|
||||
raw: "Vestibulum rutrum tortor vitae arcu varius, non vestibulum ipsum tempor. Integer nibh libero, dignissim eu velit vel, interdum posuere mi. Aliquam erat volutpat. Pellentesque id nulla ultricies, eleifend ipsum non, fringilla purus. Aliquam pretium dolor lobortis urna volutpat, vel consectetur arcu porta. In non erat quis nibh gravida pharetra consequat vel risus. Aliquam rutrum consectetur est ac posuere. Praesent mattis nunc risus, a molestie lectus accumsan porta.",
|
||||
topic_id: incoming_email.topic_id
|
||||
)
|
||||
}
|
||||
|
||||
let (:email_3) { <<~EOF
|
||||
MIME-Version: 1.0
|
||||
Date: Wed, 01 Jan 2019 12:00:00 +0200
|
||||
References: <7aN1uwcokt2xkfG3iYrpKmiuVhy4w9b5@mail.gmail.com> <topic/#{post_2.topic_id}/#{post_2.id}@test.localhost>
|
||||
In-Reply-To: <topic/#{post_2.topic_id}/#{post_2.id}@test.localhost>
|
||||
Message-ID: <w1vdxT8ebJjZQQp7XyDdEJaSscE9qRjr@mail.gmail.com>
|
||||
Subject: Re: Lorem ipsum dolor sit amet
|
||||
From: Dan Ungureanu <dan@discourse.org>
|
||||
To: team-test@discourse.org
|
||||
Content-Type: text/plain; charset="UTF-8"
|
||||
|
||||
Integer mattis suscipit facilisis. Ut ullamcorper libero at faucibus
|
||||
sodales. Ut suscipit elit ac dui porta consequat. Suspendisse potenti.
|
||||
Nam ut accumsan dui, eget commodo sapien. Etiam ultrices elementum
|
||||
cursus. Vivamus et diam et orci lobortis porttitor. Aliquam
|
||||
scelerisque ex a imperdiet ornare. Donec interdum laoreet posuere.
|
||||
Nulla sagittis, velit id posuere sollicitudin, elit nunc laoreet
|
||||
libero, vitae aliquet tortor eros at est. Donec vitae massa vehicula,
|
||||
aliquet libero non, porttitor ipsum. Phasellus pellentesque sodales
|
||||
lacus eu sagittis. Aliquam ut condimentum nisi. Nulla in placerat
|
||||
felis. Sed pellentesque, massa auctor venenatis gravida, risus lorem
|
||||
iaculis mi, at hendrerit nisi turpis sit amet metus. Nulla egestas
|
||||
ante eget nisi luctus consectetur.
|
||||
EOF
|
||||
}
|
||||
|
||||
def receive(email_string)
|
||||
Email::Receiver.new(email_string,
|
||||
destinations: [group]
|
||||
).process!
|
||||
end
|
||||
|
||||
it "makes all posts in same topic" do
|
||||
expect { receive(email_1) }.to change { Topic.count }.by(1).and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
|
||||
expect { post_2 }.to change { Topic.count }.by(0).and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
|
||||
expect { receive(email_3) }.to change { Topic.count }.by(0).and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
28
spec/components/imap/imap_helper.rb
Normal file
28
spec/components/imap/imap_helper.rb
Normal file
@ -0,0 +1,28 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class MockedImapProvider < Imap::Providers::Gmail
|
||||
def connect!; end
|
||||
def disconnect!; end
|
||||
def open_mailbox(mailbox_name, write: false); end
|
||||
|
||||
def labels
|
||||
['INBOX']
|
||||
end
|
||||
end
|
||||
|
||||
def EmailFabricator(options)
|
||||
email = +''
|
||||
email += "Date: Sat, 31 Mar 2018 17:50:19 -0700\n"
|
||||
email += "From: #{options[:from] || "Dan <dan@discourse.org>"}\n"
|
||||
email += "To: #{options[:to] || "Joffrey <joffrey@discourse.org>"}\n"
|
||||
email += "Cc: #{options[:cc]}\n" if options[:cc]
|
||||
email += "In-Reply-To: #{options[:in_reply_to]}\n" if options[:in_reply_to]
|
||||
email += "References: #{options[:in_reply_to]}\n" if options[:in_reply_to]
|
||||
email += "Message-ID: #{options[:message_id]}\n" if options[:message_id]
|
||||
email += "Subject: #{options[:subject] || "This is a test email subhect"}\n"
|
||||
email += "Mime-Version: 1.0\n"
|
||||
email += "Content-Type: #{options[:content_type] || "text/plain;\n charset=UTF-8"}\n"
|
||||
email += "Content-Transfer-Encoding: 7bit\n"
|
||||
email += "\n#{options[:body] || "This is an email *body*. :smile:"}"
|
||||
email
|
||||
end
|
||||
333
spec/components/imap/sync_spec.rb
Normal file
333
spec/components/imap/sync_spec.rb
Normal file
@ -0,0 +1,333 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'rails_helper'
|
||||
require 'imap/sync'
|
||||
require_relative 'imap_helper'
|
||||
|
||||
describe Imap::Sync do
|
||||
|
||||
before do
|
||||
SiteSetting.tagging_enabled = true
|
||||
SiteSetting.allow_staff_to_tag_pms = true
|
||||
|
||||
SiteSetting.enable_imap = true
|
||||
|
||||
Jobs.run_immediately!
|
||||
end
|
||||
|
||||
let(:group) do
|
||||
Fabricate(
|
||||
:group,
|
||||
imap_server: 'imap.gmail.com',
|
||||
imap_port: 993,
|
||||
email_username: 'discourse@example.com',
|
||||
email_password: 'discourse@example.com',
|
||||
imap_mailbox_name: '[Gmail]/All Mail'
|
||||
)
|
||||
end
|
||||
|
||||
let(:sync_handler) { Imap::Sync.new(group, provider: MockedImapProvider) }
|
||||
|
||||
context 'no previous sync' do
|
||||
let(:from) { 'john@free.fr' }
|
||||
let(:subject) { 'Testing email post' }
|
||||
let(:message_id) { "#{SecureRandom.hex}@example.com" }
|
||||
|
||||
let(:email) do
|
||||
EmailFabricator(
|
||||
from: from,
|
||||
to: group.email_username,
|
||||
subject: subject,
|
||||
message_id: message_id)
|
||||
end
|
||||
|
||||
before do
|
||||
provider = MockedImapProvider.any_instance
|
||||
provider.stubs(:open_mailbox).returns(uid_validity: 1)
|
||||
provider.stubs(:uids).with.returns([100])
|
||||
provider.stubs(:uids).with(to: 100).returns([100])
|
||||
provider.stubs(:uids).with(from: 101).returns([])
|
||||
provider.stubs(:emails).returns(
|
||||
[
|
||||
{
|
||||
'UID' => 100,
|
||||
'LABELS' => %w[\\Important test-label],
|
||||
'FLAGS' => %i[Seen],
|
||||
'RFC822' => email
|
||||
}
|
||||
]
|
||||
)
|
||||
end
|
||||
|
||||
it 'creates a topic from an incoming email' do
|
||||
expect { sync_handler.process }
|
||||
.to change { Topic.count }.by(1)
|
||||
.and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
|
||||
.and change { IncomingEmail.count }.by(1)
|
||||
|
||||
expect(group.imap_uid_validity).to eq(1)
|
||||
expect(group.imap_last_uid).to eq(100)
|
||||
|
||||
topic = Topic.last
|
||||
expect(topic.title).to eq(subject)
|
||||
expect(topic.user.email).to eq(from)
|
||||
expect(topic.tags.pluck(:name)).to eq(%w[seen important test-label])
|
||||
|
||||
post = topic.first_post
|
||||
expect(post.raw).to eq('This is an email *body*. :smile:')
|
||||
|
||||
incoming_email = post.incoming_email
|
||||
expect(incoming_email.raw.lines.map(&:strip)).to eq(email.lines.map(&:strip))
|
||||
expect(incoming_email.message_id).to eq(message_id)
|
||||
expect(incoming_email.from_address).to eq(from)
|
||||
expect(incoming_email.to_addresses).to eq(group.email_username)
|
||||
expect(incoming_email.imap_uid_validity).to eq(1)
|
||||
expect(incoming_email.imap_uid).to eq(100)
|
||||
expect(incoming_email.imap_sync).to eq(false)
|
||||
end
|
||||
|
||||
it 'does not duplicate topics' do
|
||||
expect { sync_handler.process }
|
||||
.to change { Topic.count }.by(1)
|
||||
.and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
|
||||
.and change { IncomingEmail.count }.by(1)
|
||||
|
||||
expect { sync_handler.process }
|
||||
.to change { Topic.count }.by(0)
|
||||
.and change { Post.where(post_type: Post.types[:regular]).count }.by(0)
|
||||
.and change { IncomingEmail.count }.by(0)
|
||||
end
|
||||
|
||||
it 'does not duplicate incoming emails' do
|
||||
incoming_email = Fabricate(:incoming_email, message_id: message_id)
|
||||
|
||||
expect { sync_handler.process }
|
||||
.to change { Topic.count }.by(0)
|
||||
.and change { Post.where(post_type: Post.types[:regular]).count }.by(0)
|
||||
.and change { IncomingEmail.count }.by(0)
|
||||
|
||||
incoming_email.reload
|
||||
expect(incoming_email.message_id).to eq(message_id)
|
||||
expect(incoming_email.imap_uid_validity).to eq(1)
|
||||
expect(incoming_email.imap_uid).to eq(100)
|
||||
expect(incoming_email.imap_sync).to eq(false)
|
||||
end
|
||||
end
|
||||
|
||||
context 'previous sync' do
|
||||
let(:subject) { 'Testing email post' }
|
||||
|
||||
let(:first_from) { 'john@free.fr' }
|
||||
let(:first_message_id) { SecureRandom.hex }
|
||||
let(:first_body) { 'This is the first message of this exchange.' }
|
||||
|
||||
let(:second_from) { 'sam@free.fr' }
|
||||
let(:second_message_id) { SecureRandom.hex }
|
||||
let(:second_body) { '<p>This is an <b>answer</b> to this message.</p>' }
|
||||
|
||||
it 'continues with new emails' do
|
||||
provider = MockedImapProvider.any_instance
|
||||
provider.stubs(:open_mailbox).returns(uid_validity: 1)
|
||||
|
||||
provider.stubs(:uids).with.returns([100])
|
||||
provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
|
||||
[
|
||||
{
|
||||
'UID' => 100,
|
||||
'LABELS' => %w[\\Inbox],
|
||||
'FLAGS' => %i[Seen],
|
||||
'RFC822' => EmailFabricator(
|
||||
message_id: first_message_id,
|
||||
from: first_from,
|
||||
to: group.email_username,
|
||||
cc: second_from,
|
||||
subject: subject,
|
||||
body: first_body
|
||||
)
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
expect { sync_handler.process }
|
||||
.to change { Topic.count }.by(1)
|
||||
.and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
|
||||
.and change { IncomingEmail.count }.by(1)
|
||||
|
||||
topic = Topic.last
|
||||
expect(topic.title).to eq(subject)
|
||||
expect(GroupArchivedMessage.where(topic_id: topic.id).exists?).to eq(false)
|
||||
|
||||
post = Post.where(post_type: Post.types[:regular]).last
|
||||
expect(post.user.email).to eq(first_from)
|
||||
expect(post.raw).to eq(first_body)
|
||||
expect(group.imap_uid_validity).to eq(1)
|
||||
expect(group.imap_last_uid).to eq(100)
|
||||
|
||||
provider.stubs(:uids).with(to: 100).returns([100])
|
||||
provider.stubs(:uids).with(from: 101).returns([200])
|
||||
provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS'], anything).returns(
|
||||
[
|
||||
{
|
||||
'UID' => 100,
|
||||
'LABELS' => %w[\\Inbox],
|
||||
'FLAGS' => %i[Seen]
|
||||
}
|
||||
]
|
||||
)
|
||||
provider.stubs(:emails).with([200], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
|
||||
[
|
||||
{
|
||||
'UID' => 200,
|
||||
'LABELS' => %w[\\Inbox],
|
||||
'FLAGS' => %i[Recent],
|
||||
'RFC822' => EmailFabricator(
|
||||
message_id: SecureRandom.hex,
|
||||
in_reply_to: first_message_id,
|
||||
from: second_from,
|
||||
to: group.email_username,
|
||||
subject: "Re: #{subject}",
|
||||
body: second_body
|
||||
)
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
expect { sync_handler.process }
|
||||
.to change { Topic.count }.by(0)
|
||||
.and change { Post.where(post_type: Post.types[:regular]).count }.by(1)
|
||||
.and change { IncomingEmail.count }.by(1)
|
||||
|
||||
post = Post.where(post_type: Post.types[:regular]).last
|
||||
expect(post.user.email).to eq(second_from)
|
||||
expect(post.raw).to eq(second_body)
|
||||
expect(group.imap_uid_validity).to eq(1)
|
||||
expect(group.imap_last_uid).to eq(200)
|
||||
|
||||
provider.stubs(:uids).with(to: 200).returns([100, 200])
|
||||
provider.stubs(:uids).with(from: 201).returns([])
|
||||
provider.stubs(:emails).with([100, 200], ['UID', 'FLAGS', 'LABELS'], anything).returns(
|
||||
[
|
||||
{
|
||||
'UID' => 100,
|
||||
'LABELS' => %w[],
|
||||
'FLAGS' => %i[Seen]
|
||||
},
|
||||
{
|
||||
'UID' => 200,
|
||||
'LABELS' => %w[],
|
||||
'FLAGS' => %i[Recent],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
expect { sync_handler.process }
|
||||
.to change { Topic.count }.by(0)
|
||||
.and change { Post.where(post_type: Post.types[:regular]).count }.by(0)
|
||||
.and change { IncomingEmail.count }.by(0)
|
||||
|
||||
topic = Topic.last
|
||||
expect(topic.title).to eq(subject)
|
||||
expect(GroupArchivedMessage.where(topic_id: topic.id).exists?).to eq(true)
|
||||
|
||||
expect(Topic.last.posts.where(post_type: Post.types[:regular]).count).to eq(2)
|
||||
end
|
||||
end
|
||||
|
||||
context 'invaidated previous sync' do
|
||||
let(:subject) { 'Testing email post' }
|
||||
|
||||
let(:first_from) { 'john@free.fr' }
|
||||
let(:first_message_id) { SecureRandom.hex }
|
||||
let(:first_body) { 'This is the first message of this exchange.' }
|
||||
|
||||
let(:second_from) { 'sam@free.fr' }
|
||||
let(:second_message_id) { SecureRandom.hex }
|
||||
let(:second_body) { '<p>This is an <b>answer</b> to this message.</p>' }
|
||||
|
||||
it 'is updated' do
|
||||
provider = MockedImapProvider.any_instance
|
||||
|
||||
provider.stubs(:open_mailbox).returns(uid_validity: 1)
|
||||
provider.stubs(:uids).with.returns([100, 200])
|
||||
provider.stubs(:emails).with([100, 200], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
|
||||
[
|
||||
{
|
||||
'UID' => 100,
|
||||
'LABELS' => %w[\\Inbox],
|
||||
'FLAGS' => %i[Seen],
|
||||
'RFC822' => EmailFabricator(
|
||||
message_id: first_message_id,
|
||||
from: first_from,
|
||||
to: group.email_username,
|
||||
cc: second_from,
|
||||
subject: subject,
|
||||
body: first_body
|
||||
)
|
||||
},
|
||||
{
|
||||
'UID' => 200,
|
||||
'LABELS' => %w[\\Inbox],
|
||||
'FLAGS' => %i[Recent],
|
||||
'RFC822' => EmailFabricator(
|
||||
message_id: second_message_id,
|
||||
in_reply_to: first_message_id,
|
||||
from: second_from,
|
||||
to: group.email_username,
|
||||
subject: "Re: #{subject}",
|
||||
body: second_body
|
||||
)
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
expect { sync_handler.process }
|
||||
.to change { Topic.count }.by(1)
|
||||
.and change { Post.where(post_type: Post.types[:regular]).count }.by(2)
|
||||
.and change { IncomingEmail.count }.by(2)
|
||||
|
||||
imap_data = Topic.last.incoming_email.pluck(:imap_uid_validity, :imap_uid)
|
||||
expect(imap_data).to contain_exactly([1, 100], [1, 200])
|
||||
|
||||
provider.stubs(:open_mailbox).returns(uid_validity: 2)
|
||||
provider.stubs(:uids).with.returns([111, 222])
|
||||
provider.stubs(:emails).with([111, 222], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
|
||||
[
|
||||
{
|
||||
'UID' => 111,
|
||||
'LABELS' => %w[\\Inbox],
|
||||
'FLAGS' => %i[Seen],
|
||||
'RFC822' => EmailFabricator(
|
||||
message_id: first_message_id,
|
||||
from: first_from,
|
||||
to: group.email_username,
|
||||
cc: second_from,
|
||||
subject: subject,
|
||||
body: first_body
|
||||
)
|
||||
},
|
||||
{
|
||||
'UID' => 222,
|
||||
'LABELS' => %w[\\Inbox],
|
||||
'FLAGS' => %i[Recent],
|
||||
'RFC822' => EmailFabricator(
|
||||
message_id: second_message_id,
|
||||
in_reply_to: first_message_id,
|
||||
from: second_from,
|
||||
to: group.email_username,
|
||||
subject: "Re: #{subject}",
|
||||
body: second_body
|
||||
)
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
expect { sync_handler.process }
|
||||
.to change { Topic.count }.by(0)
|
||||
.and change { Post.where(post_type: Post.types[:regular]).count }.by(0)
|
||||
.and change { IncomingEmail.count }.by(0)
|
||||
|
||||
imap_data = Topic.last.incoming_email.pluck(:imap_uid_validity, :imap_uid)
|
||||
expect(imap_data).to contain_exactly([2, 111], [2, 222])
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -7,7 +7,7 @@ describe Jobs::ProcessEmail do
|
||||
let(:mail) { "From: foo@bar.com\nTo: bar@foo.com\nSubject: FOO BAR\n\nFoo foo bar bar?" }
|
||||
|
||||
it "process an email without retry" do
|
||||
Email::Processor.expects(:process!).with(mail, false)
|
||||
Email::Processor.expects(:process!).with(mail, retry_on_rate_limit: false)
|
||||
Jobs::ProcessEmail.new.execute(mail: mail)
|
||||
end
|
||||
|
||||
|
||||
76
spec/mailers/group_smtp_mailer_spec.rb
Normal file
76
spec/mailers/group_smtp_mailer_spec.rb
Normal file
@ -0,0 +1,76 @@
|
||||
# frozen_string_literal: true
|
||||
require 'rails_helper'
|
||||
require 'email/receiver'
|
||||
|
||||
describe GroupSmtpMailer do
|
||||
let(:group) {
|
||||
Fabricate(:group,
|
||||
name: 'Testers',
|
||||
title: 'Tester',
|
||||
full_name: 'Testers Group',
|
||||
smtp_server: 'smtp.gmail.com',
|
||||
smtp_port: 587,
|
||||
smtp_ssl: true,
|
||||
imap_server: 'imap.gmail.com',
|
||||
imap_port: 993,
|
||||
imap_ssl: true,
|
||||
email_username: 'bugs@gmail.com',
|
||||
email_password: 'super$secret$password'
|
||||
)
|
||||
}
|
||||
|
||||
let(:user) {
|
||||
user = Fabricate(:user)
|
||||
group.add_owner(user)
|
||||
user
|
||||
}
|
||||
|
||||
let(:email) {
|
||||
<<~EOF
|
||||
Delivered-To: bugs@gmail.com
|
||||
MIME-Version: 1.0
|
||||
From: John Doe <john@doe.com>
|
||||
Date: Tue, 01 Jan 2019 12:00:00 +0200
|
||||
Message-ID: <a52f67a3d3560f2a35276cda8519b10b595623bcb66912bb92df6651ad5f75be@mail.gmail.com>
|
||||
Subject: Hello from John
|
||||
To: "bugs@gmail.com" <bugs@gmail.com>
|
||||
Content-Type: text/plain; charset="UTF-8"
|
||||
|
||||
Hello,
|
||||
|
||||
How are you doing?
|
||||
EOF
|
||||
}
|
||||
|
||||
let(:receiver) {
|
||||
receiver = Email::Receiver.new(email,
|
||||
destinations: [group],
|
||||
uid_validity: 1,
|
||||
uid: 10000
|
||||
)
|
||||
receiver.process!
|
||||
receiver
|
||||
}
|
||||
|
||||
let(:raw) { 'hello, how are you doing?' }
|
||||
|
||||
before do
|
||||
SiteSetting.enable_smtp = true
|
||||
Jobs.run_immediately!
|
||||
end
|
||||
|
||||
it 'sends an email as reply' do
|
||||
post = PostCreator.create(user,
|
||||
topic_id: receiver.incoming_email.topic.id,
|
||||
raw: raw
|
||||
)
|
||||
|
||||
expect(ActionMailer::Base.deliveries.size).to eq(1)
|
||||
|
||||
sent_mail = ActionMailer::Base.deliveries[0]
|
||||
expect(sent_mail.to).to contain_exactly('john@doe.com')
|
||||
expect(sent_mail.subject).to eq('Re: Hello from John')
|
||||
expect(sent_mail.to_s).to include(raw)
|
||||
end
|
||||
|
||||
end
|
||||
@ -113,4 +113,26 @@ describe BasicGroupSerializer do
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe 'admin only fields' do
|
||||
fab!(:group) { Fabricate(:group, email_username: 'foo@bar.com', email_password: 'pa$$w0rd') }
|
||||
|
||||
describe 'for a user' do
|
||||
let(:guardian) { Guardian.new(Fabricate(:user)) }
|
||||
|
||||
it 'are not visible' do
|
||||
expect(subject.as_json[:email_username]).to be_nil
|
||||
expect(subject.as_json[:email_password]).to be_nil
|
||||
end
|
||||
end
|
||||
|
||||
describe 'for an admin' do
|
||||
let(:guardian) { Guardian.new(Fabricate(:admin)) }
|
||||
|
||||
it 'are visible' do
|
||||
expect(subject.as_json[:email_username]).to eq('foo@bar.com')
|
||||
expect(subject.as_json[:email_password]).to eq('pa$$w0rd')
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@ -1165,4 +1165,76 @@ describe PostAlerter do
|
||||
expect(Notification.last.notification_type).to eq(Notification.types[:posted])
|
||||
end
|
||||
end
|
||||
|
||||
context "SMTP" do
|
||||
before do
|
||||
SiteSetting.enable_smtp = true
|
||||
Jobs.run_immediately!
|
||||
end
|
||||
|
||||
fab!(:group) do
|
||||
Fabricate(
|
||||
:group,
|
||||
smtp_server: "imap.gmail.com",
|
||||
smtp_port: 587,
|
||||
email_username: "discourse@example.com",
|
||||
email_password: "discourse@example.com"
|
||||
)
|
||||
end
|
||||
|
||||
fab!(:topic) do
|
||||
Fabricate(:private_message_topic,
|
||||
topic_allowed_groups: [
|
||||
Fabricate.build(:topic_allowed_group, group: group)
|
||||
]
|
||||
)
|
||||
end
|
||||
|
||||
it "sends notifications for new posts in topic" do
|
||||
post = Fabricate(
|
||||
:post,
|
||||
topic: topic,
|
||||
incoming_email:
|
||||
Fabricate(
|
||||
:incoming_email,
|
||||
topic: topic,
|
||||
from_address: "foo@discourse.org",
|
||||
to_addresses: group.email_username,
|
||||
cc_addresses: "bar@discourse.org"
|
||||
)
|
||||
)
|
||||
expect { PostAlerter.new.after_save_post(post, true) }.to change { ActionMailer::Base.deliveries.size }.by(0)
|
||||
|
||||
post = Fabricate(:post, topic: topic)
|
||||
expect { PostAlerter.new.after_save_post(post, true) }.to change { ActionMailer::Base.deliveries.size }.by(1)
|
||||
email = ActionMailer::Base.deliveries.last
|
||||
expect(email.from).to include(group.email_username)
|
||||
expect(email.to).to contain_exactly("foo@discourse.org", "bar@discourse.org")
|
||||
expect(email.subject).to eq("Re: #{topic.title}")
|
||||
|
||||
post = Fabricate(
|
||||
:post,
|
||||
topic: topic,
|
||||
incoming_email:
|
||||
Fabricate(
|
||||
:incoming_email,
|
||||
topic: topic,
|
||||
from_address: "bar@discourse.org",
|
||||
to_addresses: group.email_username,
|
||||
cc_addresses: "baz@discourse.org"
|
||||
)
|
||||
)
|
||||
expect { PostAlerter.new.after_save_post(post, true) }.to change { ActionMailer::Base.deliveries.size }.by(0)
|
||||
|
||||
post = Fabricate(:post, topic: topic.reload)
|
||||
expect { PostAlerter.new.after_save_post(post, true) }.to change { ActionMailer::Base.deliveries.size }.by(1)
|
||||
email = ActionMailer::Base.deliveries.last
|
||||
expect(email.from).to eq([group.email_username])
|
||||
expect(email.to).to contain_exactly("foo@discourse.org", "bar@discourse.org", "baz@discourse.org")
|
||||
expect(email.subject).to eq("Re: #{topic.title}")
|
||||
|
||||
post = Fabricate(:post, topic: topic, post_type: Post.types[:whisper])
|
||||
expect { PostAlerter.new.after_save_post(post, true) }.to change { ActionMailer::Base.deliveries.size }.by(0)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user