FEATURE: IMAP delete email sync for group inboxes (#10392)

Adds functionality to reflect topic delete in Discourse to IMAP inbox (Gmail only for now) and reflecting Gmail deletes in Discourse.

Adding lots of tests, various refactors and code improvements.

When Discourse topic is destroyed in PostDestroyer mark the topic incoming email as imap_sync: true, and do the opposite when post is recovered.

app/controllers/webhooks_controller.rb

@@ -13,7 +13,7 @@ class WebhooksController < ActionController::Base
   def sendgrid
     events = params["_json"] || [params]
     events.each do |event|
-      message_id = (event["smtp-id"] || "").tr("<>", "")
+      message_id = Email.message_id_clean((event["smtp-id"] || ""))
       to_address = event["email"]
       if event["event"] == "bounce"
         if event["status"]["4."]
@@ -150,7 +150,7 @@ class WebhooksController < ActionController::Base
     return mailgun_failure unless valid_mailgun_signature?(params["token"], params["timestamp"], params["signature"])
 
     event = params["event"]
-    message_id = params["Message-Id"].tr("<>", "")
+    message_id = Email.message_id_clean(params["Message-Id"])
     to_address = params["recipient"]
 
     # only handle soft bounces, because hard bounces are also handled

lib/email.rb

@@ -3,6 +3,8 @@
 require 'mail'
 
 module Email
+  # cute little guy ain't he?
+  MESSAGE_ID_REGEX = /<(.*@.*)+>/
 
   def self.is_valid?(email)
     return false unless String === email
@@ -39,4 +41,14 @@ module Email
     SiteSetting.email_site_title.presence || SiteSetting.title
   end
 
+  # https://tools.ietf.org/html/rfc850#section-2.1.7
+  def self.message_id_rfc_format(message_id)
+    return message_id if message_id =~ MESSAGE_ID_REGEX
+    "<#{message_id}>"
+  end
+
+  def self.message_id_clean(message_id)
+    return message_id if !(message_id =~ MESSAGE_ID_REGEX)
+    message_id.tr("<>", "")
+  end
 end

lib/email/receiver.rb

@@ -988,7 +988,9 @@ module Email
       if Array === references
         references
       elsif references.present?
-        references.split(/[\s,]/).map { |r| r.tr("<>", "") }
+        references.split(/[\s,]/).map do |r|
+          Email.message_id_clean(r)
+        end
       end
     end
 

lib/imap/providers/generic.rb

@@ -6,6 +6,19 @@ module Imap
   module Providers
     class WriteDisabledError < StandardError; end
 
+    class TrashedMailResponse
+      attr_accessor :trashed_emails, :trash_uid_validity
+    end
+
+    class BasicMail
+      attr_accessor :uid, :message_id
+
+      def initialize(uid: nil, message_id: nil)
+        @uid = uid
+        @message_id = message_id
+      end
+    end
+
     class Generic
       def initialize(server, options = {})
         @server = server
@@ -16,6 +29,10 @@ module Imap
         @timeout = options[:timeout] || 10
       end
 
+      def account_digest
+        @account_digest ||= Digest::MD5.hexdigest("#{@username}:#{@server}")
+      end
+
       def imap
         @imap ||= Net::IMAP.new(@server, port: @port, ssl: @ssl, open_timeout: @timeout)
       end
@@ -121,9 +138,27 @@ module Imap
         tag
       end
 
-      def list_mailboxes
+      def list_mailboxes(attr_filter = nil)
-        imap.list('', '*').map do |m|
+        # Basically, list all mailboxes in the root of the server.
-          next if m.attr.include?(:Noselect)
+        # ref: https://tools.ietf.org/html/rfc3501#section-6.3.8
+        imap.list('', '*').reject do |m|
+
+          # Noselect cannot be selected with the SELECT command.
+          # technically we could use this for readonly mode when
+          # SiteSetting.imap_write is disabled...maybe a later TODO
+          # ref: https://tools.ietf.org/html/rfc3501#section-7.2.2
+          m.attr.include?(:Noselect)
+        end.select do |m|
+
+          # There are Special-Use mailboxes denoted by an attribute. For
+          # example, some common ones are \Trash or \Sent.
+          # ref: https://tools.ietf.org/html/rfc6154
+          if attr_filter
+            m.attr.include? attr_filter
+          else
+            true
+          end
+        end.map do |m|
           m.name
         end
       end
@@ -131,6 +166,83 @@ module Imap
       def archive(uid)
         # do nothing by default, just removing the Inbox label should be enough
       end
+
+      def unarchive(uid)
+        # same as above
+      end
+
+      # Look for the special Trash XLIST attribute.
+      # TODO: It might be more efficient to just store this against the group.
+      # Another table is looking more and more attractive....
+      def trash_mailbox
+        Discourse.cache.fetch("imap_trash_mailbox_#{account_digest}", expires_in: 30.minutes) do
+          list_mailboxes(:Trash).first
+        end
+      end
+
+      # open the trash mailbox for inspection or writing. after the yield we
+      # close the trash and reopen the original mailbox to continue operations.
+      # the normal open_mailbox call can be made if more extensive trash ops
+      # need to be done.
+      def open_trash_mailbox(write: false)
+        open_mailbox_before_trash = @open_mailbox_name
+        open_mailbox_before_trash_write = @open_mailbox_write
+
+        trash_uid_validity = open_mailbox(trash_mailbox, write: write)[:uid_validity]
+
+        yield(trash_uid_validity) if block_given?
+
+        open_mailbox(open_mailbox_before_trash, write: open_mailbox_before_trash_write)
+        trash_uid_validity
+      end
+
+      def find_trashed_by_message_ids(message_ids)
+        trashed_emails = []
+        trash_uid_validity = open_trash_mailbox do
+          header_message_id_terms = message_ids.map do |msgid|
+            "HEADER Message-ID '#{Email.message_id_rfc_format(msgid)}'"
+          end
+
+          # OR clauses are written in Polish notation...so the query looks like this:
+          # OR OR HEADER Message-ID XXXX HEADER Message-ID XXXX HEADER Message-ID XXXX
+          or_clauses = 'OR ' * (header_message_id_terms.length - 1)
+          query = "#{or_clauses}#{header_message_id_terms.join(" ")}"
+
+          trashed_email_uids = imap.uid_search(query)
+          if trashed_email_uids.any?
+            trashed_emails = emails(trashed_email_uids, ["UID", "ENVELOPE"]).map do |e|
+              BasicMail.new(message_id: Email.message_id_clean(e['ENVELOPE'].message_id), uid: e['UID'])
+            end
+          end
+        end
+
+        TrashedMailResponse.new.tap do |resp|
+          resp.trashed_emails = trashed_emails
+          resp.trash_uid_validity = trash_uid_validity
+        end
+      end
+
+      def trash(uid)
+        # MOVE is way easier than doing the COPY \Deleted EXPUNGE dance ourselves.
+        # It is supported by Gmail and Outlook.
+        if can?('MOVE')
+          trash_move(uid)
+        else
+
+          # default behaviour for IMAP servers is to add the \Deleted flag
+          # then EXPUNGE the mailbox which permanently deletes these messages
+          # https://tools.ietf.org/html/rfc3501#section-6.4.3
+          #
+          # TODO: We may want to add the option at some point to copy to some
+          # other mailbox first before doing this (e.g. Trash)
+          store(uid, 'FLAGS', [], ["\\Deleted"])
+          imap.expunge
+        end
+      end
+
+      def trash_move(uid)
+        # up to the provider
+      end
     end
   end
 end

lib/imap/providers/gmail.rb

@@ -2,6 +2,11 @@
 
 module Imap
   module Providers
+    # Gmail has a special header for both labels (X-GM-LABELS) and their
+    # threading system (X-GM-THRID). We need to monkey-patch Net::IMAP to
+    # get access to these. Also the archiving functionality is custom,
+    # all UIDs in a thread must have the \\Inbox label removed.
+    #
     class Gmail < Generic
       X_GM_LABELS = 'X-GM-LABELS'
       X_GM_THRID = 'X-GM-THRID'
@@ -62,9 +67,9 @@ module Imap
         super(tag)
       end
 
+      # All emails in the thread must be archived in Gmail for the thread
+      # to get removed from the inbox
       def archive(uid)
-        # all emails in the thread must be archived in Gmail for the thread
-        # to get removed from the inbox
         thread_id = thread_id_from_uid(uid)
         emails_to_archive = emails_in_thread(thread_id)
         emails_to_archive.each do |email|
@@ -75,6 +80,23 @@ module Imap
         Imap::Sync::Logger.log("[IMAP] Thread ID #{thread_id} (UID #{uid}) archived in Gmail mailbox for #{@username}")
       end
 
+      # Though Gmail considers the email thread unarchived if the first email
+      # has the \\Inbox label applied, we want to do this to all emails in the
+      # thread to be consistent with archive behaviour.
+      def unarchive(uid)
+        thread_id = thread_id_from_uid(uid)
+        emails_to_unarchive = emails_in_thread(thread_id)
+        emails_to_unarchive.each do |email|
+          labels = email['LABELS']
+          new_labels = labels.dup
+          if !new_labels.include?("\\Inbox")
+            new_labels << "\\Inbox"
+          end
+          store(email["UID"], "LABELS", labels, new_labels)
+        end
+        Imap::Sync::Logger.log("[IMAP] Thread ID #{thread_id} (UID #{uid}) unarchived in Gmail mailbox for #{@username}")
+      end
+
       def thread_id_from_uid(uid)
         fetched = imap.uid_fetch(uid, [X_GM_THRID])
         if !fetched
@@ -89,6 +111,15 @@ module Imap
         emails(uids_to_fetch, ["UID", "LABELS"])
       end
 
+      def trash_move(uid)
+        thread_id = thread_id_from_uid(uid)
+        email_uids_to_trash = emails_in_thread(thread_id).map { |e| e['UID'] }
+
+        imap.uid_move(email_uids_to_trash, trash_mailbox)
+        Imap::Sync::Logger.log("[IMAP] Thread ID #{thread_id} (UID #{uid}) trashed in Gmail mailbox for #{@username}")
+        { trash_uid_validity: open_trash_mailbox, email_uids_to_trash: email_uids_to_trash }
+      end
+
       private
 
       def apply_gmail_patch(imap)

lib/imap/sync.rb

@@ -107,9 +107,9 @@ module Imap
       old_uids = old_uids.sample(old_emails_limit).sort! if old_emails_limit > -1
       new_uids = new_uids[0..new_emails_limit - 1] if new_emails_limit > 0
 
-      if old_uids.present?
+      # if there are no old_uids that is OK, this could indicate that some
-        process_old_uids(old_uids)
+      # UIDs have been sent to the trash
-      end
+      process_old_uids(old_uids)
 
       if new_uids.present?
         process_new_uids(new_uids, import_mode, all_old_uids_size, all_new_uids_size)
@@ -135,7 +135,7 @@ module Imap
 
     def process_old_uids(old_uids)
       Logger.log("[IMAP] (#{@group.name}) Syncing #{old_uids.size} randomly-selected old emails")
-      emails = @provider.emails(old_uids, ['UID', 'FLAGS', 'LABELS', 'ENVELOPE'])
+      emails = old_uids.empty? ? [] : @provider.emails(old_uids, ['UID', 'FLAGS', 'LABELS', 'ENVELOPE'])
       emails.each do |email|
         incoming_email = IncomingEmail.find_by(
           imap_uid_validity: @status[:uid_validity],
@@ -148,7 +148,7 @@ module Imap
         else
           # try finding email by message-id instead, we may be able to set the uid etc.
           incoming_email = IncomingEmail.where(
-            message_id: email['ENVELOPE'].message_id.tr("<>", ""),
+            message_id: Email.message_id_clean(email['ENVELOPE'].message_id),
             imap_uid: nil,
             imap_uid_validity: nil
           ).where("to_addresses LIKE '%#{@group.email_username}%'").first
@@ -165,6 +165,53 @@ module Imap
           end
         end
       end
+
+      handle_missing_uids(old_uids)
+    end
+
+    def handle_missing_uids(old_uids)
+      # If there are any UIDs for the mailbox missing from old_uids, this means they have been moved
+      # to some other mailbox in the mail server. They could be possibly deleted. first we can check
+      # if they have been deleted and if so delete the associated post/topic. then the remaining we
+      # can just remove the imap details from the IncomingEmail table and if they end up back in the
+      # original mailbox then they will be picked up in a future resync.
+      existing_incoming = IncomingEmail.includes(:post).where(
+        imap_group_id: @group.id, imap_uid_validity: @status[:uid_validity]
+      ).where.not(imap_uid: nil)
+
+      existing_uids = existing_incoming.map(&:imap_uid)
+      missing_uids = existing_uids - old_uids
+      missing_message_ids = existing_incoming.select do |incoming|
+        missing_uids.include?(incoming.imap_uid)
+      end.map(&:message_id)
+
+      return if missing_message_ids.empty?
+
+      # This can be done because Message-ID is unique on a mail server between mailboxes,
+      # where the UID will have changed when moving into the Trash mailbox. We need to get
+      # the new UID from the trash.
+      response = @provider.find_trashed_by_message_ids(missing_message_ids)
+      existing_incoming.each do |incoming|
+        matching_trashed = response.trashed_emails.find { |email| email.message_id == incoming.message_id }
+
+        # if the email is not in the trash then we don't know where it is... could
+        # be in any mailbox on the server or could be permanently deleted. TODO
+        # here would be some sort of more complex detection of "where in the world
+        # has this UID gone?"
+        next if !matching_trashed
+
+        # if we deleted the topic/post ourselves in discourse then the post will
+        # not exist, and this sync is just updating the old UIDs to the new ones
+        # in the trash, and we don't need to re-destroy the post
+        if incoming.post
+          Logger.log("[IMAP] (#{@group.name}) Deleting post ID #{incoming.post_id}; it has been deleted on the IMAP server.")
+          PostDestroyer.new(Discourse.system_user, incoming.post).destroy
+        end
+
+        # the email has moved mailboxes, we don't want to try trashing again next time
+        Logger.log("[IMAP] (#{@group.name}) Updating incoming ID #{incoming.id} uid data FROM [UID #{incoming.imap_uid} | UIDVALIDITY #{incoming.imap_uid_validity}] TO [UID #{matching_trashed.uid} | UIDVALIDITY #{response.trash_uid_validity}] (TRASHED)")
+        incoming.update(imap_uid_validity: response.trash_uid_validity, imap_uid: matching_trashed.uid)
+      end
     end
 
     def process_new_uids(new_uids, import_mode, all_old_uids_size, all_new_uids_size)
@@ -266,7 +313,19 @@ module Imap
 
     def update_email(incoming_email)
       return if !SiteSetting.tagging_enabled || !SiteSetting.allow_staff_to_tag_pms
-      return if incoming_email&.post&.post_number != 1 || !incoming_email.imap_sync
+      return if !incoming_email || !incoming_email.imap_sync
+
+      post = incoming_email.post
+      if !post && incoming_email.post_id
+        # post was likely deleted because topic was deleted, let's try get it
+        post = Post.with_deleted.find(incoming_email.post_id)
+      end
+
+      # don't do any of these type of updates on anything but the OP in the
+      # email thread -- archiving and deleting will be handled for the whole
+      # thread depending on provider
+      return if post&.post_number != 1
+      topic = incoming_email.topic
 
       # if email is nil, the UID does not exist in the provider, meaning....
       #
@@ -279,11 +338,14 @@ module Imap
 
       labels = email['LABELS']
       flags = email['FLAGS']
-      topic = incoming_email.topic
 
-      # TODO: Delete remote email if topic no longer exists
+      # Topic has been deleted if it is not present from the post, so we need
-      # new_flags << Net::IMAP::DELETED if !incoming_email.topic
+      # to trash the IMAP server email
-      return if !topic
+      if !topic
+        # no need to do anything further here, we will recognize the UIDs in the
+        # mail server email thread have been trashed on next sync
+        return @provider.trash(incoming_email.imap_uid)
+      end
 
       # Sync topic status and labels with email flags and labels.
       tags = topic.tags.pluck(:name)
@@ -294,7 +356,15 @@ module Imap
       # server
       topic_archived = topic.group_archived_messages.any?
       if !topic_archived
-        new_labels << '\\Inbox'
+        # TODO: This is needed right now so the store below does not take it
+        # away again...ideally we should unarchive and store the tag-labels
+        # at the same time.
+        new_labels << "\\Inbox"
+
+        Logger.log("[IMAP] (#{@group.name}) Unarchiving UID #{incoming_email.imap_uid}")
+
+        # some providers need special handling for unarchiving too
+        @provider.unarchive(incoming_email.imap_uid)
       else
         Logger.log("[IMAP] (#{@group.name}) Archiving UID #{incoming_email.imap_uid}")
 

lib/post_destroyer.rb

@@ -100,6 +100,7 @@ class PostDestroyer
       UserActionManager.topic_created(topic)
       DiscourseEvent.trigger(:topic_recovered, topic, @user)
       StaffActionLogger.new(@user).log_topic_delete_recover(topic, "recover_topic", @opts.slice(:context)) if @user.id != @post.user_id
+      update_imap_sync(@post, false)
     end
   end
 
@@ -170,6 +171,7 @@ class PostDestroyer
       end
     end
 
+    update_imap_sync(@post, true) if @post.topic&.deleted_at
     feature_users_in_the_topic if @post.topic
     @post.publish_change_to_clients! :deleted if @post.topic
     TopicTrackingState.publish_delete(@post.topic) if @post.topic && @post.post_number == 1
@@ -375,4 +377,11 @@ class PostDestroyer
     end
   end
 
+  def update_imap_sync(post, sync)
+    return if !SiteSetting.enable_imap
+    incoming = IncomingEmail.find_by(post_id: post.id, topic_id: post.topic_id)
+    return if !incoming || !incoming.imap_uid
+    incoming.update(imap_sync: sync)
+  end
+
 end

spec/components/imap/sync_spec.rb

@@ -266,6 +266,124 @@ describe Imap::Sync do
       expect(Topic.last.posts.where(post_type: Post.types[:regular]).count).to eq(2)
     end
 
+    describe "detecting deleted emails and deleting the topic in discourse" do
+      let(:provider) { MockedImapProvider.any_instance }
+      before do
+        provider.stubs(:open_mailbox).returns(uid_validity: 1)
+
+        provider.stubs(:uids).with.returns([100])
+        provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS', 'RFC822'], anything).returns(
+          [
+            {
+              'UID' => 100,
+              'LABELS' => %w[\\Inbox],
+              'FLAGS' => %i[Seen],
+              'RFC822' => EmailFabricator(
+                message_id: first_message_id,
+                from: first_from,
+                to: group.email_username,
+                cc: second_from,
+                subject: subject,
+                body: first_body
+              )
+            }
+          ]
+        )
+
+      end
+
+      it "detects previously synced UIDs are missing and deletes the posts if they are in the trash mailbox" do
+        sync_handler.process
+        incoming_100 = IncomingEmail.find_by(imap_uid: 100)
+        provider.stubs(:uids).with.returns([])
+
+        provider.stubs(:uids).with(to: 100).returns([])
+        provider.stubs(:uids).with(from: 101).returns([])
+        provider.stubs(:find_trashed_by_message_ids).returns(
+          stub(
+            trashed_emails: [
+              stub(
+                uid: 10,
+                message_id: incoming_100.message_id
+              )
+            ],
+            trash_uid_validity: 99
+          )
+        )
+        sync_handler.process
+
+        incoming_100.reload
+        expect(incoming_100.imap_uid_validity).to eq(99)
+        expect(incoming_100.imap_uid).to eq(10)
+        expect(Post.with_deleted.find(incoming_100.post_id).deleted_at).not_to eq(nil)
+        expect(Topic.with_deleted.find(incoming_100.topic_id).deleted_at).not_to eq(nil)
+      end
+
+      it "detects the topic being deleted on the discourse site and deletes on the IMAP server and
+      does not attempt to delete again on discourse site when deleted already by us on the IMAP server" do
+        SiteSetting.enable_imap_write = true
+        sync_handler.process
+        incoming_100 = IncomingEmail.find_by(imap_uid: 100)
+        provider.stubs(:uids).with.returns([100])
+
+        provider.stubs(:uids).with(to: 100).returns([100])
+        provider.stubs(:uids).with(from: 101).returns([])
+
+        PostDestroyer.new(Discourse.system_user, incoming_100.post).destroy
+        provider.stubs(:emails).with([100], ['UID', 'FLAGS', 'LABELS', 'ENVELOPE'], anything).returns(
+          [
+            {
+              'UID' => 100,
+              'LABELS' => %w[\\Inbox],
+              'FLAGS' => %i[Seen],
+              'RFC822' => EmailFabricator(
+                message_id: first_message_id,
+                from: first_from,
+                to: group.email_username,
+                cc: second_from,
+                subject: subject,
+                body: first_body
+              )
+            }
+          ]
+        )
+        provider.stubs(:emails).with(100, ['FLAGS', 'LABELS']).returns(
+          [
+            {
+              'LABELS' => %w[\\Inbox],
+              'FLAGS' => %i[Seen]
+            }
+          ]
+        )
+
+        provider.expects(:trash).with(100)
+        sync_handler.process
+
+        provider.stubs(:uids).with.returns([])
+
+        provider.stubs(:uids).with(to: 100).returns([])
+        provider.stubs(:uids).with(from: 101).returns([])
+        provider.stubs(:find_trashed_by_message_ids).returns(
+          stub(
+            trashed_emails: [
+              stub(
+                uid: 10,
+                message_id: incoming_100.message_id
+              )
+            ],
+            trash_uid_validity: 99
+          )
+        )
+        PostDestroyer.expects(:new).never
+
+        sync_handler.process
+
+        incoming_100.reload
+        expect(incoming_100.imap_uid_validity).to eq(99)
+        expect(incoming_100.imap_uid).to eq(10)
+      end
+    end
+
     describe "archiving emails" do
       let(:provider) { MockedImapProvider.any_instance }
       before do
@@ -330,12 +448,13 @@ describe Imap::Sync do
         sync_handler.process
       end
 
-      it "does not archive email if not archived in discourse" do
+      it "does not archive email if not archived in discourse, it unarchives it instead" do
         @incoming_email.update(imap_sync: true)
         provider.stubs(:store).with(100, 'FLAGS', anything, anything)
         provider.stubs(:store).with(100, 'LABELS', ["\\Inbox"], ["seen", "\\Inbox"])
 
         provider.expects(:archive).with(100).never
+        provider.expects(:unarchive).with(100)
         sync_handler.process
       end
     end

spec/components/post_destroyer_spec.rb

@@ -361,6 +361,14 @@ describe PostDestroyer do
       expect(post.like_count).to eq(2)
       expect(post.custom_fields["deleted_public_actions"]).to be_nil
     end
+
+    it "unmarks the matching incoming email for imap sync" do
+      SiteSetting.enable_imap = true
+      incoming = Fabricate(:incoming_email, imap_sync: true, post: post, topic: post.topic, imap_uid: 99)
+      PostDestroyer.new(moderator, post).recover
+      incoming.reload
+      expect(incoming.imap_sync).to eq(false)
+    end
   end
 
   describe 'basic destroying' do
@@ -756,6 +764,28 @@ describe PostDestroyer do
       end
     end
 
+    describe "incoming email and imap sync" do
+      fab!(:incoming) { Fabricate(:incoming_email, post: post, topic: post.topic) }
+
+      it "does nothing if imap not enabled" do
+        IncomingEmail.expects(:find_by).never
+        PostDestroyer.new(moderator, post).destroy
+      end
+
+      it "does nothing if the incoming email has no imap_uid" do
+        SiteSetting.enable_imap = true
+        PostDestroyer.new(moderator, post).destroy
+        expect(incoming.reload.imap_sync).to eq(false)
+      end
+
+      it "sets imap_sync to true for the matching incoming" do
+        SiteSetting.enable_imap = true
+        incoming.update(imap_uid: 999)
+        PostDestroyer.new(moderator, post).destroy
+        expect(incoming.reload.imap_sync).to eq(true)
+      end
+    end
+
     describe 'with a reply' do
 
       fab!(:reply) { Fabricate(:basic_reply, user: coding_horror, topic: post.topic) }

spec/fabricators/incoming_email_fabricator.rb

@@ -5,6 +5,7 @@ Fabricator(:incoming_email) do
   subject "Hello world"
   from_address "foo@example.com"
   to_addresses "someone@else.com"
+  imap_sync false
 
   raw <<~RAW
     Return-Path: <foo@example.com>

spec/lib/imap/providers/generic_spec.rb

@@ -16,6 +16,13 @@ RSpec.describe Imap::Providers::Generic do
       }
     )
   end
+  let(:dummy_mailboxes) do
+    [
+      Net::IMAP::MailboxList.new([], "/", "All Mail"),
+      Net::IMAP::MailboxList.new([:Noselect], "/", "Other"),
+      Net::IMAP::MailboxList.new([:Trash], "/", "Bin")
+    ]
+  end
 
   let(:imap_stub) { stub }
   before do
@@ -29,6 +36,101 @@ RSpec.describe Imap::Providers::Generic do
     end
   end
 
+  describe "#list_mailboxes" do
+    before do
+      imap_stub.expects(:list).with('', '*').returns(dummy_mailboxes)
+    end
+
+    it "does not return any mailboxes with the Noselect attribute" do
+      expect(provider.list_mailboxes).not_to include("Other")
+    end
+
+    it "filters by the provided attribute" do
+      expect(provider.list_mailboxes(:Trash)).to eq(["Bin"])
+    end
+
+    it "lists all mailboxes names" do
+      expect(provider.list_mailboxes).to eq(["All Mail", "Bin"])
+    end
+  end
+
+  describe "#trash_mailbox" do
+
+    before do
+      imap_stub.expects(:list).with('', '*').returns(dummy_mailboxes)
+      Discourse.cache.delete("imap_trash_mailbox_#{provider.account_digest}")
+    end
+
+    it "returns the mailbox with the special-use attribute \Trash" do
+      expect(provider.trash_mailbox).to eq("Bin")
+    end
+
+    it "caches the result based on the account username and server for 30 mins" do
+      provider.trash_mailbox
+      provider.expects(:list_mailboxes).never
+      provider.trash_mailbox
+    end
+  end
+
+  describe "#find_trashed_by_message_ids" do
+    before do
+      provider.stubs(:trash_mailbox).returns("Bin")
+      imap_stub.stubs(:examine).with("Inbox").twice
+      imap_stub.stubs(:responses).returns({ 'UIDVALIDITY' => [1] })
+      imap_stub.stubs(:examine).with("Bin")
+      imap_stub.stubs(:responses).returns({ 'UIDVALIDITY' => [9] })
+      provider.expects(:emails).with([4, 6], ['UID', 'ENVELOPE']).returns(
+        [
+          {
+            'ENVELOPE' => stub(message_id: "<h4786x34@test.com>"),
+            'UID' => 4
+          },
+          {
+            'ENVELOPE' => stub(message_id: "<f349xj84@test.com>"),
+            'UID' => 6
+          }
+        ]
+      )
+    end
+
+    let(:message_ids) do
+      [
+        "h4786x34@test.com",
+        "dvsfuf39@test.com",
+        "f349xj84@test.com"
+      ]
+    end
+
+    it "sends the message-id search in the correct format and returns the trashed emails and UIDVALIDITY" do
+      provider.open_mailbox("Inbox")
+      imap_stub.expects(:uid_search).with(
+        "OR OR HEADER Message-ID '<h4786x34@test.com>' HEADER Message-ID '<dvsfuf39@test.com>' HEADER Message-ID '<f349xj84@test.com>'"
+
+      ).returns([4, 6])
+      resp = provider.find_trashed_by_message_ids(message_ids)
+
+      expect(resp.trashed_emails.map(&:message_id)).to match_array(['h4786x34@test.com', 'f349xj84@test.com'])
+      expect(resp.trash_uid_validity).to eq(9)
+    end
+  end
+
+  describe "#trash" do
+    it "stores the \Deleted flag on the UID and expunges" do
+      provider.stubs(:can?).with('MOVE').returns(false)
+      provider.expects(:store).with(78, 'FLAGS', [], ['\Deleted'])
+      imap_stub.expects(:expunge)
+      provider.trash(78)
+    end
+
+    context "if the server supports MOVE" do
+      it "calls trash_move which is implemented by the provider" do
+        provider.stubs(:can?).with('MOVE').returns(true)
+        provider.expects(:trash_move).with(78)
+        provider.trash(78)
+      end
+    end
+  end
+
   describe "#uids" do
     it "can search with from and to" do
       imap_stub.expects(:uid_search).once.with("UID 5:9")