diff --git a/app/models/topic.rb b/app/models/topic.rb
index 54af4242de7..50eda089fd8 100644
--- a/app/models/topic.rb
+++ b/app/models/topic.rb
@@ -976,7 +976,6 @@ class Topic < ActiveRecord::Base
         topic_user.destroy
 
         if user.id == removed_by&.id
-          removed_by = Discourse.system_user
           add_small_action(removed_by, "user_left", user.username)
         else
           add_small_action(removed_by, "removed_user", user.username)
diff --git a/lib/post_creator.rb b/lib/post_creator.rb
index aeafa9fcb8e..b368edb0976 100644
--- a/lib/post_creator.rb
+++ b/lib/post_creator.rb
@@ -431,7 +431,7 @@ class PostCreator
 
   def ensure_in_allowed_users
     return unless @topic.private_message? && @topic.id
-    return if @post.whisper?
+    return if @post.whisper? || @post.post_type == Post.types[:small_action]
     return if @topic.topic_allowed_users.exists?(user_id: @user.id)
 
     return if @topic
diff --git a/spec/lib/post_creator_spec.rb b/spec/lib/post_creator_spec.rb
index d32e14cb5f9..5866d65bdbf 100644
--- a/spec/lib/post_creator_spec.rb
+++ b/spec/lib/post_creator_spec.rb
@@ -1041,6 +1041,21 @@ RSpec.describe PostCreator do
       )
     end
 
+    it 'does not add whisperers to allowed users of the topic' do
+      unrelated_user.update!(admin: true)
+
+      PostCreator.create!(
+        unrelated_user,
+        raw: "This is a whisper that I am testing",
+        topic_id: post.topic_id,
+        post_type: Post.types[:small_action],
+      )
+
+      expect(post.topic.topic_allowed_users.map(&:user_id)).to contain_exactly(
+        target_user1.id, target_user2.id, user.id
+      )
+    end
+
     it 'does not increase posts count for small actions' do
       topic = Fabricate(:private_message_topic, user: Fabricate(:user))
 
diff --git a/spec/models/topic_spec.rb b/spec/models/topic_spec.rb
index 54a6bd0e0ed..9d65c6d9936 100644
--- a/spec/models/topic_spec.rb
+++ b/spec/models/topic_spec.rb
@@ -1138,7 +1138,7 @@ RSpec.describe Topic do
 
             expect(topic.invite_group(topic.user, admins)).to eq(true)
             expect(topic.posts.last.action_code).to eq("removed_user")
-            expect(topic.allowed_users).to match_array([user0, user3, Discourse.system_user])
+            expect(topic.allowed_users).to match_array([user0, user3])
             expect(other_topic.allowed_users).to match_array([user1])
           end
 
@@ -1156,7 +1156,7 @@ RSpec.describe Topic do
             admins.add(user1)
 
             expect(topic.invite_group(topic.user, admins)).to eq(true)
-            expect(topic.allowed_users).to match_array([topic.user, Discourse.system_user])
+            expect(topic.allowed_users).to match_array([topic.user])
           end
         end
       end
@@ -2817,7 +2817,7 @@ RSpec.describe Topic do
 
         post = Post.last
 
-        expect(post.user).to eq(Discourse.system_user)
+        expect(post.user).to eq(user1)
         expect(post.post_type).to eq(Post.types[:small_action])
         expect(post.action_code).to eq('user_left')
       end