diff --git a/CHANGES.txt b/CHANGES.txt
index 3ac2d995c89..268c96e07e5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -702,6 +702,8 @@ Release 0.90.5 - Unreleased
    HBASE-4515  User.getCurrent() can fail to initialize the current user
    HBASE-4473  NPE when executors are down but events are still coming in
    HBASE-4537  TestUser imports breaking build against secure Hadoop
+   HBASE-4501  [replication] Shutting down a stream leaves recovered
+               sources running
 
   IMPROVEMENT
    HBASE-4205  Enhance HTable javadoc (Eric Charles)
diff --git a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index d91186348a9..c8bd2139010 100644
--- a/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -354,6 +354,7 @@ public class ReplicationSourceManager {
     LOG.info("Closing the following queue " + id + ", currently have "
         + sources.size() + " and another "
         + oldsources.size() + " that were recovered");
+    String terminateMessage = "Replication stream was removed by a user";
     ReplicationSourceInterface srcToRemove = null;
     List<ReplicationSourceInterface> oldSourcesToDelete =
         new ArrayList<ReplicationSourceInterface>();
@@ -364,6 +365,7 @@ public class ReplicationSourceManager {
       }
     }
     for (ReplicationSourceInterface src : oldSourcesToDelete) {
+      src.terminate(terminateMessage);
       closeRecoveredQueue((src));
     }
     LOG.info("Number of deleted recovered sources for " + id + ": "
@@ -379,7 +381,7 @@ public class ReplicationSourceManager {
       LOG.error("The queue we wanted to close is missing " + id);
       return;
     }
-    srcToRemove.terminate("Replication stream was removed by a user");
+    srcToRemove.terminate(terminateMessage);
     this.sources.remove(srcToRemove);
     this.zkHelper.deleteSource(id, true);
   }