diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index e62ed53be53..d855aeecead 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -175,6 +175,8 @@ Optimizations
   addresses for BINARY fields are now stored on disk instead of in memory.
   (Adrien Grand)
 
+* LUCENE-6878: Speed up TopDocs.merge. (Daniel Jelinski via Adrien Grand)
+
 Bug Fixes
 
 * LUCENE-6817: ComplexPhraseQueryParser.ComplexPhraseQuery does not display 
diff --git a/lucene/core/src/java/org/apache/lucene/search/TopDocs.java b/lucene/core/src/java/org/apache/lucene/search/TopDocs.java
index 99a90699bc0..48bdcdd6779 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TopDocs.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TopDocs.java
@@ -272,7 +272,7 @@ public class TopDocs {
       int hitUpto = 0;
       while (hitUpto < numIterOnHits) {
         assert queue.size() > 0;
-        ShardRef ref = queue.pop();
+        ShardRef ref = queue.top();
         final ScoreDoc hit = shardHits[ref.shardIndex].scoreDocs[ref.hitIndex++];
         hit.shardIndex = ref.shardIndex;
         if (hitUpto >= start) {
@@ -286,7 +286,9 @@ public class TopDocs {
 
         if (ref.hitIndex < shardHits[ref.shardIndex].scoreDocs.length) {
           // Not done with this these TopDocs yet:
-          queue.add(ref);
+          queue.updateTop();
+        } else {
+          queue.pop();
         }
       }
     }