diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3da5dd4a9a9..c1bede4c8dd 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -184,6 +184,9 @@ Optimizations
   instance if it already is modifiable, otherwise creates a new ModifiableSolrParams instance.
   (Jörg Rathlev via Koji)
 
+* SOLR-10499: facet.heatmap is now significantly faster when the docset (base query) matches everything and there are no
+  deleted docs.  It's also faster when the docset matches a small fraction of the index or none. (David Smiley)
+
 Bug Fixes
 ----------------------
 * SOLR-10281: ADMIN_PATHS is duplicated in two places and inconsistent. This can cause automatic
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java b/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java
index 4ad882c8e99..9bca5c7e141 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SpatialHeatmapFacets.java
@@ -32,6 +32,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.solr.search.DocIterator;
+import org.apache.solr.search.SolrIndexSearcher;
 import org.locationtech.spatial4j.context.SpatialContext;
 import org.locationtech.spatial4j.shape.Shape;
 import org.apache.lucene.spatial.prefix.HeatmapFacetCounter;
@@ -134,32 +137,13 @@ public class SpatialHeatmapFacets {
       gridLevel = strategy.getGrid().getLevelForDistance(distErr);
     }
 
-    // Turn docSet into Bits
-    Bits topAcceptDocs;
-    if (docSet instanceof BitDocSet) {
-      BitDocSet set = (BitDocSet) docSet;
-      topAcceptDocs = set.getBits();
-    } else {
-      topAcceptDocs = new Bits() {
-        @Override
-        public boolean get(int index) {
-          return docSet.exists(index);
-        }
-
-        @Override
-        public int length() {
-          return rb.req.getSearcher().maxDoc();
-        }
-      };
-    }
-
     //Compute!
     final HeatmapFacetCounter.Heatmap heatmap;
     try {
       heatmap = HeatmapFacetCounter.calcFacets(
           strategy,
           rb.req.getSearcher().getTopReaderContext(),
-          topAcceptDocs,
+          getTopAcceptDocs(docSet, rb.req.getSearcher()), // turn DocSet into Bits
           boundsShape,
           gridLevel,
           params.getFieldInt(fieldKey, FacetParams.FACET_HEATMAP_MAX_CELLS, 100_000) // will throw if exceeded
@@ -190,6 +174,23 @@ public class SpatialHeatmapFacets {
     return result;
   }
 
+  private static Bits getTopAcceptDocs(DocSet docSet, SolrIndexSearcher searcher) throws IOException {
+    if (searcher.getLiveDocs() == docSet) {
+      return null; // means match everything (all live docs). This can speedup things a lot.
+    } else if (docSet.size() == 0) {
+      return new Bits.MatchNoBits(searcher.maxDoc()); // can speedup things a lot
+    } else if (docSet instanceof BitDocSet) {
+      return ((BitDocSet) docSet).getBits();
+    } else {
+      // TODO DocSetBase.calcBits ought to be at DocSet level?
+      FixedBitSet bits = new FixedBitSet(searcher.maxDoc());
+      for (DocIterator iter = docSet.iterator(); iter.hasNext();) {
+        bits.set(iter.nextDoc());
+      }
+      return bits;
+    }
+  }
+
   private static void formatCountsAndAddToNL(String fieldKey, ResponseBuilder rb, SolrParams params,
                                              int columns, int rows, int[] counts, NamedList<Object> result) {
     final String format = params.getFieldParam(fieldKey, FacetParams.FACET_HEATMAP_FORMAT, FORMAT_INTS2D);