diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
index 2ad8e9c2a16..b81128e52a4 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
@@ -454,6 +454,9 @@ public class HnswGraphBuilder implements HnswBuilder {
       int[] eps = new int[1];
       for (Component c : components) {
         if (c != c0) {
+          if (c.start() == NO_MORE_DOCS) {
+            continue;
+          }
           beam.clear();
           eps[0] = c0.start();
           RandomVectorScorer scorer = scorerSupplier.scorer(c.start());