diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c865712ac89..7798dc8cfb9 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -168,6 +168,9 @@ Improvements
   is too old. It also logs partial vectorization support if old CPU or disabled AVX.
   (Uwe Schindler, Robert Muir)
 
+* GITHUB#12677: Better detect vector module in non-default setups (e.g., custom module layers).
+  (Uwe Schindler)
+
 Optimizations
 ---------------------
 * GITHUB#12183: Make TermStates#build concurrent. (Shubham Chaudhary)
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
index 3192af72b2c..f1936199b0b 100644
--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
@@ -25,6 +25,7 @@ import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Logger;
 import org.apache.lucene.util.SuppressForbidden;
@@ -82,11 +83,14 @@ public abstract class VectorizationProvider {
       }
       // is the incubator module present and readable (JVM providers may to exclude them or it is
       // build with jlink)
-      if (!vectorModulePresentAndReadable()) {
+      final var vectorMod = lookupVectorModule();
+      if (vectorMod.isEmpty()) {
         LOG.warning(
             "Java vector incubator module is not readable. For optimal vector performance, pass '--add-modules jdk.incubator.vector' to enable Vector API.");
         return new DefaultVectorizationProvider();
       }
+      vectorMod.ifPresent(VectorizationProvider.class.getModule()::addReads);
+      // check if client VM
       if (!testMode && isClientVM()) {
         LOG.warning("C2 compiler is disabled; Java vector incubator API can't be enabled");
         return new DefaultVectorizationProvider();
@@ -119,7 +123,7 @@ public abstract class VectorizationProvider {
     } else if (runtimeVersion >= 22) {
       LOG.warning(
           "You are running with Java 22 or later. To make full use of the Vector API, please update Apache Lucene.");
-    } else if (vectorModulePresentAndReadable()) {
+    } else if (lookupVectorModule().isPresent()) {
       LOG.warning(
           "Java vector incubator module was enabled by command line flags, but your Java version is too old: "
               + runtimeVersion);
@@ -127,16 +131,13 @@ public abstract class VectorizationProvider {
     return new DefaultVectorizationProvider();
   }
 
-  private static boolean vectorModulePresentAndReadable() {
-    var opt =
-        ModuleLayer.boot().modules().stream()
-            .filter(m -> m.getName().equals("jdk.incubator.vector"))
-            .findFirst();
-    if (opt.isPresent()) {
-      VectorizationProvider.class.getModule().addReads(opt.get());
-      return true;
-    }
-    return false;
+  /**
+   * Looks up the vector module from Lucene's {@link ModuleLayer} or the root layer (if unnamed).
+   */
+  private static Optional<Module> lookupVectorModule() {
+    return Optional.ofNullable(VectorizationProvider.class.getModule().getLayer())
+        .orElse(ModuleLayer.boot())
+        .findModule("jdk.incubator.vector");
   }
 
   /**
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.java
index 840d3fc6b8b..11e6af63a52 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.java
@@ -131,6 +131,8 @@ public class TestVectorSimilarityValuesSource extends LuceneTestCase {
             VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT));
     document3.add(new KnnByteVectorField("knnByteField5", new byte[] {-120, -2, 3}));
     iw.addDocument(document3);
+    iw.commit();
+    iw.forceMerge(1);
 
     reader = iw.getReader();
     searcher = newSearcher(reader);
@@ -313,11 +315,12 @@ public class TestVectorSimilarityValuesSource extends LuceneTestCase {
     DoubleValues dv =
         DoubleValuesSource.similarityToQueryVector(
             searcher.reader.leaves().get(0), floatQueryVector, "knnFloatField4");
-    assertTrue(
-        dv.advanceExact(0)
-            && dv.doubleValue()
-                == VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT.compare(
-                    new float[] {-1.3f, 1.0f, 1.0f}, floatQueryVector));
+    assertTrue(dv.advanceExact(0));
+    assertEquals(
+        VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT.compare(
+            new float[] {-1.3f, 1.0f, 1.0f}, floatQueryVector),
+        dv.doubleValue(),
+        0.0001);
     assertFalse(dv.advanceExact(1));
     assertFalse(dv.advanceExact(2));
 
@@ -326,21 +329,24 @@ public class TestVectorSimilarityValuesSource extends LuceneTestCase {
     dv =
         DoubleValuesSource.similarityToQueryVector(
             searcher.reader.leaves().get(0), byteQueryVector, "knnByteField4");
-    assertTrue(
-        dv.advanceExact(0)
-            && dv.doubleValue()
-                == VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT.compare(
-                    new byte[] {-127, 127, 127}, byteQueryVector));
-    assertTrue(
-        dv.advanceExact(1)
-            && dv.doubleValue()
-                == VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT.compare(
-                    new byte[] {14, 29, 31}, byteQueryVector));
-    assertTrue(
-        dv.advanceExact(2)
-            && dv.doubleValue()
-                == VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT.compare(
-                    new byte[] {-4, -2, -128}, byteQueryVector));
+    assertTrue(dv.advanceExact(0));
+    assertEquals(
+        VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT.compare(
+            new byte[] {-127, 127, 127}, byteQueryVector),
+        dv.doubleValue(),
+        0.0001);
+    assertTrue(dv.advanceExact(1));
+    assertEquals(
+        VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT.compare(
+            new byte[] {14, 29, 31}, byteQueryVector),
+        dv.doubleValue(),
+        0.0001);
+    assertTrue(dv.advanceExact(2));
+    assertEquals(
+        VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT.compare(
+            new byte[] {-4, -2, -128}, byteQueryVector),
+        dv.doubleValue(),
+        0.0001);
   }
 
   public void testFailuresWithSimilarityValuesSource() throws Exception {