diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java index 23d607a1c77..1b45c7fe44b 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java @@ -84,8 +84,8 @@ public final class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFo * @param beamWidth the size of the queue maintained during graph construction. * @param numMergeWorkers number of workers (threads) that will be used when doing merge. If * larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec - * @param configuredQuantile the quantile for scalar quantizing the vectors, when `null` it is - * calculated based on the vector field dimensions. + * @param confidenceInterval the confidenceInterval for scalar quantizing the vectors, when `null` + * it is calculated based on the vector field dimensions. * @param mergeExec the {@link ExecutorService} that will be used by ALL vector writers that are * generated by this format to do the merge */ @@ -93,7 +93,7 @@ public final class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFo int maxConn, int beamWidth, int numMergeWorkers, - Float configuredQuantile, + Float confidenceInterval, ExecutorService mergeExec) { super("Lucene99HnswScalarQuantizedVectorsFormat"); if (maxConn <= 0 || maxConn > MAXIMUM_MAX_CONN) { @@ -122,7 +122,7 @@ public final class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFo } this.numMergeWorkers = numMergeWorkers; this.mergeExec = mergeExec; - this.flatVectorsFormat = new Lucene99ScalarQuantizedVectorsFormat(configuredQuantile); + this.flatVectorsFormat = new Lucene99ScalarQuantizedVectorsFormat(confidenceInterval); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java index f6550a220e3..27e6ef9e363 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java @@ -43,17 +43,17 @@ public final class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsForma private static final FlatVectorsFormat rawVectorFormat = new Lucene99FlatVectorsFormat(); - /** The minimum quantile */ - private static final float MINIMUM_QUANTILE = 0.9f; + /** The minimum confidence interval */ + private static final float MINIMUM_CONFIDENCE_INTERVAL = 0.9f; - /** The maximum quantile */ - private static final float MAXIMUM_QUANTILE = 1f; + /** The maximum confidence interval */ + private static final float MAXIMUM_CONFIDENCE_INTERVAL = 1f; /** - * Controls the quantile used to scalar quantize the vectors the default quantile is calculated as - * `1-1/(vector_dimensions + 1)` + * Controls the confidence interval used to scalar quantize the vectors the default value is + * calculated as `1-1/(vector_dimensions + 1)` */ - final Float quantile; + final Float confidenceInterval; /** Constructs a format using default graph construction parameters */ public Lucene99ScalarQuantizedVectorsFormat() { @@ -63,24 +63,26 @@ public final class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsForma /** * Constructs a format using the given graph construction parameters. * - * @param quantile the quantile for scalar quantizing the vectors, when `null` it is calculated - * based on the vector field dimensions. + * @param confidenceInterval the confidenceInterval for scalar quantizing the vectors, when `null` + * it is calculated based on the vector field dimensions. */ - public Lucene99ScalarQuantizedVectorsFormat(Float quantile) { - if (quantile != null && (quantile < MINIMUM_QUANTILE || quantile > MAXIMUM_QUANTILE)) { + public Lucene99ScalarQuantizedVectorsFormat(Float confidenceInterval) { + if (confidenceInterval != null + && (confidenceInterval < MINIMUM_CONFIDENCE_INTERVAL + || confidenceInterval > MAXIMUM_CONFIDENCE_INTERVAL)) { throw new IllegalArgumentException( - "quantile must be between " - + MINIMUM_QUANTILE + "confidenceInterval must be between " + + MINIMUM_CONFIDENCE_INTERVAL + " and " - + MAXIMUM_QUANTILE - + "; quantile=" - + quantile); + + MAXIMUM_CONFIDENCE_INTERVAL + + "; confidenceInterval=" + + confidenceInterval); } - this.quantile = quantile; + this.confidenceInterval = confidenceInterval; } - static float calculateDefaultQuantile(int vectorDimension) { - return Math.max(MINIMUM_QUANTILE, 1f - (1f / (vectorDimension + 1))); + static float calculateDefaultConfidenceInterval(int vectorDimension) { + return Math.max(MINIMUM_CONFIDENCE_INTERVAL, 1f - (1f / (vectorDimension + 1))); } @Override @@ -88,8 +90,8 @@ public final class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsForma return NAME + "(name=" + NAME - + ", quantile=" - + quantile + + ", confidenceInterval=" + + confidenceInterval + ", rawVectorFormat=" + rawVectorFormat + ")"; @@ -98,7 +100,7 @@ public final class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsForma @Override public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { return new Lucene99ScalarQuantizedVectorsWriter( - state, quantile, rawVectorFormat.fieldsWriter(state)); + state, confidenceInterval, rawVectorFormat.fieldsWriter(state)); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java index aaf706cefc2..e9c488a86ae 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java @@ -303,10 +303,10 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade dimension = input.readVInt(); size = input.readInt(); if (size > 0) { - float configuredQuantile = Float.intBitsToFloat(input.readInt()); + float confidenceInterval = Float.intBitsToFloat(input.readInt()); float minQuantile = Float.intBitsToFloat(input.readInt()); float maxQuantile = Float.intBitsToFloat(input.readInt()); - scalarQuantizer = new ScalarQuantizer(minQuantile, maxQuantile, configuredQuantile); + scalarQuantizer = new ScalarQuantizer(minQuantile, maxQuantile, confidenceInterval); } else { scalarQuantizer = null; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java index e74217b9a8f..760e56df13c 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java @@ -19,7 +19,7 @@ package org.apache.lucene.codecs.lucene99; import static org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.QUANTIZED_VECTOR_COMPONENT; -import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.calculateDefaultQuantile; +import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.calculateDefaultConfidenceInterval; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; @@ -91,14 +91,14 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite private final List fields = new ArrayList<>(); private final IndexOutput meta, quantizedVectorData; - private final Float quantile; + private final Float confidenceInterval; private final FlatVectorsWriter rawVectorDelegate; private boolean finished; Lucene99ScalarQuantizedVectorsWriter( - SegmentWriteState state, Float quantile, FlatVectorsWriter rawVectorDelegate) + SegmentWriteState state, Float confidenceInterval, FlatVectorsWriter rawVectorDelegate) throws IOException { - this.quantile = quantile; + this.confidenceInterval = confidenceInterval; segmentWriteState = state; String metaFileName = IndexFileNames.segmentFileName( @@ -142,12 +142,12 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite public FlatFieldVectorsWriter addField( FieldInfo fieldInfo, KnnFieldVectorsWriter indexWriter) throws IOException { if (fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32)) { - float quantile = - this.quantile == null - ? calculateDefaultQuantile(fieldInfo.getVectorDimension()) - : this.quantile; + float confidenceInterval = + this.confidenceInterval == null + ? calculateDefaultConfidenceInterval(fieldInfo.getVectorDimension()) + : this.confidenceInterval; FieldWriter quantizedWriter = - new FieldWriter(quantile, fieldInfo, segmentWriteState.infoStream, indexWriter); + new FieldWriter(confidenceInterval, fieldInfo, segmentWriteState.infoStream, indexWriter); fields.add(quantizedWriter); indexWriter = quantizedWriter; } @@ -169,16 +169,16 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite DocsWithFieldSet docsWithField = writeQuantizedVectorData(quantizedVectorData, byteVectorValues); long vectorDataLength = quantizedVectorData.getFilePointer() - vectorDataOffset; - float quantile = - this.quantile == null - ? calculateDefaultQuantile(fieldInfo.getVectorDimension()) - : this.quantile; + float confidenceInterval = + this.confidenceInterval == null + ? calculateDefaultConfidenceInterval(fieldInfo.getVectorDimension()) + : this.confidenceInterval; writeMeta( fieldInfo, segmentWriteState.segmentInfo.maxDoc(), vectorDataOffset, vectorDataLength, - quantile, + confidenceInterval, mergedQuantizationState.getLowerQuantile(), mergedQuantizationState.getUpperQuantile(), docsWithField); @@ -251,7 +251,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite maxDoc, vectorDataOffset, vectorDataLength, - quantile, + confidenceInterval, fieldData.minQuantile, fieldData.maxQuantile, fieldData.docsWithField); @@ -262,7 +262,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite int maxDoc, long vectorDataOffset, long vectorDataLength, - Float configuredQuantizationQuantile, + Float confidenceInterval, Float lowerQuantile, Float upperQuantile, DocsWithFieldSet docsWithField) @@ -279,9 +279,9 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite assert Float.isFinite(lowerQuantile) && Float.isFinite(upperQuantile); meta.writeInt( Float.floatToIntBits( - configuredQuantizationQuantile != null - ? configuredQuantizationQuantile - : calculateDefaultQuantile(field.getVectorDimension()))); + confidenceInterval != null + ? confidenceInterval + : calculateDefaultConfidenceInterval(field.getVectorDimension()))); meta.writeInt(Float.floatToIntBits(lowerQuantile)); meta.writeInt(Float.floatToIntBits(upperQuantile)); } @@ -344,7 +344,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite maxDoc, vectorDataOffset, quantizedVectorLength, - quantile, + confidenceInterval, fieldData.minQuantile, fieldData.maxQuantile, newDocsWithField); @@ -374,11 +374,11 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite private ScalarQuantizer mergeQuantiles(FieldInfo fieldInfo, MergeState mergeState) throws IOException { assert fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT32; - float quantile = - this.quantile == null - ? calculateDefaultQuantile(fieldInfo.getVectorDimension()) - : this.quantile; - return mergeAndRecalculateQuantiles(mergeState, fieldInfo, quantile); + float confidenceInterval = + this.confidenceInterval == null + ? calculateDefaultConfidenceInterval(fieldInfo.getVectorDimension()) + : this.confidenceInterval; + return mergeAndRecalculateQuantiles(mergeState, fieldInfo, confidenceInterval); } private ScalarQuantizedCloseableRandomVectorScorerSupplier mergeOneFieldToIndex( @@ -408,16 +408,16 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite quantizationDataInput, quantizationDataInput.length() - CodecUtil.footerLength()); long vectorDataLength = quantizedVectorData.getFilePointer() - vectorDataOffset; CodecUtil.retrieveChecksum(quantizationDataInput); - float quantile = - this.quantile == null - ? calculateDefaultQuantile(fieldInfo.getVectorDimension()) - : this.quantile; + float confidenceInterval = + this.confidenceInterval == null + ? calculateDefaultConfidenceInterval(fieldInfo.getVectorDimension()) + : this.confidenceInterval; writeMeta( fieldInfo, segmentWriteState.segmentInfo.maxDoc(), vectorDataOffset, vectorDataLength, - quantile, + confidenceInterval, mergedQuantizationState.getLowerQuantile(), mergedQuantizationState.getUpperQuantile(), docsWithField); @@ -446,7 +446,9 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite } static ScalarQuantizer mergeQuantiles( - List quantizationStates, List segmentSizes, float quantile) { + List quantizationStates, + List segmentSizes, + float confidenceInterval) { assert quantizationStates.size() == segmentSizes.size(); if (quantizationStates.isEmpty()) { return null; @@ -464,7 +466,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite } lowerQuantile /= totalCount; upperQuantile /= totalCount; - return new ScalarQuantizer(lowerQuantile, upperQuantile, quantile); + return new ScalarQuantizer(lowerQuantile, upperQuantile, confidenceInterval); } /** @@ -521,7 +523,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite } static ScalarQuantizer mergeAndRecalculateQuantiles( - MergeState mergeState, FieldInfo fieldInfo, float quantile) throws IOException { + MergeState mergeState, FieldInfo fieldInfo, float confidenceInterval) throws IOException { List quantizationStates = new ArrayList<>(mergeState.liveDocs.length); List segmentSizes = new ArrayList<>(mergeState.liveDocs.length); for (int i = 0; i < mergeState.liveDocs.length; i++) { @@ -536,7 +538,8 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite segmentSizes.add(fvv.size()); } } - ScalarQuantizer mergedQuantiles = mergeQuantiles(quantizationStates, segmentSizes, quantile); + ScalarQuantizer mergedQuantiles = + mergeQuantiles(quantizationStates, segmentSizes, confidenceInterval); // Segments no providing quantization state indicates that their quantiles were never // calculated. // To be safe, we should always recalculate given a sample set over all the float vectors in the @@ -545,7 +548,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite if (mergedQuantiles == null || shouldRecomputeQuantiles(mergedQuantiles, quantizationStates)) { FloatVectorValues vectorValues = KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); - mergedQuantiles = ScalarQuantizer.fromVectors(vectorValues, quantile); + mergedQuantiles = ScalarQuantizer.fromVectors(vectorValues, confidenceInterval); } return mergedQuantiles; } @@ -599,7 +602,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite private static final long SHALLOW_SIZE = shallowSizeOfInstance(FieldWriter.class); private final List floatVectors; private final FieldInfo fieldInfo; - private final float quantile; + private final float confidenceInterval; private final InfoStream infoStream; private final boolean normalize; private float minQuantile = Float.POSITIVE_INFINITY; @@ -609,12 +612,12 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite @SuppressWarnings("unchecked") FieldWriter( - float quantile, + float confidenceInterval, FieldInfo fieldInfo, InfoStream infoStream, KnnFieldVectorsWriter indexWriter) { super((KnnFieldVectorsWriter) indexWriter); - this.quantile = quantile; + this.confidenceInterval = confidenceInterval; this.fieldInfo = fieldInfo; this.normalize = fieldInfo.getVectorSimilarityFunction() == VectorSimilarityFunction.COSINE; this.floatVectors = new ArrayList<>(); @@ -635,15 +638,15 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite new FloatVectorWrapper( floatVectors, fieldInfo.getVectorSimilarityFunction() == VectorSimilarityFunction.COSINE), - quantile); + confidenceInterval); minQuantile = quantizer.getLowerQuantile(); maxQuantile = quantizer.getUpperQuantile(); if (infoStream.isEnabled(QUANTIZED_VECTOR_COMPONENT)) { infoStream.message( QUANTIZED_VECTOR_COMPONENT, "quantized field=" - + " quantile=" - + quantile + + " confidenceInterval=" + + confidenceInterval + " minQuantile=" + minQuantile + " maxQuantile=" @@ -654,7 +657,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite ScalarQuantizer createQuantizer() { assert finished; - return new ScalarQuantizer(minQuantile, maxQuantile, quantile); + return new ScalarQuantizer(minQuantile, maxQuantile, confidenceInterval); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/util/ScalarQuantizer.java b/lucene/core/src/java/org/apache/lucene/util/ScalarQuantizer.java index d5c5646caf0..41ec4fe7ef8 100644 --- a/lucene/core/src/java/org/apache/lucene/util/ScalarQuantizer.java +++ b/lucene/core/src/java/org/apache/lucene/util/ScalarQuantizer.java @@ -28,15 +28,15 @@ import org.apache.lucene.index.VectorSimilarityFunction; /** * Will scalar quantize float vectors into `int8` byte values. This is a lossy transformation. * Scalar quantization works by first calculating the quantiles of the float vector values. The - * quantiles are calculated using the configured quantile/confidence interval. The [minQuantile, - * maxQuantile] are then used to scale the values into the range [0, 127] and bucketed into the - * nearest byte values. + * quantiles are calculated using the configured confidence interval. The [minQuantile, maxQuantile] + * are then used to scale the values into the range [0, 127] and bucketed into the nearest byte + * values. * *

How Scalar Quantization Works

* - *

The basic mathematical equations behind this are fairly straight forward. Given a float vector - * `v` and a quantile `q` we can calculate the quantiles of the vector values [minQuantile, - * maxQuantile]. + *

The basic mathematical equations behind this are fairly straight forward and based on min/max + * normalization. Given a float vector `v` and a confidenceInterval `q` we can calculate the + * quantiles of the vector values [minQuantile, maxQuantile]. * *

  *   byte = (float - minQuantile) * 127/(maxQuantile - minQuantile)
@@ -69,21 +69,20 @@ public class ScalarQuantizer {
 
   private final float alpha;
   private final float scale;
-  private final float minQuantile, maxQuantile, configuredQuantile;
+  private final float minQuantile, maxQuantile, confidenceInterval;
 
   /**
    * @param minQuantile the lower quantile of the distribution
    * @param maxQuantile the upper quantile of the distribution
-   * @param configuredQuantile The configured quantile/confidence interval used to calculate the
-   *     quantiles.
+   * @param confidenceInterval The configured confidence interval used to calculate the quantiles.
    */
-  public ScalarQuantizer(float minQuantile, float maxQuantile, float configuredQuantile) {
+  public ScalarQuantizer(float minQuantile, float maxQuantile, float confidenceInterval) {
     assert maxQuantile >= minQuantile;
     this.minQuantile = minQuantile;
     this.maxQuantile = maxQuantile;
     this.scale = 127f / (maxQuantile - minQuantile);
     this.alpha = (maxQuantile - minQuantile) / 127f;
-    this.configuredQuantile = configuredQuantile;
+    this.confidenceInterval = confidenceInterval;
   }
 
   /**
@@ -171,8 +170,8 @@ public class ScalarQuantizer {
     return maxQuantile;
   }
 
-  public float getConfiguredQuantile() {
-    return configuredQuantile;
+  public float getConfidenceInterval() {
+    return confidenceInterval;
   }
 
   public float getConstantMultiplier() {
@@ -186,8 +185,8 @@ public class ScalarQuantizer {
         + minQuantile
         + ", maxQuantile="
         + maxQuantile
-        + ", configuredQuantile="
-        + configuredQuantile
+        + ", confidenceInterval="
+        + confidenceInterval
         + '}';
   }
 
@@ -201,17 +200,17 @@ public class ScalarQuantizer {
    * #SCALAR_QUANTIZATION_SAMPLE_SIZE} will be read and the quantiles calculated.
    *
    * @param floatVectorValues the float vector values from which to calculate the quantiles
-   * @param quantile the quantile/confidence interval used to calculate the quantiles
+   * @param confidenceInterval the confidence interval used to calculate the quantiles
    * @return A new {@link ScalarQuantizer} instance
    * @throws IOException if there is an error reading the float vector values
    */
-  public static ScalarQuantizer fromVectors(FloatVectorValues floatVectorValues, float quantile)
-      throws IOException {
-    assert 0.9f <= quantile && quantile <= 1f;
+  public static ScalarQuantizer fromVectors(
+      FloatVectorValues floatVectorValues, float confidenceInterval) throws IOException {
+    assert 0.9f <= confidenceInterval && confidenceInterval <= 1f;
     if (floatVectorValues.size() == 0) {
-      return new ScalarQuantizer(0f, 0f, quantile);
+      return new ScalarQuantizer(0f, 0f, confidenceInterval);
     }
-    if (quantile == 1f) {
+    if (confidenceInterval == 1f) {
       float min = Float.POSITIVE_INFINITY;
       float max = Float.NEGATIVE_INFINITY;
       while (floatVectorValues.nextDoc() != NO_MORE_DOCS) {
@@ -220,7 +219,7 @@ public class ScalarQuantizer {
           max = Math.max(max, v);
         }
       }
-      return new ScalarQuantizer(min, max, quantile);
+      return new ScalarQuantizer(min, max, confidenceInterval);
     }
     int dim = floatVectorValues.dimension();
     if (floatVectorValues.size() < SCALAR_QUANTIZATION_SAMPLE_SIZE) {
@@ -231,8 +230,8 @@ public class ScalarQuantizer {
         System.arraycopy(floatVector, 0, values, copyOffset, floatVector.length);
         copyOffset += dim;
       }
-      float[] upperAndLower = getUpperAndLowerQuantile(values, quantile);
-      return new ScalarQuantizer(upperAndLower[0], upperAndLower[1], quantile);
+      float[] upperAndLower = getUpperAndLowerQuantile(values, confidenceInterval);
+      return new ScalarQuantizer(upperAndLower[0], upperAndLower[1], confidenceInterval);
     }
     int numFloatVecs = floatVectorValues.size();
     // Reservoir sample the vector ordinals we want to read
@@ -258,22 +257,23 @@ public class ScalarQuantizer {
       System.arraycopy(floatVector, 0, values, copyOffset, floatVector.length);
       copyOffset += dim;
     }
-    float[] upperAndLower = getUpperAndLowerQuantile(values, quantile);
-    return new ScalarQuantizer(upperAndLower[0], upperAndLower[1], quantile);
+    float[] upperAndLower = getUpperAndLowerQuantile(values, confidenceInterval);
+    return new ScalarQuantizer(upperAndLower[0], upperAndLower[1], confidenceInterval);
   }
 
   /**
    * Takes an array of floats, sorted or not, and returns a minimum and maximum value. These values
-   * are such that they reside on the `(1 - quantile)/2` and `quantile/2` percentiles. Example:
-   * providing floats `[0..100]` and asking for `90` quantiles will return `5` and `95`.
+   * are such that they reside on the `(1 - confidenceInterval)/2` and `confidenceInterval/2`
+   * percentiles. Example: providing floats `[0..100]` and asking for `90` quantiles will return `5`
+   * and `95`.
    *
    * @param arr array of floats
-   * @param quantileFloat the configured quantile
+   * @param confidenceInterval the configured confidence interval
    * @return lower and upper quantile values
    */
-  static float[] getUpperAndLowerQuantile(float[] arr, float quantileFloat) {
-    assert 0.9f <= quantileFloat && quantileFloat <= 1f;
-    int selectorIndex = (int) (arr.length * (1f - quantileFloat) / 2f + 0.5f);
+  static float[] getUpperAndLowerQuantile(float[] arr, float confidenceInterval) {
+    assert 0.9f <= confidenceInterval && confidenceInterval <= 1f;
+    int selectorIndex = (int) (arr.length * (1f - confidenceInterval) / 2f + 0.5f);
     if (selectorIndex > 0) {
       Selector selector = new FloatSelector(arr);
       selector.select(0, arr.length, arr.length - selectorIndex);
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
index f8e1fc128db..2c866bd9474 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
@@ -37,6 +37,7 @@ import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
+import org.apache.lucene.util.SameThreadExecutorService;
 import org.apache.lucene.util.ScalarQuantizer;
 import org.apache.lucene.util.VectorUtil;
 
@@ -64,11 +65,12 @@ public class TestLucene99HnswQuantizedVectorsFormat extends BaseKnnVectorsFormat
     for (int i = 0; i < numVectors; i++) {
       vectors.add(randomVector(dim));
     }
-    float quantile = Lucene99ScalarQuantizedVectorsFormat.calculateDefaultQuantile(dim);
+    float confidenceInterval =
+        Lucene99ScalarQuantizedVectorsFormat.calculateDefaultConfidenceInterval(dim);
     ScalarQuantizer scalarQuantizer =
         ScalarQuantizer.fromVectors(
             new Lucene99ScalarQuantizedVectorsWriter.FloatVectorWrapper(vectors, normalize),
-            quantile);
+            confidenceInterval);
     float[] expectedCorrections = new float[numVectors];
     byte[][] expectedVectors = new byte[numVectors][];
     for (int i = 0; i < numVectors; i++) {
@@ -148,7 +150,38 @@ public class TestLucene99HnswQuantizedVectorsFormat extends BaseKnnVectorsFormat
           }
         };
     String expectedString =
-        "Lucene99HnswScalarQuantizedVectorsFormat(name=Lucene99HnswScalarQuantizedVectorsFormat, maxConn=10, beamWidth=20, flatVectorFormat=Lucene99ScalarQuantizedVectorsFormat(name=Lucene99ScalarQuantizedVectorsFormat, quantile=0.9, rawVectorFormat=Lucene99FlatVectorsFormat()))";
+        "Lucene99HnswScalarQuantizedVectorsFormat(name=Lucene99HnswScalarQuantizedVectorsFormat, maxConn=10, beamWidth=20, flatVectorFormat=Lucene99ScalarQuantizedVectorsFormat(name=Lucene99ScalarQuantizedVectorsFormat, confidenceInterval=0.9, rawVectorFormat=Lucene99FlatVectorsFormat()))";
     assertEquals(expectedString, customCodec.knnVectorsFormat().toString());
   }
+
+  public void testLimits() {
+    expectThrows(
+        IllegalArgumentException.class, () -> new Lucene99HnswScalarQuantizedVectorsFormat(-1, 20));
+    expectThrows(
+        IllegalArgumentException.class, () -> new Lucene99HnswScalarQuantizedVectorsFormat(0, 20));
+    expectThrows(
+        IllegalArgumentException.class, () -> new Lucene99HnswScalarQuantizedVectorsFormat(20, 0));
+    expectThrows(
+        IllegalArgumentException.class, () -> new Lucene99HnswScalarQuantizedVectorsFormat(20, -1));
+    expectThrows(
+        IllegalArgumentException.class,
+        () -> new Lucene99HnswScalarQuantizedVectorsFormat(512 + 1, 20));
+    expectThrows(
+        IllegalArgumentException.class,
+        () -> new Lucene99HnswScalarQuantizedVectorsFormat(20, 3201));
+    expectThrows(
+        IllegalArgumentException.class,
+        () -> new Lucene99HnswScalarQuantizedVectorsFormat(20, 100, 0, 1.1f, null));
+    expectThrows(
+        IllegalArgumentException.class,
+        () -> new Lucene99HnswScalarQuantizedVectorsFormat(20, 100, 0, 0.8f, null));
+    expectThrows(
+        IllegalArgumentException.class,
+        () -> new Lucene99HnswScalarQuantizedVectorsFormat(20, 100, 100, null, null));
+    expectThrows(
+        IllegalArgumentException.class,
+        () ->
+            new Lucene99HnswScalarQuantizedVectorsFormat(
+                20, 100, 1, null, new SameThreadExecutorService()));
+  }
 }
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java
index 085a203dad9..c444802679a 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java
@@ -21,6 +21,7 @@ import org.apache.lucene.codecs.FilterCodec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
 import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.SameThreadExecutorService;
 
 public class TestLucene99HnswVectorsFormat extends BaseKnnVectorsFormatTestCase {
   @Override
@@ -48,5 +49,10 @@ public class TestLucene99HnswVectorsFormat extends BaseKnnVectorsFormatTestCase
     expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, -1));
     expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(512 + 1, 20));
     expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 3201));
+    expectThrows(
+        IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 100, 100, null));
+    expectThrows(
+        IllegalArgumentException.class,
+        () -> new Lucene99HnswVectorsFormat(20, 100, 1, new SameThreadExecutorService()));
   }
 }
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestScalarQuantizedVectorSimilarity.java b/lucene/core/src/test/org/apache/lucene/util/TestScalarQuantizedVectorSimilarity.java
index 9e3996cca9c..addcbf23188 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestScalarQuantizedVectorSimilarity.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestScalarQuantizedVectorSimilarity.java
@@ -32,10 +32,11 @@ public class TestScalarQuantizedVectorSimilarity extends LuceneTestCase {
     int numVecs = 100;
 
     float[][] floats = randomFloats(numVecs, dims);
-    for (float quantile : new float[] {0.9f, 0.95f, 0.99f, (1 - 1f / (dims + 1)), 1f}) {
-      float error = Math.max((100 - quantile) * 0.01f, 0.01f);
+    for (float confidenceInterval : new float[] {0.9f, 0.95f, 0.99f, (1 - 1f / (dims + 1)), 1f}) {
+      float error = Math.max((100 - confidenceInterval) * 0.01f, 0.01f);
       FloatVectorValues floatVectorValues = fromFloats(floats);
-      ScalarQuantizer scalarQuantizer = ScalarQuantizer.fromVectors(floatVectorValues, quantile);
+      ScalarQuantizer scalarQuantizer =
+          ScalarQuantizer.fromVectors(floatVectorValues, confidenceInterval);
       byte[][] quantized = new byte[floats.length][];
       float[] offsets =
           quantizeVectors(scalarQuantizer, floats, quantized, VectorSimilarityFunction.EUCLIDEAN);
@@ -61,10 +62,11 @@ public class TestScalarQuantizedVectorSimilarity extends LuceneTestCase {
 
     float[][] floats = randomFloats(numVecs, dims);
 
-    for (float quantile : new float[] {0.9f, 0.95f, 0.99f, (1 - 1f / (dims + 1)), 1f}) {
-      float error = Math.max((100 - quantile) * 0.01f, 0.01f);
+    for (float confidenceInterval : new float[] {0.9f, 0.95f, 0.99f, (1 - 1f / (dims + 1)), 1f}) {
+      float error = Math.max((100 - confidenceInterval) * 0.01f, 0.01f);
       FloatVectorValues floatVectorValues = fromFloatsNormalized(floats);
-      ScalarQuantizer scalarQuantizer = ScalarQuantizer.fromVectors(floatVectorValues, quantile);
+      ScalarQuantizer scalarQuantizer =
+          ScalarQuantizer.fromVectors(floatVectorValues, confidenceInterval);
       byte[][] quantized = new byte[floats.length][];
       float[] offsets =
           quantizeVectorsNormalized(
@@ -94,10 +96,11 @@ public class TestScalarQuantizedVectorSimilarity extends LuceneTestCase {
     for (float[] fs : floats) {
       VectorUtil.l2normalize(fs);
     }
-    for (float quantile : new float[] {0.9f, 0.95f, 0.99f, (1 - 1f / (dims + 1)), 1f}) {
-      float error = Math.max((100 - quantile) * 0.01f, 0.01f);
+    for (float confidenceInterval : new float[] {0.9f, 0.95f, 0.99f, (1 - 1f / (dims + 1)), 1f}) {
+      float error = Math.max((100 - confidenceInterval) * 0.01f, 0.01f);
       FloatVectorValues floatVectorValues = fromFloats(floats);
-      ScalarQuantizer scalarQuantizer = ScalarQuantizer.fromVectors(floatVectorValues, quantile);
+      ScalarQuantizer scalarQuantizer =
+          ScalarQuantizer.fromVectors(floatVectorValues, confidenceInterval);
       byte[][] quantized = new byte[floats.length][];
       float[] offsets =
           quantizeVectors(scalarQuantizer, floats, quantized, VectorSimilarityFunction.DOT_PRODUCT);
@@ -123,10 +126,11 @@ public class TestScalarQuantizedVectorSimilarity extends LuceneTestCase {
     int numVecs = 100;
 
     float[][] floats = randomFloats(numVecs, dims);
-    for (float quantile : new float[] {0.9f, 0.95f, 0.99f, (1 - 1f / (dims + 1)), 1f}) {
-      float error = Math.max((100 - quantile) * 0.5f, 0.5f);
+    for (float confidenceInterval : new float[] {0.9f, 0.95f, 0.99f, (1 - 1f / (dims + 1)), 1f}) {
+      float error = Math.max((100 - confidenceInterval) * 0.5f, 0.5f);
       FloatVectorValues floatVectorValues = fromFloats(floats);
-      ScalarQuantizer scalarQuantizer = ScalarQuantizer.fromVectors(floatVectorValues, quantile);
+      ScalarQuantizer scalarQuantizer =
+          ScalarQuantizer.fromVectors(floatVectorValues, confidenceInterval);
       byte[][] quantized = new byte[floats.length][];
       float[] offsets =
           quantizeVectors(