From 77cec4442306cf5410ff80f1739403e84ff6ae0f Mon Sep 17 00:00:00 2001 From: Alessandro Benedetti Date: Tue, 21 May 2024 11:30:13 +0200 Subject: [PATCH] Vectors Format Refactor first draft --- ...RWHnswScalarQuantizationVectorsFormat.java | 12 +-- .../TestInt8HnswBackwardsCompatibility.java | 4 +- .../bitvectors/HnswBitVectorsFormat.java | 18 ++-- lucene/core/src/java/module-info.java | 6 +- .../lucene/codecs/KnnVectorsFormat.java | 2 +- .../lucene/codecs/hnsw/package-info.java | 4 +- .../lucene/codecs/lucene99/Lucene99Codec.java | 1 + .../lucene/codecs/lucene99/package-info.java | 4 +- ...ene99HnswScalarQuantizedVectorsFormat.java | 94 +++---------------- .../graph}/Lucene99HnswVectorsFormat.java | 38 ++++---- .../graph}/Lucene99HnswVectorsReader.java | 2 +- .../graph}/Lucene99HnswVectorsWriter.java | 6 +- .../storage}/Lucene99FlatVectorsFormat.java | 2 +- .../storage}/Lucene99FlatVectorsReader.java | 6 +- .../storage}/Lucene99FlatVectorsWriter.java | 4 +- .../Lucene99ScalarQuantizedVectorScorer.java | 2 +- .../Lucene99ScalarQuantizedVectorsFormat.java | 2 +- .../Lucene99ScalarQuantizedVectorsReader.java | 6 +- .../Lucene99ScalarQuantizedVectorsWriter.java | 12 +-- .../OffHeapQuantizedByteVectorValues.java | 2 +- .../org.apache.lucene.codecs.KnnVectorsFormat | 4 +- .../codecs/hnsw/TestFlatVectorScorer.java | 2 +- ...estLucene99HnswQuantizedVectorsFormat.java | 4 +- .../graph}/TestLucene99HnswVectorsFormat.java | 4 +- ...stLucene99ScalarQuantizedVectorScorer.java | 8 +- .../TestPerFieldKnnVectorsFormat.java | 2 +- .../org/apache/lucene/index/TestKnnGraph.java | 6 +- .../lucene/util/hnsw/HnswGraphTestCase.java | 4 +- .../codecs/vector/ConfigurableMCodec.java | 2 +- .../apache/lucene/tests/util/TestUtil.java | 2 +- 30 files changed, 102 insertions(+), 163 deletions(-) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/graph}/Lucene99HnswScalarQuantizedVectorsFormat.java (52%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/graph}/Lucene99HnswVectorsFormat.java (85%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/graph}/Lucene99HnswVectorsReader.java (99%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/graph}/Lucene99HnswVectorsWriter.java (98%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/storage}/Lucene99FlatVectorsFormat.java (98%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/storage}/Lucene99FlatVectorsReader.java (97%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/storage}/Lucene99FlatVectorsWriter.java (99%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/storage}/Lucene99ScalarQuantizedVectorScorer.java (99%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/storage}/Lucene99ScalarQuantizedVectorsFormat.java (98%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/storage}/Lucene99ScalarQuantizedVectorsReader.java (98%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/storage}/Lucene99ScalarQuantizedVectorsWriter.java (98%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{ => vectors/storage}/OffHeapQuantizedByteVectorValues.java (99%) rename lucene/core/src/test/org/apache/lucene/codecs/lucene99/{ => vectors/graph}/TestLucene99HnswQuantizedVectorsFormat.java (98%) rename lucene/core/src/test/org/apache/lucene/codecs/lucene99/{ => vectors/graph}/TestLucene99HnswVectorsFormat.java (93%) rename lucene/core/src/test/org/apache/lucene/codecs/lucene99/{ => vectors/storage}/TestLucene99ScalarQuantizedVectorScorer.java (96%) diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java index a1d7648d077..a55d97f03d9 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java @@ -23,12 +23,12 @@ import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; import org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer; -import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; -import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsWriter; +import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsWriter; +import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsWriter; import org.apache.lucene.index.SegmentWriteState; class Lucene99RWHnswScalarQuantizationVectorsFormat diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java index d840a00ca5b..12543111bd6 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java @@ -23,8 +23,8 @@ import java.io.IOException; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java index f5888c0a03c..0a30fa17e80 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java @@ -17,11 +17,11 @@ package org.apache.lucene.codecs.bitvectors; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN; +import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; +import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.MAXIMUM_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN; import java.io.IOException; import java.util.concurrent.ExecutorService; @@ -30,10 +30,10 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; +import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsWriter; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentReadState; diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.java index 94ff818c499..69c749beacb 100644 --- a/lucene/core/src/java/module-info.java +++ b/lucene/core/src/java/module-info.java @@ -64,6 +64,8 @@ module org.apache.lucene.core { exports org.apache.lucene.util.quantization; exports org.apache.lucene.codecs.hnsw; + exports org.apache.lucene.codecs.lucene99.vectors.storage; + exports org.apache.lucene.codecs.lucene99.vectors.graph; provides org.apache.lucene.analysis.TokenizerFactory with org.apache.lucene.analysis.standard.StandardTokenizerFactory; @@ -72,8 +74,8 @@ module org.apache.lucene.core { provides org.apache.lucene.codecs.DocValuesFormat with org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; provides org.apache.lucene.codecs.KnnVectorsFormat with - org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat, - org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat; + org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat, + org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat; provides org.apache.lucene.codecs.PostingsFormat with org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; provides org.apache.lucene.index.SortFieldProvider with diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java index 840d3242c03..0aca5658464 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java @@ -55,7 +55,7 @@ public abstract class KnnVectorsFormat implements NamedSPILoader.NamedSPI { } } - private final String name; + protected final String name; /** Sole constructor */ protected KnnVectorsFormat(String name) { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java index 4907ad67cd4..73850af955a 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java @@ -21,7 +21,7 @@ * storage formats or scoring without significant changes to the HNSW code. Some examples for * scoring include {@link org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer} and {@link * org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer}. Some examples for storing include {@link - * org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat} and {@link - * org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat}. + * org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat} and {@link + * org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsFormat}. */ package org.apache.lucene.codecs.hnsw; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java index ce6c40f9402..7432b7e7cfc 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java @@ -20,6 +20,7 @@ import java.util.Objects; import org.apache.lucene.codecs.*; import org.apache.lucene.codecs.lucene90.*; import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java index 2df2cfc9a30..e76425edf29 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java @@ -180,7 +180,7 @@ * of files, recording dimensionally indexed fields, to enable fast numeric range filtering * and large numeric values like BigInteger and BigDecimal (1D) and geographic shape * intersection (2D, 3D). - *
  • {@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values}. The + *
  • {@link org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat Vector values}. The * vector format stores numeric vectors in a format optimized for random access and * computation, supporting high-dimensional nearest-neighbor search. * @@ -310,7 +310,7 @@ * Holds indexed points * * - * {@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values} + * {@link org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat Vector values} * .vec, .vem, .veq, vex * Holds indexed vectors; .vec files contain the raw vector data, * .vem the vector metadata, .veq the quantized vector data, and .vex the diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswScalarQuantizedVectorsFormat.java similarity index 52% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswScalarQuantizedVectorsFormat.java index ff41ac01a96..9409ce5de6c 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswScalarQuantizedVectorsFormat.java @@ -15,24 +15,11 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.graph; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN; +import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsFormat; -import java.io.IOException; import java.util.concurrent.ExecutorService; -import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.KnnVectorsReader; -import org.apache.lucene.codecs.KnnVectorsWriter; -import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.search.TaskExecutor; -import org.apache.lucene.util.hnsw.HnswGraph; /** * Lucene 9.9 vector format, which encodes numeric vector values into an associated graph connecting @@ -42,32 +29,20 @@ import org.apache.lucene.util.hnsw.HnswGraph; * * @lucene.experimental */ -public class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat { +public class Lucene99HnswScalarQuantizedVectorsFormat extends Lucene99HnswVectorsFormat { public static final String NAME = "Lucene99HnswScalarQuantizedVectorsFormat"; - + /** - * Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to - * {@link Lucene99HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details. + * to motivate */ - private final int maxConn; + public static final int DEFAULT_QUANTIZATION_BITS = 7; - /** - * The number of candidate neighbors to track while searching the graph for each newly inserted - * node. Defaults to {@link Lucene99HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link HnswGraph} - * for details. - */ - private final int beamWidth; - /** The format for storing, reading, merging vectors on disk */ - private final FlatVectorsFormat flatVectorsFormat; - - private final int numMergeWorkers; - private final TaskExecutor mergeExec; /** Constructs a format using default graph construction parameters */ public Lucene99HnswScalarQuantizedVectorsFormat() { - this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, 7, true, null, null); + this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, DEFAULT_QUANTIZATION_BITS, true, null, null); } /** @@ -77,7 +52,7 @@ public class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat { * @param beamWidth the size of the queue maintained during graph construction. */ public Lucene99HnswScalarQuantizedVectorsFormat(int maxConn, int beamWidth) { - this(maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, 7, true, null, null); + this(maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, DEFAULT_QUANTIZATION_BITS, true, null, null); } /** @@ -105,58 +80,11 @@ public class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat { boolean compress, Float confidenceInterval, ExecutorService mergeExec) { - super(NAME); - if (maxConn <= 0 || maxConn > MAXIMUM_MAX_CONN) { - throw new IllegalArgumentException( - "maxConn must be positive and less than or equal to " - + MAXIMUM_MAX_CONN - + "; maxConn=" - + maxConn); - } - if (beamWidth <= 0 || beamWidth > MAXIMUM_BEAM_WIDTH) { - throw new IllegalArgumentException( - "beamWidth must be positive and less than or equal to " - + MAXIMUM_BEAM_WIDTH - + "; beamWidth=" - + beamWidth); - } - this.maxConn = maxConn; - this.beamWidth = beamWidth; - if (numMergeWorkers == 1 && mergeExec != null) { - throw new IllegalArgumentException( - "No executor service is needed as we'll use single thread to merge"); - } - this.numMergeWorkers = numMergeWorkers; - if (mergeExec != null) { - this.mergeExec = new TaskExecutor(mergeExec); - } else { - this.mergeExec = null; - } - this.flatVectorsFormat = + super(NAME, maxConn, beamWidth, numMergeWorkers, mergeExec); + super.flatVectorsFormat = new Lucene99ScalarQuantizedVectorsFormat(confidenceInterval, bits, compress); } - - @Override - public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return new Lucene99HnswVectorsWriter( - state, - maxConn, - beamWidth, - flatVectorsFormat.fieldsWriter(state), - numMergeWorkers, - mergeExec); - } - - @Override - public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { - return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state)); - } - - @Override - public int getMaxDimensions(String fieldName) { - return 1024; - } - + @Override public String toString() { return "Lucene99HnswScalarQuantizedVectorsFormat(name=Lucene99HnswScalarQuantizedVectorsFormat, maxConn=" diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsFormat.java similarity index 85% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsFormat.java index 8c78a0cb0a0..fc32b5d67c8 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsFormat.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.graph; import java.io.IOException; import java.util.concurrent.ExecutorService; @@ -25,6 +25,7 @@ import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.lucene90.IndexedDISI; +import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.index.SegmentReadState; @@ -86,8 +87,8 @@ import org.apache.lucene.util.hnsw.HnswGraph; * * @lucene.experimental */ -public final class Lucene99HnswVectorsFormat extends KnnVectorsFormat { - +public class Lucene99HnswVectorsFormat extends KnnVectorsFormat { + static final String NAME = "Lucene99HnswVectorsFormat"; static final String META_CODEC_NAME = "Lucene99HnswVectorsFormatMeta"; static final String VECTOR_INDEX_CODEC_NAME = "Lucene99HnswVectorsFormatIndex"; static final String META_EXTENSION = "vem"; @@ -128,25 +129,25 @@ public final class Lucene99HnswVectorsFormat extends KnnVectorsFormat { * Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to * {@link Lucene99HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details. */ - private final int maxConn; + protected final int maxConn; /** * The number of candidate neighbors to track while searching the graph for each newly inserted * node. Defaults to {@link Lucene99HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link HnswGraph} * for details. */ - private final int beamWidth; + protected final int beamWidth; /** The format for storing, reading, merging vectors on disk */ - private static final FlatVectorsFormat flatVectorsFormat = + protected static FlatVectorsFormat flatVectorsFormat = new Lucene99FlatVectorsFormat(new DefaultFlatVectorScorer()); - private final int numMergeWorkers; - private final TaskExecutor mergeExec; + protected final int numMergeWorkers; + protected final TaskExecutor mergeExec; /** Constructs a format using default graph construction parameters */ public Lucene99HnswVectorsFormat() { - this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, null); + this(NAME, DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, null); } /** @@ -156,23 +157,24 @@ public final class Lucene99HnswVectorsFormat extends KnnVectorsFormat { * @param beamWidth the size of the queue maintained during graph construction. */ public Lucene99HnswVectorsFormat(int maxConn, int beamWidth) { - this(maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null); + this(NAME, maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null); } /** * Constructs a format using the given graph construction parameters and scalar quantization. * - * @param maxConn the maximum number of connections to a node in the HNSW graph - * @param beamWidth the size of the queue maintained during graph construction. + * @param name name of the codec + * @param maxConn the maximum number of connections to a node in the HNSW graph + * @param beamWidth the size of the queue maintained during graph construction. * @param numMergeWorkers number of workers (threads) that will be used when doing merge. If - * larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec - * @param mergeExec the {@link ExecutorService} that will be used by ALL vector writers that are - * generated by this format to do the merge. If null, the configured {@link - * MergeScheduler#getIntraMergeExecutor(MergePolicy.OneMerge)} is used. + * larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec + * @param mergeExec the {@link ExecutorService} that will be used by ALL vector writers that are + * generated by this format to do the merge. If null, the configured {@link + * MergeScheduler#getIntraMergeExecutor(MergePolicy.OneMerge)} is used. */ public Lucene99HnswVectorsFormat( - int maxConn, int beamWidth, int numMergeWorkers, ExecutorService mergeExec) { - super("Lucene99HnswVectorsFormat"); + String name, int maxConn, int beamWidth, int numMergeWorkers, ExecutorService mergeExec) { + super(name); if (maxConn <= 0 || maxConn > MAXIMUM_MAX_CONN) { throw new IllegalArgumentException( "maxConn must be positive and less than or equal to " diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsReader.java similarity index 99% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsReader.java index 38908efbe68..247cc3b0c1d 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsReader.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.graph; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsWriter.java similarity index 98% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsWriter.java index 8f715993a2b..3c7690b9869 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsWriter.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.graph; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; +import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; +import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; import java.io.IOException; import java.util.ArrayList; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsFormat.java similarity index 98% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsFormat.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsFormat.java index dc6fe4e7178..ee6be687f65 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsFormat.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.storage; import java.io.IOException; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsReader.java similarity index 97% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsReader.java index 0613c9c82b8..afdf06d467a 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsReader.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.storage; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; +import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader.readSimilarityFunction; +import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader.readVectorEncoding; import java.io.IOException; import java.util.HashMap; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsWriter.java similarity index 99% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsWriter.java index 288a6ae6df9..d81c9316da7 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsWriter.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.storage; -import static org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; +import static org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import java.io.Closeable; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorScorer.java similarity index 99% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorScorer.java index 96c9358cc90..0afaec33354 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorScorer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.storage; import static org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer.quantizeQuery; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsFormat.java similarity index 98% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsFormat.java index c10f87da2a6..04b01409f3f 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsFormat.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.storage; import java.io.IOException; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsReader.java similarity index 98% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsReader.java index 5a20770bb98..237a739ee51 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsReader.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.storage; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; +import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader.readSimilarityFunction; +import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader.readVectorEncoding; import java.io.IOException; import java.util.HashMap; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsWriter.java similarity index 98% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsWriter.java index ae13be02467..7b3bc3574cc 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsWriter.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.storage; -import static org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; -import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.QUANTIZED_VECTOR_COMPONENT; -import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.calculateDefaultConfidenceInterval; +import static org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; +import static org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsFormat.QUANTIZED_VECTOR_COMPONENT; +import static org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsFormat.calculateDefaultConfidenceInterval; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; @@ -839,13 +839,13 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite } } - static class FloatVectorWrapper extends FloatVectorValues { + public static class FloatVectorWrapper extends FloatVectorValues { private final List vectorList; private final float[] copy; private final boolean normalize; protected int curDoc = -1; - FloatVectorWrapper(List vectorList, boolean normalize) { + public FloatVectorWrapper(List vectorList, boolean normalize) { this.vectorList = vectorList; this.copy = new float[vectorList.get(0).length]; this.normalize = normalize; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/OffHeapQuantizedByteVectorValues.java similarity index 99% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/OffHeapQuantizedByteVectorValues.java index 655dcca1166..1e71f9e40c1 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/OffHeapQuantizedByteVectorValues.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.storage; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index ba0de7b2464..42e8844ffdb 100644 --- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -13,5 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat -org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat +org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat +org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat diff --git a/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java index 5dbebf08df5..dc8a5aeef97 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java @@ -32,7 +32,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.codecs.lucene95.OffHeapByteVectorValues; import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues; -import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorScorer; +import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorScorer; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswQuantizedVectorsFormat.java similarity index 98% rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java rename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswQuantizedVectorsFormat.java index ca815aa1169..f4ca15dbacf 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswQuantizedVectorsFormat.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.graph; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; @@ -25,6 +25,8 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsWriter; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.KnnFloatVectorField; diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswVectorsFormat.java similarity index 93% rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java rename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswVectorsFormat.java index 0f84f8ab4ae..217721bf4c2 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswVectorsFormat.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.graph; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; @@ -51,6 +51,6 @@ public class TestLucene99HnswVectorsFormat extends BaseKnnVectorsFormatTestCase expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 3201)); expectThrows( IllegalArgumentException.class, - () -> new Lucene99HnswVectorsFormat(20, 100, 1, new SameThreadExecutorService())); + () -> new Lucene99HnswVectorsFormat("Lucene99HnswVectorsFormat", 20, 100, 1, new SameThreadExecutorService())); } } diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/storage/TestLucene99ScalarQuantizedVectorScorer.java similarity index 96% rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java rename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/storage/TestLucene99ScalarQuantizedVectorScorer.java index a244302692e..9522596d8ea 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/storage/TestLucene99ScalarQuantizedVectorScorer.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99; +package org.apache.lucene.codecs.lucene99.vectors.storage; -import static org.apache.lucene.codecs.lucene99.OffHeapQuantizedByteVectorValues.compressBytes; +import static org.apache.lucene.codecs.lucene99.vectors.storage.OffHeapQuantizedByteVectorValues.compressBytes; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -27,6 +27,10 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; diff --git a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java index 45814144d10..beb63d70b03 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java @@ -29,7 +29,7 @@ import org.apache.lucene.codecs.KnnFieldVectorsWriter; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.KnnFloatVectorField; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java index 72be0bd929f..ad958ab63c9 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java @@ -32,9 +32,9 @@ import java.util.concurrent.CountDownLatch; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java index 72a4841581d..f013ad00861 100644 --- a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java +++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java @@ -41,8 +41,8 @@ import java.util.stream.Collectors; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java index 11aa4aced97..c50ccd2e55e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java @@ -18,7 +18,7 @@ package org.apache.lucene.tests.codecs.vector; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; import org.apache.lucene.tests.util.TestUtil; /** diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java index 8bed935f7db..c08954ac12f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java @@ -57,7 +57,7 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;