From da6bd22fd934fea8e6f6c03d0f215abf4d8494da Mon Sep 17 00:00:00 2001 From: Alessandro Benedetti Date: Tue, 21 May 2024 11:32:26 +0200 Subject: [PATCH] Revert "Vectors Format Refactor first draft" This reverts commit 77cec4442306cf5410ff80f1739403e84ff6ae0f. --- ...RWHnswScalarQuantizationVectorsFormat.java | 12 +-- .../TestInt8HnswBackwardsCompatibility.java | 4 +- .../bitvectors/HnswBitVectorsFormat.java | 18 ++-- lucene/core/src/java/module-info.java | 6 +- .../lucene/codecs/KnnVectorsFormat.java | 2 +- .../lucene/codecs/hnsw/package-info.java | 4 +- .../lucene/codecs/lucene99/Lucene99Codec.java | 1 - .../Lucene99FlatVectorsFormat.java | 2 +- .../Lucene99FlatVectorsReader.java | 6 +- .../Lucene99FlatVectorsWriter.java | 4 +- ...ene99HnswScalarQuantizedVectorsFormat.java | 94 ++++++++++++++++--- .../graph => }/Lucene99HnswVectorsFormat.java | 38 ++++---- .../graph => }/Lucene99HnswVectorsReader.java | 2 +- .../graph => }/Lucene99HnswVectorsWriter.java | 6 +- .../Lucene99ScalarQuantizedVectorScorer.java | 2 +- .../Lucene99ScalarQuantizedVectorsFormat.java | 2 +- .../Lucene99ScalarQuantizedVectorsReader.java | 6 +- .../Lucene99ScalarQuantizedVectorsWriter.java | 12 +-- .../OffHeapQuantizedByteVectorValues.java | 2 +- .../lucene/codecs/lucene99/package-info.java | 4 +- .../org.apache.lucene.codecs.KnnVectorsFormat | 4 +- .../codecs/hnsw/TestFlatVectorScorer.java | 2 +- ...estLucene99HnswQuantizedVectorsFormat.java | 4 +- .../TestLucene99HnswVectorsFormat.java | 4 +- ...stLucene99ScalarQuantizedVectorScorer.java | 8 +- .../TestPerFieldKnnVectorsFormat.java | 2 +- .../org/apache/lucene/index/TestKnnGraph.java | 6 +- .../lucene/util/hnsw/HnswGraphTestCase.java | 4 +- .../codecs/vector/ConfigurableMCodec.java | 2 +- .../apache/lucene/tests/util/TestUtil.java | 2 +- 30 files changed, 163 insertions(+), 102 deletions(-) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/storage => }/Lucene99FlatVectorsFormat.java (98%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/storage => }/Lucene99FlatVectorsReader.java (97%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/storage => }/Lucene99FlatVectorsWriter.java (99%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/graph => }/Lucene99HnswScalarQuantizedVectorsFormat.java (52%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/graph => }/Lucene99HnswVectorsFormat.java (85%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/graph => }/Lucene99HnswVectorsReader.java (99%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/graph => }/Lucene99HnswVectorsWriter.java (98%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/storage => }/Lucene99ScalarQuantizedVectorScorer.java (99%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/storage => }/Lucene99ScalarQuantizedVectorsFormat.java (98%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/storage => }/Lucene99ScalarQuantizedVectorsReader.java (98%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/storage => }/Lucene99ScalarQuantizedVectorsWriter.java (98%) rename lucene/core/src/java/org/apache/lucene/codecs/lucene99/{vectors/storage => }/OffHeapQuantizedByteVectorValues.java (99%) rename lucene/core/src/test/org/apache/lucene/codecs/lucene99/{vectors/graph => }/TestLucene99HnswQuantizedVectorsFormat.java (98%) rename lucene/core/src/test/org/apache/lucene/codecs/lucene99/{vectors/graph => }/TestLucene99HnswVectorsFormat.java (93%) rename lucene/core/src/test/org/apache/lucene/codecs/lucene99/{vectors/storage => }/TestLucene99ScalarQuantizedVectorScorer.java (96%) diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java index a55d97f03d9..a1d7648d077 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWHnswScalarQuantizationVectorsFormat.java @@ -23,12 +23,12 @@ import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; import org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer; -import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsWriter; -import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsWriter; import org.apache.lucene.index.SegmentWriteState; class Lucene99RWHnswScalarQuantizationVectorsFormat diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java index 12543111bd6..d840a00ca5b 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java @@ -23,8 +23,8 @@ import java.io.IOException; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java index 0a30fa17e80..f5888c0a03c 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java @@ -17,11 +17,11 @@ package org.apache.lucene.codecs.bitvectors; -import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; -import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; -import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.MAXIMUM_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN; import java.io.IOException; import java.util.concurrent.ExecutorService; @@ -30,10 +30,10 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentReadState; diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.java index 69c749beacb..94ff818c499 100644 --- a/lucene/core/src/java/module-info.java +++ b/lucene/core/src/java/module-info.java @@ -64,8 +64,6 @@ module org.apache.lucene.core { exports org.apache.lucene.util.quantization; exports org.apache.lucene.codecs.hnsw; - exports org.apache.lucene.codecs.lucene99.vectors.storage; - exports org.apache.lucene.codecs.lucene99.vectors.graph; provides org.apache.lucene.analysis.TokenizerFactory with org.apache.lucene.analysis.standard.StandardTokenizerFactory; @@ -74,8 +72,8 @@ module org.apache.lucene.core { provides org.apache.lucene.codecs.DocValuesFormat with org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; provides org.apache.lucene.codecs.KnnVectorsFormat with - org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat, - org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat; + org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat, + org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat; provides org.apache.lucene.codecs.PostingsFormat with org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; provides org.apache.lucene.index.SortFieldProvider with diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java index 0aca5658464..840d3242c03 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java @@ -55,7 +55,7 @@ public abstract class KnnVectorsFormat implements NamedSPILoader.NamedSPI { } } - protected final String name; + private final String name; /** Sole constructor */ protected KnnVectorsFormat(String name) { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java index 73850af955a..4907ad67cd4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/package-info.java @@ -21,7 +21,7 @@ * storage formats or scoring without significant changes to the HNSW code. Some examples for * scoring include {@link org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer} and {@link * org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer}. Some examples for storing include {@link - * org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat} and {@link - * org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsFormat}. + * org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat} and {@link + * org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat}. */ package org.apache.lucene.codecs.hnsw; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java index 7432b7e7cfc..ce6c40f9402 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java @@ -20,7 +20,6 @@ import java.util.Objects; import org.apache.lucene.codecs.*; import org.apache.lucene.codecs.lucene90.*; import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsFormat.java similarity index 98% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsFormat.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsFormat.java index ee6be687f65..dc6fe4e7178 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsFormat.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.storage; +package org.apache.lucene.codecs.lucene99; import java.io.IOException; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java similarity index 97% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsReader.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java index afdf06d467a..0613c9c82b8 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.storage; +package org.apache.lucene.codecs.lucene99; -import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader.readSimilarityFunction; -import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader.readVectorEncoding; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; import java.io.IOException; import java.util.HashMap; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java similarity index 99% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsWriter.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java index d81c9316da7..288a6ae6df9 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99FlatVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.storage; +package org.apache.lucene.codecs.lucene99; -import static org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; +import static org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import java.io.Closeable; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java similarity index 52% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswScalarQuantizedVectorsFormat.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java index 9409ce5de6c..ff41ac01a96 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswScalarQuantizedVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java @@ -15,11 +15,24 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.graph; +package org.apache.lucene.codecs.lucene99; -import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsFormat; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN; +import java.io.IOException; import java.util.concurrent.ExecutorService; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.search.TaskExecutor; +import org.apache.lucene.util.hnsw.HnswGraph; /** * Lucene 9.9 vector format, which encodes numeric vector values into an associated graph connecting @@ -29,20 +42,32 @@ import java.util.concurrent.ExecutorService; * * @lucene.experimental */ -public class Lucene99HnswScalarQuantizedVectorsFormat extends Lucene99HnswVectorsFormat { +public class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat { public static final String NAME = "Lucene99HnswScalarQuantizedVectorsFormat"; - + /** - * to motivate + * Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to + * {@link Lucene99HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details. */ - public static final int DEFAULT_QUANTIZATION_BITS = 7; + private final int maxConn; + /** + * The number of candidate neighbors to track while searching the graph for each newly inserted + * node. Defaults to {@link Lucene99HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link HnswGraph} + * for details. + */ + private final int beamWidth; + /** The format for storing, reading, merging vectors on disk */ + private final FlatVectorsFormat flatVectorsFormat; + + private final int numMergeWorkers; + private final TaskExecutor mergeExec; /** Constructs a format using default graph construction parameters */ public Lucene99HnswScalarQuantizedVectorsFormat() { - this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, DEFAULT_QUANTIZATION_BITS, true, null, null); + this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, 7, true, null, null); } /** @@ -52,7 +77,7 @@ public class Lucene99HnswScalarQuantizedVectorsFormat extends Lucene99HnswVector * @param beamWidth the size of the queue maintained during graph construction. */ public Lucene99HnswScalarQuantizedVectorsFormat(int maxConn, int beamWidth) { - this(maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, DEFAULT_QUANTIZATION_BITS, true, null, null); + this(maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, 7, true, null, null); } /** @@ -80,11 +105,58 @@ public class Lucene99HnswScalarQuantizedVectorsFormat extends Lucene99HnswVector boolean compress, Float confidenceInterval, ExecutorService mergeExec) { - super(NAME, maxConn, beamWidth, numMergeWorkers, mergeExec); - super.flatVectorsFormat = + super(NAME); + if (maxConn <= 0 || maxConn > MAXIMUM_MAX_CONN) { + throw new IllegalArgumentException( + "maxConn must be positive and less than or equal to " + + MAXIMUM_MAX_CONN + + "; maxConn=" + + maxConn); + } + if (beamWidth <= 0 || beamWidth > MAXIMUM_BEAM_WIDTH) { + throw new IllegalArgumentException( + "beamWidth must be positive and less than or equal to " + + MAXIMUM_BEAM_WIDTH + + "; beamWidth=" + + beamWidth); + } + this.maxConn = maxConn; + this.beamWidth = beamWidth; + if (numMergeWorkers == 1 && mergeExec != null) { + throw new IllegalArgumentException( + "No executor service is needed as we'll use single thread to merge"); + } + this.numMergeWorkers = numMergeWorkers; + if (mergeExec != null) { + this.mergeExec = new TaskExecutor(mergeExec); + } else { + this.mergeExec = null; + } + this.flatVectorsFormat = new Lucene99ScalarQuantizedVectorsFormat(confidenceInterval, bits, compress); } - + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new Lucene99HnswVectorsWriter( + state, + maxConn, + beamWidth, + flatVectorsFormat.fieldsWriter(state), + numMergeWorkers, + mergeExec); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state)); + } + + @Override + public int getMaxDimensions(String fieldName) { + return 1024; + } + @Override public String toString() { return "Lucene99HnswScalarQuantizedVectorsFormat(name=Lucene99HnswScalarQuantizedVectorsFormat, maxConn=" diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java similarity index 85% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsFormat.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java index fc32b5d67c8..8c78a0cb0a0 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.graph; +package org.apache.lucene.codecs.lucene99; import java.io.IOException; import java.util.concurrent.ExecutorService; @@ -25,7 +25,6 @@ import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.lucene90.IndexedDISI; -import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.index.SegmentReadState; @@ -87,8 +86,8 @@ import org.apache.lucene.util.hnsw.HnswGraph; * * @lucene.experimental */ -public class Lucene99HnswVectorsFormat extends KnnVectorsFormat { - static final String NAME = "Lucene99HnswVectorsFormat"; +public final class Lucene99HnswVectorsFormat extends KnnVectorsFormat { + static final String META_CODEC_NAME = "Lucene99HnswVectorsFormatMeta"; static final String VECTOR_INDEX_CODEC_NAME = "Lucene99HnswVectorsFormatIndex"; static final String META_EXTENSION = "vem"; @@ -129,25 +128,25 @@ public class Lucene99HnswVectorsFormat extends KnnVectorsFormat { * Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to * {@link Lucene99HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details. */ - protected final int maxConn; + private final int maxConn; /** * The number of candidate neighbors to track while searching the graph for each newly inserted * node. Defaults to {@link Lucene99HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link HnswGraph} * for details. */ - protected final int beamWidth; + private final int beamWidth; /** The format for storing, reading, merging vectors on disk */ - protected static FlatVectorsFormat flatVectorsFormat = + private static final FlatVectorsFormat flatVectorsFormat = new Lucene99FlatVectorsFormat(new DefaultFlatVectorScorer()); - protected final int numMergeWorkers; - protected final TaskExecutor mergeExec; + private final int numMergeWorkers; + private final TaskExecutor mergeExec; /** Constructs a format using default graph construction parameters */ public Lucene99HnswVectorsFormat() { - this(NAME, DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, null); + this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, null); } /** @@ -157,24 +156,23 @@ public class Lucene99HnswVectorsFormat extends KnnVectorsFormat { * @param beamWidth the size of the queue maintained during graph construction. */ public Lucene99HnswVectorsFormat(int maxConn, int beamWidth) { - this(NAME, maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null); + this(maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null); } /** * Constructs a format using the given graph construction parameters and scalar quantization. * - * @param name name of the codec - * @param maxConn the maximum number of connections to a node in the HNSW graph - * @param beamWidth the size of the queue maintained during graph construction. + * @param maxConn the maximum number of connections to a node in the HNSW graph + * @param beamWidth the size of the queue maintained during graph construction. * @param numMergeWorkers number of workers (threads) that will be used when doing merge. If - * larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec - * @param mergeExec the {@link ExecutorService} that will be used by ALL vector writers that are - * generated by this format to do the merge. If null, the configured {@link - * MergeScheduler#getIntraMergeExecutor(MergePolicy.OneMerge)} is used. + * larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec + * @param mergeExec the {@link ExecutorService} that will be used by ALL vector writers that are + * generated by this format to do the merge. If null, the configured {@link + * MergeScheduler#getIntraMergeExecutor(MergePolicy.OneMerge)} is used. */ public Lucene99HnswVectorsFormat( - String name, int maxConn, int beamWidth, int numMergeWorkers, ExecutorService mergeExec) { - super(name); + int maxConn, int beamWidth, int numMergeWorkers, ExecutorService mergeExec) { + super("Lucene99HnswVectorsFormat"); if (maxConn <= 0 || maxConn > MAXIMUM_MAX_CONN) { throw new IllegalArgumentException( "maxConn must be positive and less than or equal to " diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java similarity index 99% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsReader.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java index 247cc3b0c1d..38908efbe68 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.graph; +package org.apache.lucene.codecs.lucene99; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java similarity index 98% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsWriter.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java index 3c7690b9869..8f715993a2b 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/graph/Lucene99HnswVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.graph; +package org.apache.lucene.codecs.lucene99; -import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; -import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; import java.io.IOException; import java.util.ArrayList; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java similarity index 99% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorScorer.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java index 0afaec33354..96c9358cc90 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorScorer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.storage; +package org.apache.lucene.codecs.lucene99; import static org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer.quantizeQuery; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java similarity index 98% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsFormat.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java index 04b01409f3f..c10f87da2a6 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.storage; +package org.apache.lucene.codecs.lucene99; import java.io.IOException; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java similarity index 98% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsReader.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java index 237a739ee51..5a20770bb98 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.storage; +package org.apache.lucene.codecs.lucene99; -import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader.readSimilarityFunction; -import static org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader.readVectorEncoding; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; import java.io.IOException; import java.util.HashMap; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java similarity index 98% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsWriter.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java index 7b3bc3574cc..ae13be02467 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/Lucene99ScalarQuantizedVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.storage; +package org.apache.lucene.codecs.lucene99; -import static org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; -import static org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsFormat.QUANTIZED_VECTOR_COMPONENT; -import static org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsFormat.calculateDefaultConfidenceInterval; +import static org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; +import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.QUANTIZED_VECTOR_COMPONENT; +import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.calculateDefaultConfidenceInterval; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; @@ -839,13 +839,13 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite } } - public static class FloatVectorWrapper extends FloatVectorValues { + static class FloatVectorWrapper extends FloatVectorValues { private final List vectorList; private final float[] copy; private final boolean normalize; protected int curDoc = -1; - public FloatVectorWrapper(List vectorList, boolean normalize) { + FloatVectorWrapper(List vectorList, boolean normalize) { this.vectorList = vectorList; this.copy = new float[vectorList.get(0).length]; this.normalize = normalize; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/OffHeapQuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java similarity index 99% rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/OffHeapQuantizedByteVectorValues.java rename to lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java index 1e71f9e40c1..655dcca1166 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/vectors/storage/OffHeapQuantizedByteVectorValues.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.storage; +package org.apache.lucene.codecs.lucene99; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java index e76425edf29..2df2cfc9a30 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java @@ -180,7 +180,7 @@ * of files, recording dimensionally indexed fields, to enable fast numeric range filtering * and large numeric values like BigInteger and BigDecimal (1D) and geographic shape * intersection (2D, 3D). - *
  • {@link org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat Vector values}. The + *
  • {@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values}. The * vector format stores numeric vectors in a format optimized for random access and * computation, supporting high-dimensional nearest-neighbor search. * @@ -310,7 +310,7 @@ * Holds indexed points * * - * {@link org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat Vector values} + * {@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values} * .vec, .vem, .veq, vex * Holds indexed vectors; .vec files contain the raw vector data, * .vem the vector metadata, .veq the quantized vector data, and .vex the diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index 42e8844ffdb..ba0de7b2464 100644 --- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -13,5 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat -org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat +org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat +org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat diff --git a/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java index dc8a5aeef97..5dbebf08df5 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java @@ -32,7 +32,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.codecs.lucene95.OffHeapByteVectorValues; import org.apache.lucene.codecs.lucene95.OffHeapFloatVectorValues; -import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorScorer; +import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorScorer; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java similarity index 98% rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswQuantizedVectorsFormat.java rename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java index f4ca15dbacf..ca815aa1169 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswQuantizedVectorsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.graph; +package org.apache.lucene.codecs.lucene99; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; @@ -25,8 +25,6 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.codecs.lucene99.vectors.storage.Lucene99ScalarQuantizedVectorsWriter; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.KnnFloatVectorField; diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java similarity index 93% rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswVectorsFormat.java rename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java index 217721bf4c2..0f84f8ab4ae 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/graph/TestLucene99HnswVectorsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.graph; +package org.apache.lucene.codecs.lucene99; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; @@ -51,6 +51,6 @@ public class TestLucene99HnswVectorsFormat extends BaseKnnVectorsFormatTestCase expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 3201)); expectThrows( IllegalArgumentException.class, - () -> new Lucene99HnswVectorsFormat("Lucene99HnswVectorsFormat", 20, 100, 1, new SameThreadExecutorService())); + () -> new Lucene99HnswVectorsFormat(20, 100, 1, new SameThreadExecutorService())); } } diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/storage/TestLucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java similarity index 96% rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/storage/TestLucene99ScalarQuantizedVectorScorer.java rename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java index 9522596d8ea..a244302692e 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/vectors/storage/TestLucene99ScalarQuantizedVectorScorer.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.lucene.codecs.lucene99.vectors.storage; +package org.apache.lucene.codecs.lucene99; -import static org.apache.lucene.codecs.lucene99.vectors.storage.OffHeapQuantizedByteVectorValues.compressBytes; +import static org.apache.lucene.codecs.lucene99.OffHeapQuantizedByteVectorValues.compressBytes; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -27,10 +27,6 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; diff --git a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java index beb63d70b03..45814144d10 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java @@ -29,7 +29,7 @@ import org.apache.lucene.codecs.KnnFieldVectorsWriter; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.KnnFloatVectorField; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java index ad958ab63c9..72be0bd929f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java @@ -32,9 +32,9 @@ import java.util.concurrent.CountDownLatch; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswScalarQuantizedVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader; +import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java index f013ad00861..72a4841581d 100644 --- a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java +++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java @@ -41,8 +41,8 @@ import java.util.stream.Collectors; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsReader; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java index c50ccd2e55e..11aa4aced97 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java @@ -18,7 +18,7 @@ package org.apache.lucene.tests.codecs.vector; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.tests.util.TestUtil; /** diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java index c08954ac12f..8bed935f7db 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java @@ -57,7 +57,7 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.codecs.lucene99.vectors.graph.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;