Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename vector float classes and methods (#12105) 

We recently introduced KnnByteVectorField, KnnByteVectorQuery and ByteVectorValues. The corresponding float variants of the same classes don't follow the same naming convention: KnnVectorField, KnnVectoryQuery and VectorValues. Ideally their names would reflect that they are the float variant of the vector field, vector query and vector values.

This commit aims at clarifying this in the public facing API, by deprecating the current float classes in favour of new ones that are their exact copy but follow the same naming conventions as the byte ones.

As a result, LeafReader#getVectorValues is also deprecated in favour of newly introduced getFloatVectorValues method that returns FloatVectorValues.

Relates to #11963
This commit is contained in:
Luca Cavanna 2023-01-24 10:20:24 +01:00 committed by GitHub
parent d4006c0362
commit 92e67ec626
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
71 changed files with 734 additions and 544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -168,6 +168,9 @@ API Changes
accessed through their high-level representation, via
VectorValues#vectorValue(). (Adrien Grand)
* GITHUB#12105: Deprecate KnnVectorField in favour of KnnFloatVectorField, KnnVectoryQuery in favour of
KnnFloatVectorQuery, and LeafReader#getVectorValues in favour of LeafReader#getFloatVectorValues
New Features
---------------------
* GITHUB#11795: Add ByteWritesTrackingDirectoryWrapper to expose metrics for bytes merged, flushed, and overall

17
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
View File

@ -30,10 +30,10 @@ import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
@ -226,7 +226,7 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
FieldEntry fieldEntry = fields.get(field);
return getOffHeapVectorValues(fieldEntry);
}
@ -248,7 +248,7 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
// bound k by total number of vectors to prevent oversizing data structures
k = Math.min(k, fieldEntry.size());
OffHeapVectorValues vectorValues = getOffHeapVectorValues(fieldEntry);
OffHeapFloatVectorValues vectorValues = getOffHeapVectorValues(fieldEntry);
// use a seed that is fixed for the index so we get reproducible results for the same query
final SplittableRandom random = new SplittableRandom(checksumSeed);
NeighborQueue results =
@ -283,10 +283,11 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
throw new UnsupportedOperationException();
}
private OffHeapVectorValues getOffHeapVectorValues(FieldEntry fieldEntry) throws IOException {
private OffHeapFloatVectorValues getOffHeapVectorValues(FieldEntry fieldEntry)
throws IOException {
IndexInput bytesSlice =
vectorData.slice("vector-data", fieldEntry.vectorDataOffset, fieldEntry.vectorDataLength);
return new OffHeapVectorValues(fieldEntry.dimension, fieldEntry.ordToDoc, bytesSlice);
return new OffHeapFloatVectorValues(fieldEntry.dimension, fieldEntry.ordToDoc, bytesSlice);
}
private Bits getAcceptOrds(Bits acceptDocs, FieldEntry fieldEntry) {
@ -356,7 +357,7 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
}
/** Read the vector values from the index input. This supports both iterated and random access. */
static class OffHeapVectorValues extends VectorValues
static class OffHeapFloatVectorValues extends FloatVectorValues
implements RandomAccessVectorValues<float[]> {
final int dimension;
@ -369,7 +370,7 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
int ord = -1;
int doc = -1;
OffHeapVectorValues(int dimension, int[] ordToDoc, IndexInput dataIn) {
OffHeapFloatVectorValues(int dimension, int[] ordToDoc, IndexInput dataIn) {
this.dimension = dimension;
this.ordToDoc = ordToDoc;
this.dataIn = dataIn;
@ -428,7 +429,7 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
@Override
public RandomAccessVectorValues<float[]> copy() {
return new OffHeapVectorValues(dimension, ordToDoc, dataIn.clone());
return new OffHeapFloatVectorValues(dimension, ordToDoc, dataIn.clone());
}
@Override

17
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
View File

@ -30,11 +30,11 @@ import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
@ -218,7 +218,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
FieldEntry fieldEntry = fields.get(field);
return getOffHeapVectorValues(fieldEntry);
}
@ -239,7 +239,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
// bound k by total number of vectors to prevent oversizing data structures
k = Math.min(k, fieldEntry.size());
OffHeapVectorValues vectorValues = getOffHeapVectorValues(fieldEntry);
OffHeapFloatVectorValues vectorValues = getOffHeapVectorValues(fieldEntry);
NeighborQueue results =
HnswGraphSearcher.search(
@ -274,10 +274,11 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
throw new UnsupportedOperationException();
}
private OffHeapVectorValues getOffHeapVectorValues(FieldEntry fieldEntry) throws IOException {
private OffHeapFloatVectorValues getOffHeapVectorValues(FieldEntry fieldEntry)
throws IOException {
IndexInput bytesSlice =
vectorData.slice("vector-data", fieldEntry.vectorDataOffset, fieldEntry.vectorDataLength);
return new OffHeapVectorValues(
return new OffHeapFloatVectorValues(
fieldEntry.dimension, fieldEntry.size(), fieldEntry.ordToDoc, bytesSlice);
}
@ -402,7 +403,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
}
/** Read the vector values from the index input. This supports both iterated and random access. */
static class OffHeapVectorValues extends VectorValues
static class OffHeapFloatVectorValues extends FloatVectorValues
implements RandomAccessVectorValues<float[]> {
private final int dimension;
@ -416,7 +417,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
private int ord = -1;
private int doc = -1;
OffHeapVectorValues(int dimension, int size, int[] ordToDoc, IndexInput dataIn) {
OffHeapFloatVectorValues(int dimension, int size, int[] ordToDoc, IndexInput dataIn) {
this.dimension = dimension;
this.size = size;
this.ordToDoc = ordToDoc;
@ -481,7 +482,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
@Override
public RandomAccessVectorValues<float[]> copy() {
return new OffHeapVectorValues(dimension, size, ordToDoc, dataIn.clone());
return new OffHeapFloatVectorValues(dimension, size, ordToDoc, dataIn.clone());
}
@Override

8
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java
View File

@ -29,11 +29,11 @@ import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
@ -214,9 +214,9 @@ public final class Lucene92HnswVectorsReader extends KnnVectorsReader {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
FieldEntry fieldEntry = fields.get(field);
return OffHeapVectorValues.load(fieldEntry, vectorData);
return OffHeapFloatVectorValues.load(fieldEntry, vectorData);
}
@Override
@ -235,7 +235,7 @@ public final class Lucene92HnswVectorsReader extends KnnVectorsReader {
// bound k by total number of vectors to prevent oversizing data structures
k = Math.min(k, fieldEntry.size());
OffHeapVectorValues vectorValues = OffHeapVectorValues.load(fieldEntry, vectorData);
OffHeapFloatVectorValues vectorValues = OffHeapFloatVectorValues.load(fieldEntry, vectorData);
NeighborQueue results =
HnswGraphSearcher.search(

14
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/OffHeapVectorValues.java → lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/OffHeapFloatVectorValues.java
View File

@ -19,7 +19,7 @@ package org.apache.lucene.backward_codecs.lucene92;
import java.io.IOException;
import org.apache.lucene.codecs.lucene90.IndexedDISI;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.Bits;
@ -27,7 +27,7 @@ import org.apache.lucene.util.hnsw.RandomAccessVectorValues;
import org.apache.lucene.util.packed.DirectMonotonicReader;
/** Read the vector values from the index input. This supports both iterated and random access. */
abstract class OffHeapVectorValues extends VectorValues
abstract class OffHeapFloatVectorValues extends FloatVectorValues
implements RandomAccessVectorValues<float[]> {
protected final int dimension;
@ -36,7 +36,7 @@ abstract class OffHeapVectorValues extends VectorValues
protected final int byteSize;
protected final float[] value;
OffHeapVectorValues(int dimension, int size, IndexInput slice) {
OffHeapFloatVectorValues(int dimension, int size, IndexInput slice) {
this.dimension = dimension;
this.size = size;
this.slice = slice;
@ -63,7 +63,7 @@ abstract class OffHeapVectorValues extends VectorValues
public abstract int ordToDoc(int ord);
static OffHeapVectorValues load(
static OffHeapFloatVectorValues load(
Lucene92HnswVectorsReader.FieldEntry fieldEntry, IndexInput vectorData) throws IOException {
if (fieldEntry.docsWithFieldOffset == -2) {
return new EmptyOffHeapVectorValues(fieldEntry.dimension);
@ -79,7 +79,7 @@ abstract class OffHeapVectorValues extends VectorValues
abstract Bits getAcceptOrds(Bits acceptDocs);
static class DenseOffHeapVectorValues extends OffHeapVectorValues {
static class DenseOffHeapVectorValues extends OffHeapFloatVectorValues {
private int doc = -1;
@ -129,7 +129,7 @@ abstract class OffHeapVectorValues extends VectorValues
}
}
private static class SparseOffHeapVectorValues extends OffHeapVectorValues {
private static class SparseOffHeapVectorValues extends OffHeapFloatVectorValues {
private final DirectMonotonicReader ordToDoc;
private final IndexedDISI disi;
// dataIn was used to init a new IndexedDIS for #randomAccess()
@ -208,7 +208,7 @@ abstract class OffHeapVectorValues extends VectorValues
}
}
private static class EmptyOffHeapVectorValues extends OffHeapVectorValues {
private static class EmptyOffHeapVectorValues extends OffHeapFloatVectorValues {
public EmptyOffHeapVectorValues(int dimension) {
super(dimension, 0, null);

8
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
View File

@ -29,11 +29,11 @@ import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
@ -231,7 +231,7 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
FieldEntry fieldEntry = fields.get(field);
if (fieldEntry.vectorEncoding != VectorEncoding.FLOAT32) {
throw new IllegalArgumentException(
@ -242,7 +242,7 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
+ " expected: "
+ VectorEncoding.FLOAT32);
}
return OffHeapVectorValues.load(fieldEntry, vectorData);
return OffHeapFloatVectorValues.load(fieldEntry, vectorData);
}
@Override
@ -271,7 +271,7 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
// bound k by total number of vectors to prevent oversizing data structures
k = Math.min(k, fieldEntry.size());
OffHeapVectorValues vectorValues = OffHeapVectorValues.load(fieldEntry, vectorData);
OffHeapFloatVectorValues vectorValues = OffHeapFloatVectorValues.load(fieldEntry, vectorData);
NeighborQueue results =
HnswGraphSearcher.search(

14
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/OffHeapVectorValues.java → lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/OffHeapFloatVectorValues.java
View File

@ -19,7 +19,7 @@ package org.apache.lucene.backward_codecs.lucene94;
import java.io.IOException;
import org.apache.lucene.codecs.lucene90.IndexedDISI;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.Bits;
@ -27,7 +27,7 @@ import org.apache.lucene.util.hnsw.RandomAccessVectorValues;
import org.apache.lucene.util.packed.DirectMonotonicReader;
/** Read the vector values from the index input. This supports both iterated and random access. */
abstract class OffHeapVectorValues extends VectorValues
abstract class OffHeapFloatVectorValues extends FloatVectorValues
implements RandomAccessVectorValues<float[]> {
protected final int dimension;
@ -36,7 +36,7 @@ abstract class OffHeapVectorValues extends VectorValues
protected final int byteSize;
protected final float[] value;
OffHeapVectorValues(int dimension, int size, IndexInput slice, int byteSize) {
OffHeapFloatVectorValues(int dimension, int size, IndexInput slice, int byteSize) {
this.dimension = dimension;
this.size = size;
this.slice = slice;
@ -63,7 +63,7 @@ abstract class OffHeapVectorValues extends VectorValues
public abstract int ordToDoc(int ord);
static OffHeapVectorValues load(
static OffHeapFloatVectorValues load(
Lucene94HnswVectorsReader.FieldEntry fieldEntry, IndexInput vectorData) throws IOException {
if (fieldEntry.docsWithFieldOffset == -2) {
return new EmptyOffHeapVectorValues(fieldEntry.dimension);
@ -85,7 +85,7 @@ abstract class OffHeapVectorValues extends VectorValues
abstract Bits getAcceptOrds(Bits acceptDocs);
static class DenseOffHeapVectorValues extends OffHeapVectorValues {
static class DenseOffHeapVectorValues extends OffHeapFloatVectorValues {
private int doc = -1;
@ -135,7 +135,7 @@ abstract class OffHeapVectorValues extends VectorValues
}
}
private static class SparseOffHeapVectorValues extends OffHeapVectorValues {
private static class SparseOffHeapVectorValues extends OffHeapFloatVectorValues {
private final DirectMonotonicReader ordToDoc;
private final IndexedDISI disi;
// dataIn was used to init a new IndexedDIS for #randomAccess()
@ -217,7 +217,7 @@ abstract class OffHeapVectorValues extends VectorValues
}
}
private static class EmptyOffHeapVectorValues extends OffHeapVectorValues {
private static class EmptyOffHeapVectorValues extends OffHeapFloatVectorValues {
public EmptyOffHeapVectorValues(int dimension) {
super(dimension, 0, null, 0);

12
lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsWriter.java
View File

@ -27,10 +27,10 @@ import org.apache.lucene.codecs.BufferingKnnVectorsWriter;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
@ -110,7 +110,7 @@ public final class Lucene90HnswVectorsWriter extends BufferingKnnVectorsWriter {
public void writeField(FieldInfo fieldInfo, KnnVectorsReader knnVectorsReader, int maxDoc)
throws IOException {
long vectorDataOffset = vectorData.alignFilePointer(Float.BYTES);
VectorValues vectors = knnVectorsReader.getVectorValues(fieldInfo.name);
FloatVectorValues vectors = knnVectorsReader.getFloatVectorValues(fieldInfo.name);
IndexOutput tempVectorData =
segmentWriteState.directory.createTempOutput(
@ -132,8 +132,8 @@ public final class Lucene90HnswVectorsWriter extends BufferingKnnVectorsWriter {
CodecUtil.retrieveChecksum(vectorDataInput);
// build the graph using the temporary vector data
Lucene90HnswVectorsReader.OffHeapVectorValues offHeapVectors =
new Lucene90HnswVectorsReader.OffHeapVectorValues(
Lucene90HnswVectorsReader.OffHeapFloatVectorValues offHeapVectors =
new Lucene90HnswVectorsReader.OffHeapFloatVectorValues(
vectors.dimension(), docIds, vectorDataInput);
long[] offsets = new long[docIds.length];
@ -173,9 +173,9 @@ public final class Lucene90HnswVectorsWriter extends BufferingKnnVectorsWriter {
/**
* Writes the vector values to the output and returns a mapping from dense ordinals to document
* IDs. The length of the returned array matches the total number of documents with a vector
* (which excludes deleted documents), so it may be less than {@link VectorValues#size()}.
* (which excludes deleted documents), so it may be less than {@link FloatVectorValues#size()}.
*/
private static int[] writeVectorData(IndexOutput output, VectorValues vectors)
private static int[] writeVectorData(IndexOutput output, FloatVectorValues vectors)
throws IOException {
int[] docIds = new int[vectors.size()];
int count = 0;

10
lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsWriter.java
View File

@ -28,10 +28,10 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.index.DocsWithFieldSet;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@ -112,7 +112,7 @@ public final class Lucene91HnswVectorsWriter extends BufferingKnnVectorsWriter {
public void writeField(FieldInfo fieldInfo, KnnVectorsReader knnVectorsReader, int maxDoc)
throws IOException {
long vectorDataOffset = vectorData.alignFilePointer(Float.BYTES);
VectorValues vectors = knnVectorsReader.getVectorValues(fieldInfo.name);
FloatVectorValues vectors = knnVectorsReader.getFloatVectorValues(fieldInfo.name);
IndexOutput tempVectorData =
segmentWriteState.directory.createTempOutput(
@ -137,8 +137,8 @@ public final class Lucene91HnswVectorsWriter extends BufferingKnnVectorsWriter {
// build the graph using the temporary vector data
// we pass null for ordToDoc mapping, for the graph construction doesn't need to know docIds
// TODO: separate random access vector values from DocIdSetIterator?
Lucene91HnswVectorsReader.OffHeapVectorValues offHeapVectors =
new Lucene91HnswVectorsReader.OffHeapVectorValues(
Lucene91HnswVectorsReader.OffHeapFloatVectorValues offHeapVectors =
new Lucene91HnswVectorsReader.OffHeapFloatVectorValues(
vectors.dimension(), docsWithField.cardinality(), null, vectorDataInput);
Lucene91OnHeapHnswGraph graph =
offHeapVectors.size() == 0
@ -170,7 +170,7 @@ public final class Lucene91HnswVectorsWriter extends BufferingKnnVectorsWriter {
/**
* Writes the vector values to the output and returns a set of documents that contains vectors.
*/
private static DocsWithFieldSet writeVectorData(IndexOutput output, VectorValues vectors)
private static DocsWithFieldSet writeVectorData(IndexOutput output, FloatVectorValues vectors)
throws IOException {
DocsWithFieldSet docsWithField = new DocsWithFieldSet();
ByteBuffer binaryVector =

10
lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsWriter.java
View File

@ -30,11 +30,11 @@ import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.lucene90.IndexedDISI;
import org.apache.lucene.index.DocsWithFieldSet;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@ -118,7 +118,7 @@ public final class Lucene92HnswVectorsWriter extends BufferingKnnVectorsWriter {
public void writeField(FieldInfo fieldInfo, KnnVectorsReader knnVectorsReader, int maxDoc)
throws IOException {
long vectorDataOffset = vectorData.alignFilePointer(Float.BYTES);
VectorValues vectors = knnVectorsReader.getVectorValues(fieldInfo.name);
FloatVectorValues vectors = knnVectorsReader.getFloatVectorValues(fieldInfo.name);
IndexOutput tempVectorData =
segmentWriteState.directory.createTempOutput(
@ -144,8 +144,8 @@ public final class Lucene92HnswVectorsWriter extends BufferingKnnVectorsWriter {
// we use Lucene92HnswVectorsReader.DenseOffHeapVectorValues for the graph construction
// doesn't need to know docIds
// TODO: separate random access vector values from DocIdSetIterator?
OffHeapVectorValues offHeapVectors =
new OffHeapVectorValues.DenseOffHeapVectorValues(
OffHeapFloatVectorValues offHeapVectors =
new OffHeapFloatVectorValues.DenseOffHeapVectorValues(
vectors.dimension(), docsWithField.cardinality(), vectorDataInput);
OnHeapHnswGraph graph =
offHeapVectors.size() == 0
@ -178,7 +178,7 @@ public final class Lucene92HnswVectorsWriter extends BufferingKnnVectorsWriter {
/**
* Writes the vector values to the output and returns a set of documents that contains vectors.
*/
private static DocsWithFieldSet writeVectorData(IndexOutput output, VectorValues vectors)
private static DocsWithFieldSet writeVectorData(IndexOutput output, FloatVectorValues vectors)
throws IOException {
DocsWithFieldSet docsWithField = new DocsWithFieldSet();
ByteBuffer binaryVector =

8
lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java
View File

@ -33,12 +33,12 @@ import org.apache.lucene.codecs.lucene90.IndexedDISI;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DocsWithFieldSet;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Sorter;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@ -432,8 +432,8 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
yield hnswGraphBuilder.build(vectorValues.copy());
}
case FLOAT32 -> {
OffHeapVectorValues.DenseOffHeapVectorValues vectorValues =
new OffHeapVectorValues.DenseOffHeapVectorValues(
OffHeapFloatVectorValues.DenseOffHeapVectorValues vectorValues =
new OffHeapFloatVectorValues.DenseOffHeapVectorValues(
fieldInfo.getVectorDimension(),
docsWithField.cardinality(),
vectorDataInput,
@ -605,7 +605,7 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
* Writes the vector values to the output and returns a set of documents that contains vectors.
*/
private static DocsWithFieldSet writeVectorData(
IndexOutput output, VectorValues floatVectorValues) throws IOException {
IndexOutput output, FloatVectorValues floatVectorValues) throws IOException {
DocsWithFieldSet docsWithField = new DocsWithFieldSet();
ByteBuffer binaryVector =
ByteBuffer.allocate(floatVectorValues.dimension() * VectorEncoding.FLOAT32.byteSize)

14
lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
View File

@ -53,7 +53,7 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FloatDocValuesField;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
@ -65,6 +65,7 @@ import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.index.IndexOptions;
@ -97,12 +98,11 @@ import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.FieldExistsQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.KnnVectorQuery;
import org.apache.lucene.search.KnnFloatVectorQuery;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
@ -167,7 +167,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
private static final int KNN_VECTOR_MIN_SUPPORTED_VERSION = LUCENE_9_0_0.major;
private static final String KNN_VECTOR_FIELD = "knn_field";
private static final FieldType KNN_VECTOR_FIELD_TYPE =
KnnVectorField.createFieldType(3, VectorSimilarityFunction.COSINE);
KnnFloatVectorField.createFieldType(3, VectorSimilarityFunction.COSINE);
private static final float[] KNN_VECTOR = {0.2f, -0.1f, 0.1f};
public void testCreateCFS() throws IOException {
@ -1327,7 +1327,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
// test vector values
int cnt = 0;
for (LeafReaderContext ctx : reader.leaves()) {
VectorValues values = ctx.reader().getVectorValues(KNN_VECTOR_FIELD);
FloatVectorValues values = ctx.reader().getFloatVectorValues(KNN_VECTOR_FIELD);
if (values != null) {
assertEquals(KNN_VECTOR_FIELD_TYPE.vectorDimension(), values.dimension());
for (int doc = values.nextDoc(); doc != NO_MORE_DOCS; doc = values.nextDoc()) {
@ -1360,7 +1360,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
String expectedFirstDocId)
throws IOException {
ScoreDoc[] hits =
searcher.search(new KnnVectorQuery(KNN_VECTOR_FIELD, queryVector, k), k).scoreDocs;
searcher.search(new KnnFloatVectorQuery(KNN_VECTOR_FIELD, queryVector, k), k).scoreDocs;
assertEquals("wrong number of hits", expectedHitsCount, hits.length);
Document d = searcher.storedFields().document(hits[0].doc);
assertEquals("wrong first document", expectedFirstDocId, d.get("id"));
@ -1598,7 +1598,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
doc.add(new Field("content6", "here is more content with aaa aaa aaa", customType4));
float[] vector = {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * id};
doc.add(new KnnVectorField(KNN_VECTOR_FIELD, vector, KNN_VECTOR_FIELD_TYPE));
doc.add(new KnnFloatVectorField(KNN_VECTOR_FIELD, vector, KNN_VECTOR_FIELD_TYPE));
// TODO:
// index different norms types via similarity (we use a random one currently?!)

12
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
View File

@ -32,10 +32,10 @@ import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.HitQueue;
import org.apache.lucene.search.ScoreDoc;
@ -115,7 +115,7 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
FieldInfo info = readState.fieldInfos.fieldInfo(field);
if (info == null) {
// mirror the handling in Lucene90VectorReader#getVectorValues
@ -144,7 +144,7 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
}
IndexInput bytesSlice =
dataIn.slice("vector-data", fieldEntry.vectorDataOffset, fieldEntry.vectorDataLength);
return new SimpleTextVectorValues(fieldEntry, bytesSlice);
return new SimpleTextFloatVectorValues(fieldEntry, bytesSlice);
}
@Override
@ -183,7 +183,7 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
@Override
public TopDocs search(String field, float[] target, int k, Bits acceptDocs, int visitedLimit)
throws IOException {
VectorValues values = getVectorValues(field);
FloatVectorValues values = getFloatVectorValues(field);
if (target.length != values.dimension()) {
throw new IllegalArgumentException(
"vector query dimension: "
@ -337,7 +337,7 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
}
}
private static class SimpleTextVectorValues extends VectorValues
private static class SimpleTextFloatVectorValues extends FloatVectorValues
implements RandomAccessVectorValues<float[]> {
private final BytesRefBuilder scratch = new BytesRefBuilder();
@ -347,7 +347,7 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
int curOrd;
SimpleTextVectorValues(FieldEntry entry, IndexInput in) throws IOException {
SimpleTextFloatVectorValues(FieldEntry entry, IndexInput in) throws IOException {
this.entry = entry;
this.in = in;
values = new float[entry.size()][entry.dimension];

6
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsWriter.java
View File

@ -26,9 +26,9 @@ import java.util.List;
import org.apache.lucene.codecs.BufferingKnnVectorsWriter;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@ -75,7 +75,7 @@ public class SimpleTextKnnVectorsWriter extends BufferingKnnVectorsWriter {
@Override
public void writeField(FieldInfo fieldInfo, KnnVectorsReader knnVectorsReader, int maxDoc)
throws IOException {
VectorValues vectors = knnVectorsReader.getVectorValues(fieldInfo.name);
FloatVectorValues vectors = knnVectorsReader.getFloatVectorValues(fieldInfo.name);
long vectorDataOffset = vectorData.getFilePointer();
List<Integer> docIds = new ArrayList<>();
int docV;
@ -87,7 +87,7 @@ public class SimpleTextKnnVectorsWriter extends BufferingKnnVectorsWriter {
writeMeta(fieldInfo, vectorDataOffset, vectorDataLength, docIds);
}
private void writeVectorValue(VectorValues vectors) throws IOException {
private void writeVectorValue(FloatVectorValues vectors) throws IOException {
// write vector value
float[] value = vectors.vectorValue();
assert value.length == vectors.dimension();

10
lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java
View File

@ -23,9 +23,9 @@ import java.util.List;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DocsWithFieldSet;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.Sorter;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.ArrayUtil;
@ -72,7 +72,7 @@ public abstract class BufferingKnnVectorsWriter extends KnnVectorsWriter {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
BufferedVectorValues vectorValues =
new BufferedVectorValues(
fieldData.docsWithField,
@ -106,7 +106,7 @@ public abstract class BufferingKnnVectorsWriter extends KnnVectorsWriter {
}
/** Sorting VectorValues that iterate over documents in the order of the provided sortMap */
private static class SortingVectorValues extends VectorValues {
private static class SortingVectorValues extends FloatVectorValues {
private final BufferedVectorValues randomAccess;
private final int[] docIdOffsets;
private int docId = -1;
@ -196,7 +196,7 @@ public abstract class BufferingKnnVectorsWriter extends KnnVectorsWriter {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
return MergedVectorValues.mergeVectorValues(fieldInfo, mergeState);
}
@ -261,7 +261,7 @@ public abstract class BufferingKnnVectorsWriter extends KnnVectorsWriter {
}
}
private static class BufferedVectorValues extends VectorValues {
private static class BufferedVectorValues extends FloatVectorValues {
final DocsWithFieldSet docsWithField;

4
lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java
View File

@ -19,9 +19,9 @@ package org.apache.lucene.codecs;
import java.io.IOException;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.NamedSPILoader;
@ -94,7 +94,7 @@ public abstract class KnnVectorsFormat implements NamedSPILoader.NamedSPI {
public void checkIntegrity() {}
@Override
public VectorValues getVectorValues(String field) {
public FloatVectorValues getFloatVectorValues(String field) {
throw new UnsupportedOperationException();
}

8
lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java
View File

@ -21,7 +21,7 @@ import java.io.Closeable;
import java.io.IOException;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
@ -45,11 +45,11 @@ public abstract class KnnVectorsReader implements Closeable, Accountable {
public abstract void checkIntegrity() throws IOException;
/**
* Returns the {@link VectorValues} for the given {@code field}. The behavior is undefined if the
* given field doesn't have KNN vectors enabled on its {@link FieldInfo}. The return value is
* Returns the {@link FloatVectorValues} for the given {@code field}. The behavior is undefined if
* the given field doesn't have KNN vectors enabled on its {@link FieldInfo}. The return value is
* never {@code null}.
*/
public abstract VectorValues getVectorValues(String field) throws IOException;
public abstract FloatVectorValues getFloatVectorValues(String field) throws IOException;
/**
* Returns the {@link ByteVectorValues} for the given {@code field}. The behavior is undefined if

17
lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java
View File

@ -24,10 +24,10 @@ import java.util.List;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DocIDMerger;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.Sorter;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Accountable;
@ -61,7 +61,8 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
case FLOAT32:
KnnFieldVectorsWriter<float[]> floatWriter =
(KnnFieldVectorsWriter<float[]>) addField(fieldInfo);
VectorValues mergedFloats = MergedVectorValues.mergeVectorValues(fieldInfo, mergeState);
FloatVectorValues mergedFloats =
MergedVectorValues.mergeVectorValues(fieldInfo, mergeState);
for (int doc = mergedFloats.nextDoc();
doc != DocIdSetIterator.NO_MORE_DOCS;
doc = mergedFloats.nextDoc()) {
@ -107,9 +108,9 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
/** Tracks state of one sub-reader that we are merging */
private static class VectorValuesSub extends DocIDMerger.Sub {
final VectorValues values;
final FloatVectorValues values;
VectorValuesSub(MergeState.DocMap docMap, VectorValues values) {
VectorValuesSub(MergeState.DocMap docMap, FloatVectorValues values) {
super(docMap);
this.values = values;
assert values.docID() == -1;
@ -141,8 +142,8 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
protected static final class MergedVectorValues {
private MergedVectorValues() {}
/** Returns a merged view over all the segment's {@link VectorValues}. */
public static VectorValues mergeVectorValues(FieldInfo fieldInfo, MergeState mergeState)
/** Returns a merged view over all the segment's {@link FloatVectorValues}. */
public static FloatVectorValues mergeVectorValues(FieldInfo fieldInfo, MergeState mergeState)
throws IOException {
assert fieldInfo != null && fieldInfo.hasVectorValues();
if (fieldInfo.getVectorEncoding() != VectorEncoding.FLOAT32) {
@ -153,7 +154,7 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
for (int i = 0; i < mergeState.knnVectorsReaders.length; i++) {
KnnVectorsReader knnVectorsReader = mergeState.knnVectorsReaders[i];
if (knnVectorsReader != null) {
VectorValues values = knnVectorsReader.getVectorValues(fieldInfo.name);
FloatVectorValues values = knnVectorsReader.getFloatVectorValues(fieldInfo.name);
if (values != null) {
subs.add(new VectorValuesSub(mergeState.docMaps[i], values));
}
@ -183,7 +184,7 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
return new MergedByteVectorValues(subs, mergeState);
}
static class MergedFloat32VectorValues extends VectorValues {
static class MergedFloat32VectorValues extends FloatVectorValues {
private final List<VectorValuesSub> subs;
private final DocIDMerger<VectorValuesSub> docIdMerger;
private final int size;

8
lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsReader.java
View File

@ -29,11 +29,11 @@ import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
@ -236,7 +236,7 @@ public final class Lucene95HnswVectorsReader extends KnnVectorsReader {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
FieldEntry fieldEntry = fields.get(field);
if (fieldEntry.vectorEncoding != VectorEncoding.FLOAT32) {
throw new IllegalArgumentException(
@ -247,7 +247,7 @@ public final class Lucene95HnswVectorsReader extends KnnVectorsReader {
+ " expected: "
+ VectorEncoding.FLOAT32);
}
return OffHeapVectorValues.load(fieldEntry, vectorData);
return OffHeapFloatVectorValues.load(fieldEntry, vectorData);
}
@Override
@ -279,7 +279,7 @@ public final class Lucene95HnswVectorsReader extends KnnVectorsReader {
// bound k by total number of vectors to prevent oversizing data structures
k = Math.min(k, fieldEntry.size());
OffHeapVectorValues vectorValues = OffHeapVectorValues.load(fieldEntry, vectorData);
OffHeapFloatVectorValues vectorValues = OffHeapFloatVectorValues.load(fieldEntry, vectorData);
NeighborQueue results =
HnswGraphSearcher.search(

6
lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsWriter.java
View File

@ -445,8 +445,8 @@ public final class Lucene95HnswVectorsWriter extends KnnVectorsWriter {
yield hnswGraphBuilder.build(vectorValues.copy());
}
case FLOAT32 -> {
OffHeapVectorValues.DenseOffHeapVectorValues vectorValues =
new OffHeapVectorValues.DenseOffHeapVectorValues(
OffHeapFloatVectorValues.DenseOffHeapVectorValues vectorValues =
new OffHeapFloatVectorValues.DenseOffHeapVectorValues(
fieldInfo.getVectorDimension(),
docsWithField.cardinality(),
vectorDataInput,
@ -656,7 +656,7 @@ public final class Lucene95HnswVectorsWriter extends KnnVectorsWriter {
* Writes the vector values to the output and returns a set of documents that contains vectors.
*/
private static DocsWithFieldSet writeVectorData(
IndexOutput output, VectorValues floatVectorValues) throws IOException {
IndexOutput output, FloatVectorValues floatVectorValues) throws IOException {
DocsWithFieldSet docsWithField = new DocsWithFieldSet();
ByteBuffer buffer =
ByteBuffer.allocate(floatVectorValues.dimension() * VectorEncoding.FLOAT32.byteSize)

14
lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapVectorValues.java → lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java
View File

@ -19,8 +19,8 @@ package org.apache.lucene.codecs.lucene95;
import java.io.IOException;
import org.apache.lucene.codecs.lucene90.IndexedDISI;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.Bits;
@ -28,7 +28,7 @@ import org.apache.lucene.util.hnsw.RandomAccessVectorValues;
import org.apache.lucene.util.packed.DirectMonotonicReader;
/** Read the vector values from the index input. This supports both iterated and random access. */
abstract class OffHeapVectorValues extends VectorValues
abstract class OffHeapFloatVectorValues extends FloatVectorValues
implements RandomAccessVectorValues<float[]> {
protected final int dimension;
@ -37,7 +37,7 @@ abstract class OffHeapVectorValues extends VectorValues
protected final int byteSize;
protected final float[] value;
OffHeapVectorValues(int dimension, int size, IndexInput slice, int byteSize) {
OffHeapFloatVectorValues(int dimension, int size, IndexInput slice, int byteSize) {
this.dimension = dimension;
this.size = size;
this.slice = slice;
@ -64,7 +64,7 @@ abstract class OffHeapVectorValues extends VectorValues
public abstract int ordToDoc(int ord);
static OffHeapVectorValues load(
static OffHeapFloatVectorValues load(
Lucene95HnswVectorsReader.FieldEntry fieldEntry, IndexInput vectorData) throws IOException {
if (fieldEntry.docsWithFieldOffset == -2
|| fieldEntry.vectorEncoding != VectorEncoding.FLOAT32) {
@ -83,7 +83,7 @@ abstract class OffHeapVectorValues extends VectorValues
abstract Bits getAcceptOrds(Bits acceptDocs);
static class DenseOffHeapVectorValues extends OffHeapVectorValues {
static class DenseOffHeapVectorValues extends OffHeapFloatVectorValues {
private int doc = -1;
@ -133,7 +133,7 @@ abstract class OffHeapVectorValues extends VectorValues
}
}
private static class SparseOffHeapVectorValues extends OffHeapVectorValues {
private static class SparseOffHeapVectorValues extends OffHeapFloatVectorValues {
private final DirectMonotonicReader ordToDoc;
private final IndexedDISI disi;
// dataIn was used to init a new IndexedDIS for #randomAccess()
@ -215,7 +215,7 @@ abstract class OffHeapVectorValues extends VectorValues
}
}
private static class EmptyOffHeapVectorValues extends OffHeapVectorValues {
private static class EmptyOffHeapVectorValues extends OffHeapFloatVectorValues {
public EmptyOffHeapVectorValues(int dimension) {
super(dimension, 0, null, 0);

6
lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java
View File

@ -29,11 +29,11 @@ import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.KnnVectorsWriter;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Sorter;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
@ -246,12 +246,12 @@ public abstract class PerFieldKnnVectorsFormat extends KnnVectorsFormat {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
KnnVectorsReader knnVectorsReader = fields.get(field);
if (knnVectorsReader == null) {
return null;
} else {
return knnVectorsReader.getVectorValues(field);
return knnVectorsReader.getFloatVectorValues(field);
}
}

6
lucene/core/src/java/org/apache/lucene/document/FieldType.java
View File

@ -21,12 +21,12 @@ import java.util.Map;
import java.util.Objects;
import org.apache.lucene.analysis.Analyzer; // javadocs
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
/** Describes the properties of a field. */
public class FieldType implements IndexableFieldType {
@ -378,10 +378,10 @@ public class FieldType implements IndexableFieldType {
if (numDimensions <= 0) {
throw new IllegalArgumentException("vector numDimensions must be > 0; got " + numDimensions);
}
if (numDimensions > VectorValues.MAX_DIMENSIONS) {
if (numDimensions > FloatVectorValues.MAX_DIMENSIONS) {
throw new IllegalArgumentException(
"vector numDimensions must be <= VectorValues.MAX_DIMENSIONS (="
+ VectorValues.MAX_DIMENSIONS
+ FloatVectorValues.MAX_DIMENSIONS
+ "); got "
+ numDimensions);
}

163
lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java Normal file
View File

@ -0,0 +1,163 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.document;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.search.KnnFloatVectorQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.VectorUtil;
/**
* A field that contains a single floating-point numeric vector (or none) for each document. Vectors
* are dense - that is, every dimension of a vector contains an explicit value, stored packed into
* an array (of type float[]) whose length is the vector dimension. Values can be retrieved using
* {@link FloatVectorValues}, which is a forward-only docID-based iterator and also offers
* random-access by dense ordinal (not docId). {@link VectorSimilarityFunction} may be used to
* compare vectors at query time (for example as part of result ranking). A KnnVectorField may be
* associated with a search similarity function defining the metric used for nearest-neighbor search
* among vectors of that field.
*
* @lucene.experimental
*/
public class KnnFloatVectorField extends Field {
private static FieldType createType(float[] v, VectorSimilarityFunction similarityFunction) {
if (v == null) {
throw new IllegalArgumentException("vector value must not be null");
}
int dimension = v.length;
if (dimension == 0) {
throw new IllegalArgumentException("cannot index an empty vector");
}
if (dimension > FloatVectorValues.MAX_DIMENSIONS) {
throw new IllegalArgumentException(
"cannot index vectors with dimension greater than " + FloatVectorValues.MAX_DIMENSIONS);
}
if (similarityFunction == null) {
throw new IllegalArgumentException("similarity function must not be null");
}
FieldType type = new FieldType();
type.setVectorAttributes(dimension, VectorEncoding.FLOAT32, similarityFunction);
type.freeze();
return type;
}
/**
* A convenience method for creating a vector field type.
*
* @param dimension dimension of vectors
* @param similarityFunction a function defining vector proximity.
* @throws IllegalArgumentException if any parameter is null, or has dimension &gt; 1024.
*/
public static FieldType createFieldType(
int dimension, VectorSimilarityFunction similarityFunction) {
FieldType type = new FieldType();
type.setVectorAttributes(dimension, VectorEncoding.FLOAT32, similarityFunction);
type.freeze();
return type;
}
/**
* Create a new vector query for the provided field targeting the float vector
*
* @param field The field to query
* @param queryVector The float vector target
* @param k The number of nearest neighbors to gather
* @return A new vector query
*/
public static Query newVectorQuery(String field, float[] queryVector, int k) {
return new KnnFloatVectorQuery(field, queryVector, k);
}
/**
* Creates a numeric vector field. Fields are single-valued: each document has either one value or
* no value. Vectors of a single field share the same dimension and similarity function. Note that
* some vector similarities (like {@link VectorSimilarityFunction#DOT_PRODUCT}) require values to
* be unit-length, which can be enforced using {@link VectorUtil#l2normalize(float[])}.
*
* @param name field name
* @param vector value
* @param similarityFunction a function defining vector proximity.
* @throws IllegalArgumentException if any parameter is null, or the vector is empty or has
* dimension &gt; 1024.
*/
public KnnFloatVectorField(
String name, float[] vector, VectorSimilarityFunction similarityFunction) {
super(name, createType(vector, similarityFunction));
fieldsData = vector;
}
/**
* Creates a numeric vector field with the default EUCLIDEAN_HNSW (L2) similarity. Fields are
* single-valued: each document has either one value or no value. Vectors of a single field share
* the same dimension and similarity function.
*
* @param name field name
* @param vector value
* @throws IllegalArgumentException if any parameter is null, or the vector is empty or has
* dimension &gt; 1024.
*/
public KnnFloatVectorField(String name, float[] vector) {
this(name, vector, VectorSimilarityFunction.EUCLIDEAN);
}
/**
* Creates a numeric vector field. Fields are single-valued: each document has either one value or
* no value. Vectors of a single field share the same dimension and similarity function.
*
* @param name field name
* @param vector value
* @param fieldType field type
* @throws IllegalArgumentException if any parameter is null, or the vector is empty or has
* dimension &gt; 1024.
*/
public KnnFloatVectorField(String name, float[] vector, FieldType fieldType) {
super(name, fieldType);
if (fieldType.vectorEncoding() != VectorEncoding.FLOAT32) {
throw new IllegalArgumentException(
"Attempt to create a vector for field "
+ name
+ " using float[] but the field encoding is "
+ fieldType.vectorEncoding());
}
fieldsData = vector;
}
/** Return the vector value of this field */
public float[] vectorValue() {
return (float[]) fieldsData;
}
/**
* Set the vector value of this field
*
* @param value the value to set; must not be null, and length must match the field type
*/
public void setVectorValue(float[] value) {
if (value == null) {
throw new IllegalArgumentException("value must not be null");
}
if (value.length != type.vectorDimension()) {
throw new IllegalArgumentException(
"value length " + value.length + " must match field dimension " + type.vectorDimension());
}
fieldsData = value;
}
}

105
lucene/core/src/java/org/apache/lucene/document/KnnVectorField.java
View File

@ -14,77 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.document;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.KnnVectorQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.VectorUtil;
/**
* A field that contains a single floating-point numeric vector (or none) for each document. Vectors
* are dense - that is, every dimension of a vector contains an explicit value, stored packed into
* an array (of type float[]) whose length is the vector dimension. Values can be retrieved using
* {@link VectorValues}, which is a forward-only docID-based iterator and also offers random-access
* by dense ordinal (not docId). {@link VectorSimilarityFunction} may be used to compare vectors at
* query time (for example as part of result ranking). A KnnVectorField may be associated with a
* search similarity function defining the metric used for nearest-neighbor search among vectors of
* that field.
* {@link FloatVectorValues}, which is a forward-only docID-based iterator and also offers
* random-access by dense ordinal (not docId). {@link VectorSimilarityFunction} may be used to
* compare vectors at query time (for example as part of result ranking). A KnnVectorField may be
* associated with a search similarity function defining the metric used for nearest-neighbor search
* among vectors of that field.
*
* @lucene.experimental
* @deprecated use {@link KnnFloatVectorField} instead
*/
public class KnnVectorField extends Field {
private static FieldType createType(float[] v, VectorSimilarityFunction similarityFunction) {
if (v == null) {
throw new IllegalArgumentException("vector value must not be null");
}
int dimension = v.length;
if (dimension == 0) {
throw new IllegalArgumentException("cannot index an empty vector");
}
if (dimension > VectorValues.MAX_DIMENSIONS) {
throw new IllegalArgumentException(
"cannot index vectors with dimension greater than " + VectorValues.MAX_DIMENSIONS);
}
if (similarityFunction == null) {
throw new IllegalArgumentException("similarity function must not be null");
}
FieldType type = new FieldType();
type.setVectorAttributes(dimension, VectorEncoding.FLOAT32, similarityFunction);
type.freeze();
return type;
}
/**
* A convenience method for creating a vector field type.
*
* @param dimension dimension of vectors
* @param similarityFunction a function defining vector proximity.
* @throws IllegalArgumentException if any parameter is null, or has dimension &gt; 1024.
*/
public static FieldType createFieldType(
int dimension, VectorSimilarityFunction similarityFunction) {
FieldType type = new FieldType();
type.setVectorAttributes(dimension, VectorEncoding.FLOAT32, similarityFunction);
type.freeze();
return type;
}
/**
* Create a new vector query for the provided field targeting the float vector
*
* @param field The field to query
* @param queryVector The float vector target
* @param k The number of nearest neighbors to gather
* @return A new vector query
*/
public static Query newVectorQuery(String field, float[] queryVector, int k) {
return new KnnVectorQuery(field, queryVector, k);
}
@Deprecated
public class KnnVectorField extends KnnFloatVectorField {
/**
* Creates a numeric vector field. Fields are single-valued: each document has either one value or
@ -99,8 +48,7 @@ public class KnnVectorField extends Field {
* dimension &gt; 1024.
*/
public KnnVectorField(String name, float[] vector, VectorSimilarityFunction similarityFunction) {
super(name, createType(vector, similarityFunction));
fieldsData = vector;
super(name, vector, similarityFunction);
}
/**
@ -114,7 +62,7 @@ public class KnnVectorField extends Field {
* dimension &gt; 1024.
*/
public KnnVectorField(String name, float[] vector) {
this(name, vector, VectorSimilarityFunction.EUCLIDEAN);
super(name, vector);
}
/**
@ -128,35 +76,6 @@ public class KnnVectorField extends Field {
* dimension &gt; 1024.
*/
public KnnVectorField(String name, float[] vector, FieldType fieldType) {
super(name, fieldType);
if (fieldType.vectorEncoding() != VectorEncoding.FLOAT32) {
throw new IllegalArgumentException(
"Attempt to create a vector for field "
+ name
+ " using float[] but the field encoding is "
+ fieldType.vectorEncoding());
}
fieldsData = vector;
}
/** Return the vector value of this field */
public float[] vectorValue() {
return (float[]) fieldsData;
}
/**
* Set the vector value of this field
*
* @param value the value to set; must not be null, and length must match the field type
*/
public void setVectorValue(float[] value) {
if (value == null) {
throw new IllegalArgumentException("value must not be null");
}
if (value.length != type.vectorDimension()) {
throw new IllegalArgumentException(
"value length " + value.length + " must match field dimension " + type.vectorDimension());
}
fieldsData = value;
super(name, vector, fieldType);
}
}

6
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -2589,7 +2589,7 @@ public final class CheckIndex implements Closeable {
+ "\" has vector values but dimension is "
+ dimension);
}
if (reader.getVectorValues(fieldInfo.name) == null
if (reader.getFloatVectorValues(fieldInfo.name) == null
&& reader.getByteVectorValues(fieldInfo.name) == null) {
continue;
}
@ -2605,7 +2605,7 @@ public final class CheckIndex implements Closeable {
break;
case FLOAT32:
checkFloatVectorValues(
Objects.requireNonNull(reader.getVectorValues(fieldInfo.name)),
Objects.requireNonNull(reader.getFloatVectorValues(fieldInfo.name)),
fieldInfo,
status,
reader);
@ -2644,7 +2644,7 @@ public final class CheckIndex implements Closeable {
}
private static void checkFloatVectorValues(
VectorValues values,
FloatVectorValues values,
FieldInfo fieldInfo,
CheckIndex.Status.VectorValuesStatus status,
CodecReader codecReader)

4
lucene/core/src/java/org/apache/lucene/index/CodecReader.java
View File

@ -214,7 +214,7 @@ public abstract class CodecReader extends LeafReader {
}
@Override
public final VectorValues getVectorValues(String field) throws IOException {
public final FloatVectorValues getFloatVectorValues(String field) throws IOException {
ensureOpen();
FieldInfo fi = getFieldInfos().fieldInfo(field);
if (fi == null
@ -224,7 +224,7 @@ public abstract class CodecReader extends LeafReader {
return null;
}
return getVectorReader().getVectorValues(field);
return getVectorReader().getFloatVectorValues(field);
}
@Override

2
lucene/core/src/java/org/apache/lucene/index/DocValuesLeafReader.java
View File

@ -48,7 +48,7 @@ abstract class DocValuesLeafReader extends LeafReader {
}
@Override
public final VectorValues getVectorValues(String field) throws IOException {
public final FloatVectorValues getFloatVectorValues(String field) throws IOException {
throw new UnsupportedOperationException();
}

12
lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java
View File

@ -315,12 +315,12 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
final VectorValues vectorValues = in.getVectorValues(field);
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
final FloatVectorValues vectorValues = in.getFloatVectorValues(field);
if (vectorValues == null) {
return null;
}
return new ExitableVectorValues(vectorValues);
return new ExitableFloatVectorValues(vectorValues);
}
@Override
@ -396,11 +396,11 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {
}
}
private class ExitableVectorValues extends VectorValues {
private class ExitableFloatVectorValues extends FloatVectorValues {
private int docToCheck;
private final VectorValues vectorValues;
private final FloatVectorValues vectorValues;
public ExitableVectorValues(VectorValues vectorValues) {
public ExitableFloatVectorValues(FloatVectorValues vectorValues) {
this.vectorValues = vectorValues;
docToCheck = 0;
}

4
lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
View File

@ -347,8 +347,8 @@ public abstract class FilterLeafReader extends LeafReader {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
return in.getVectorValues(field);
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
return in.getFloatVectorValues(field);
}
@Override

6
lucene/core/src/java/org/apache/lucene/index/FilterVectorValues.java
View File

@ -20,14 +20,14 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Objects;
/** Delegates all methods to a wrapped {@link VectorValues}. */
/** Delegates all methods to a wrapped {@link FloatVectorValues}. */
public abstract class FilterVectorValues extends VectorValues {
/** Wrapped values */
protected final VectorValues in;
protected final FloatVectorValues in;
/** Sole constructor */
protected FilterVectorValues(VectorValues in) {
protected FilterVectorValues(FloatVectorValues in) {
Objects.requireNonNull(in);
this.in = in;
}

60
lucene/core/src/java/org/apache/lucene/index/FloatVectorValues.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.search.DocIdSetIterator;
/**
* This class provides access to per-document floating point vector values indexed as {@link
* KnnFloatVectorField}.
*
* @lucene.experimental
*/
public abstract class FloatVectorValues extends DocIdSetIterator {
/** The maximum length of a vector */
public static final int MAX_DIMENSIONS = 1024;
/** Sole constructor */
protected FloatVectorValues() {}
/** Return the dimension of the vectors */
public abstract int dimension();
/**
* Return the number of vectors for this field.
*
* @return the number of vectors returned by this iterator
*/
public abstract int size();
@Override
public final long cost() {
return size();
}
/**
* Return the vector value for the current document ID. It is illegal to call this method when the
* iterator is not positioned: before advancing, or after failing to advance. The returned array
* may be shared across calls, re-used, and modified as the iterator advances.
*
* @return the vector value
*/
public abstract float[] vectorValue() throws IOException;
}

4
lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
View File

@ -39,7 +39,7 @@ import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PointsWriter;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KnnByteVectorField;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
@ -964,7 +964,7 @@ final class IndexingChain implements Accountable {
case BYTE -> ((KnnFieldVectorsWriter<byte[]>) pf.knnFieldVectorsWriter)
.addValue(docID, ((KnnByteVectorField) field).vectorValue());
case FLOAT32 -> ((KnnFieldVectorsWriter<float[]>) pf.knnFieldVectorsWriter)
.addValue(docID, ((KnnVectorField) field).vectorValue());
.addValue(docID, ((KnnFloatVectorField) field).vectorValue());
}
}

13
lucene/core/src/java/org/apache/lucene/index/LeafReader.java
View File

@ -203,9 +203,20 @@ public abstract class LeafReader extends IndexReader {
* Returns {@link VectorValues} for this field, or null if no {@link VectorValues} were indexed.
* The returned instance should only be used by a single thread.
*
* @deprecated use {@link #getFloatVectorValues(String)} instead
*/
@Deprecated
public VectorValues getVectorValues(String field) throws IOException {
return new FilterVectorValues(getFloatVectorValues(field)) {};
}
/**
* Returns {@link FloatVectorValues} for this field, or null if no {@link FloatVectorValues} were
* indexed. The returned instance should only be used by a single thread.
*
* @lucene.experimental
*/
public abstract VectorValues getVectorValues(String field) throws IOException;
public abstract FloatVectorValues getFloatVectorValues(String field) throws IOException;
/**
* Returns {@link ByteVectorValues} for this field, or null if no {@link ByteVectorValues} were

4
lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
View File

@ -401,10 +401,10 @@ public class ParallelLeafReader extends LeafReader {
}
@Override
public VectorValues getVectorValues(String fieldName) throws IOException {
public FloatVectorValues getFloatVectorValues(String fieldName) throws IOException {
ensureOpen();
LeafReader reader = fieldToReader.get(fieldName);
return reader == null ? null : reader.getVectorValues(fieldName);
return reader == null ? null : reader.getFloatVectorValues(fieldName);
}
@Override

4
lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java
View File

@ -163,8 +163,8 @@ public final class SlowCodecReaderWrapper {
private static KnnVectorsReader readerToVectorReader(LeafReader reader) {
return new KnnVectorsReader() {
@Override
public VectorValues getVectorValues(String field) throws IOException {
return reader.getVectorValues(field);
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
return reader.getFloatVectorValues(field);
}
@Override

8
lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
View File

@ -213,7 +213,7 @@ public final class SortingCodecReader extends FilterCodecReader {
}
/** Sorting VectorValues that iterate over documents in the order of the provided sortMap */
private static class SortingVectorValues extends VectorValues {
private static class SortingFloatVectorValues extends FloatVectorValues {
final int size;
final int dimension;
final FixedBitSet docsWithField;
@ -221,7 +221,7 @@ public final class SortingCodecReader extends FilterCodecReader {
private int docId = -1;
SortingVectorValues(VectorValues delegate, Sorter.DocMap sortMap) throws IOException {
SortingFloatVectorValues(FloatVectorValues delegate, Sorter.DocMap sortMap) throws IOException {
this.size = delegate.size();
this.dimension = delegate.dimension();
docsWithField = new FixedBitSet(sortMap.size());
@ -488,8 +488,8 @@ public final class SortingCodecReader extends FilterCodecReader {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
return new SortingVectorValues(delegate.getVectorValues(field), docMap);
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
return new SortingFloatVectorValues(delegate.getFloatVectorValues(field), docMap);
}
@Override

42
lucene/core/src/java/org/apache/lucene/index/VectorValues.java
View File

@ -16,45 +16,17 @@
*/
package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.document.KnnFloatVectorField;
/**
* This class provides access to per-document floating point vector values indexed as {@link
* KnnVectorField}.
* KnnFloatVectorField}.
*
* @lucene.experimental
* @deprecated use {@link FloatVectorValues} instead
*/
public abstract class VectorValues extends DocIdSetIterator {
@Deprecated
public abstract class VectorValues extends FloatVectorValues {
/** The maximum length of a vector */
public static final int MAX_DIMENSIONS = 1024;
/** Sole constructor */
protected VectorValues() {}
/** Return the dimension of the vectors */
public abstract int dimension();
/**
* Return the number of vectors for this field.
*
* @return the number of vectors returned by this iterator
*/
public abstract int size();
@Override
public final long cost() {
return size();
}
/**
* Return the vector value for the current document ID. It is illegal to call this method when the
* iterator is not positioned: before advancing, or after failing to advance. The returned array
* may be shared across calls, re-used, and modified as the iterator advances.
*
* @return the vector value
*/
public abstract float[] vectorValue() throws IOException;
/** Creates an instance to hold floating point vector values for a single document */
VectorValues() {}
}

10
lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java
View File

@ -18,6 +18,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Objects;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
@ -30,9 +31,8 @@ import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.Terms;
/**
* A {@link Query} that matches documents that contain either a {@link
* org.apache.lucene.document.KnnVectorField}, {@link org.apache.lucene.document.KnnByteVectorField}
* or a field that indexes norms or doc values.
* A {@link Query} that matches documents that contain either a {@link KnnFloatVectorField}, {@link
* org.apache.lucene.document.KnnByteVectorField} or a field that indexes norms or doc values.
*/
public class FieldExistsQuery extends Query {
private String field;
@ -130,7 +130,7 @@ public class FieldExistsQuery extends Query {
} else if (fieldInfo.getVectorDimension() != 0) { // the field indexes vectors
int numVectors =
switch (fieldInfo.getVectorEncoding()) {
case FLOAT32 -> leaf.getVectorValues(field).size();
case FLOAT32 -> leaf.getFloatVectorValues(field).size();
case BYTE -> leaf.getByteVectorValues(field).size();
};
if (numVectors != leaf.maxDoc()) {
@ -183,7 +183,7 @@ public class FieldExistsQuery extends Query {
} else if (fieldInfo.getVectorDimension() != 0) { // the field indexes vectors
iterator =
switch (fieldInfo.getVectorEncoding()) {
case FLOAT32 -> context.reader().getVectorValues(field);
case FLOAT32 -> context.reader().getFloatVectorValues(field);
case BYTE -> context.reader().getByteVectorValues(field);
};
} else if (fieldInfo.getDocValuesType()

6
lucene/core/src/java/org/apache/lucene/search/KnnByteVectorQuery.java
View File

@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.VectorEncoding;
@ -50,7 +50,7 @@ public class KnnByteVectorQuery extends AbstractKnnVectorQuery {
* Find the <code>k</code> nearest documents to the target vector according to the vectors in the
* given field. <code>target</code> vector.
*
* @param field a field that has been indexed as a {@link KnnVectorField}.
* @param field a field that has been indexed as a {@link KnnFloatVectorField}.
* @param target the target of the search
* @param k the number of documents to find
* @throws IllegalArgumentException if <code>k</code> is less than 1
@ -63,7 +63,7 @@ public class KnnByteVectorQuery extends AbstractKnnVectorQuery {
* Find the <code>k</code> nearest documents to the target vector according to the vectors in the
* given field. <code>target</code> vector.
*
* @param field a field that has been indexed as a {@link KnnVectorField}.
* @param field a field that has been indexed as a {@link KnnFloatVectorField}.
* @param target the target of the search
* @param k the number of documents to find
* @param filter a filter applied before the vector search

118
lucene/core/src/java/org/apache/lucene/search/KnnFloatVectorQuery.java Normal file
View File

@ -0,0 +1,118 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.search;
import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
/**
* Uses {@link KnnVectorsReader#search(String, float[], int, Bits, int)} to perform nearest
* neighbour search.
*
* <p>This query also allows for performing a kNN search subject to a filter. In this case, it first
* executes the filter for each leaf, then chooses a strategy dynamically:
*
* <ul>
* <li>If the filter cost is less than k, just execute an exact search
* <li>Otherwise run a kNN search subject to the filter
* <li>If the kNN search visits too many vectors without completing, stop and run an exact search
* </ul>
*/
public class KnnFloatVectorQuery extends AbstractKnnVectorQuery {
private static final TopDocs NO_RESULTS = TopDocsCollector.EMPTY_TOPDOCS;
private final float[] target;
/**
* Find the <code>k</code> nearest documents to the target vector according to the vectors in the
* given field. <code>target</code> vector.
*
* @param field a field that has been indexed as a {@link KnnFloatVectorField}.
* @param target the target of the search
* @param k the number of documents to find
* @throws IllegalArgumentException if <code>k</code> is less than 1
*/
public KnnFloatVectorQuery(String field, float[] target, int k) {
this(field, target, k, null);
}
/**
* Find the <code>k</code> nearest documents to the target vector according to the vectors in the
* given field. <code>target</code> vector.
*
* @param field a field that has been indexed as a {@link KnnFloatVectorField}.
* @param target the target of the search
* @param k the number of documents to find
* @param filter a filter applied before the vector search
* @throws IllegalArgumentException if <code>k</code> is less than 1
*/
public KnnFloatVectorQuery(String field, float[] target, int k, Query filter) {
super(field, k, filter);
this.target = target;
}
@Override
protected TopDocs approximateSearch(LeafReaderContext context, Bits acceptDocs, int visitedLimit)
throws IOException {
TopDocs results =
context.reader().searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
return results != null ? results : NO_RESULTS;
}
@Override
VectorScorer createVectorScorer(LeafReaderContext context, FieldInfo fi) throws IOException {
if (fi.getVectorEncoding() != VectorEncoding.FLOAT32) {
return null;
}
return VectorScorer.create(context, fi, target);
}
@Override
public String toString(String field) {
return getClass().getSimpleName() + ":" + this.field + "[" + target[0] + ",...][" + k + "]";
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (super.equals(o) == false) return false;
KnnFloatVectorQuery that = (KnnFloatVectorQuery) o;
return Arrays.equals(target, that.target);
}
@Override
public int hashCode() {
int result = super.hashCode();
result = 31 * result + Arrays.hashCode(target);
return result;
}
/**
* @return the target query vector of the search. Each vector element is a float.
*/
public float[] getTargetCopy() {
return ArrayUtil.copyOfSubArray(target, 0, target.length);
}
}

83
lucene/core/src/java/org/apache/lucene/search/KnnVectorQuery.java
View File

@ -16,14 +16,7 @@
*/
package org.apache.lucene.search;
import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
/**
@ -38,81 +31,17 @@ import org.apache.lucene.util.Bits;
* <li>Otherwise run a kNN search subject to the filter
* <li>If the kNN search visits too many vectors without completing, stop and run an exact search
* </ul>
*
* @deprecated use {@link KnnFloatVectorQuery} instead
*/
public class KnnVectorQuery extends AbstractKnnVectorQuery {
@Deprecated
public class KnnVectorQuery extends KnnFloatVectorQuery {
private static final TopDocs NO_RESULTS = TopDocsCollector.EMPTY_TOPDOCS;
private final float[] target;
/**
* Find the <code>k</code> nearest documents to the target vector according to the vectors in the
* given field. <code>target</code> vector.
*
* @param field a field that has been indexed as a {@link KnnVectorField}.
* @param target the target of the search
* @param k the number of documents to find
* @throws IllegalArgumentException if <code>k</code> is less than 1
*/
public KnnVectorQuery(String field, float[] target, int k) {
this(field, target, k, null);
super(field, target, k);
}
/**
* Find the <code>k</code> nearest documents to the target vector according to the vectors in the
* given field. <code>target</code> vector.
*
* @param field a field that has been indexed as a {@link KnnVectorField}.
* @param target the target of the search
* @param k the number of documents to find
* @param filter a filter applied before the vector search
* @throws IllegalArgumentException if <code>k</code> is less than 1
*/
public KnnVectorQuery(String field, float[] target, int k, Query filter) {
super(field, k, filter);
this.target = target;
}
@Override
protected TopDocs approximateSearch(LeafReaderContext context, Bits acceptDocs, int visitedLimit)
throws IOException {
TopDocs results =
context.reader().searchNearestVectors(field, target, k, acceptDocs, visitedLimit);
return results != null ? results : NO_RESULTS;
}
@Override
VectorScorer createVectorScorer(LeafReaderContext context, FieldInfo fi) throws IOException {
if (fi.getVectorEncoding() != VectorEncoding.FLOAT32) {
return null;
}
return VectorScorer.create(context, fi, target);
}
@Override
public String toString(String field) {
return getClass().getSimpleName() + ":" + this.field + "[" + target[0] + ",...][" + k + "]";
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (super.equals(o) == false) return false;
KnnVectorQuery that = (KnnVectorQuery) o;
return Arrays.equals(target, that.target);
}
@Override
public int hashCode() {
int result = super.hashCode();
result = 31 * result + Arrays.hashCode(target);
return result;
}
/**
* @return the target query vector of the search. Each vector element is a float.
*/
public float[] getTargetCopy() {
return ArrayUtil.copyOfSubArray(target, 0, target.length);
super(field, target, k, filter);
}
}

12
lucene/core/src/java/org/apache/lucene/search/VectorScorer.java
View File

@ -19,14 +19,14 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
/**
* Computes the similarity score between a given query vector and different document vectors. This
* is primarily used by {@link org.apache.lucene.search.KnnVectorQuery} to run an exact, exhaustive
* search over the vectors.
* is primarily used by {@link KnnFloatVectorQuery} to run an exact, exhaustive search over the
* vectors.
*/
abstract class VectorScorer {
protected final VectorSimilarityFunction similarity;
@ -40,7 +40,7 @@ abstract class VectorScorer {
*/
static FloatVectorScorer create(LeafReaderContext context, FieldInfo fi, float[] query)
throws IOException {
VectorValues values = context.reader().getVectorValues(fi.name);
FloatVectorValues values = context.reader().getFloatVectorValues(fi.name);
final VectorSimilarityFunction similarity = fi.getVectorSimilarityFunction();
return new FloatVectorScorer(values, query, similarity);
}
@ -93,10 +93,10 @@ abstract class VectorScorer {
private static class FloatVectorScorer extends VectorScorer {
private final float[] query;
private final VectorValues values;
private final FloatVectorValues values;
protected FloatVectorScorer(
VectorValues values, float[] query, VectorSimilarityFunction similarity) {
FloatVectorValues values, float[] query, VectorSimilarityFunction similarity) {
super(similarity);
this.query = query;
this.values = values;

4
lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraph.java
View File

@ -22,7 +22,7 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import java.io.IOException;
import java.util.NoSuchElementException;
import java.util.PrimitiveIterator;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.index.FloatVectorValues;
/**
* Hierarchical Navigable Small World graph. Provides efficient approximate nearest neighbor search
@ -57,7 +57,7 @@ public abstract class HnswGraph {
*
* @param level level of the graph
* @param target ordinal of a node in the graph, must be &ge; 0 and &lt; {@link
* VectorValues#size()}.
* FloatVectorValues#size()}.
*/
public abstract void seek(int level, int target) throws IOException;

10
lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java
View File

@ -31,7 +31,7 @@ import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.KnnVectorsWriter;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexReader;
@ -90,12 +90,12 @@ public class TestPerFieldKnnVectorsFormat extends BaseKnnVectorsFormatTestCase {
try (IndexWriter iwriter = new IndexWriter(directory, iwc)) {
Document doc = new Document();
doc.add(newTextField("id", "1", Field.Store.YES));
doc.add(new KnnVectorField("field1", new float[] {1, 2, 3}));
doc.add(new KnnFloatVectorField("field1", new float[] {1, 2, 3}));
iwriter.addDocument(doc);
doc.clear();
doc.add(newTextField("id", "2", Field.Store.YES));
doc.add(new KnnVectorField("field2", new float[] {4, 5, 6}));
doc.add(new KnnFloatVectorField("field2", new float[] {4, 5, 6}));
iwriter.addDocument(doc);
}
@ -128,8 +128,8 @@ public class TestPerFieldKnnVectorsFormat extends BaseKnnVectorsFormatTestCase {
for (int i = 0; i < 3; i++) {
Document doc = new Document();
doc.add(newTextField("id", "1", Field.Store.YES));
doc.add(new KnnVectorField("field1", new float[] {1, 2, 3}));
doc.add(new KnnVectorField("field2", new float[] {1, 2, 3}));
doc.add(new KnnFloatVectorField("field1", new float[] {1, 2, 3}));
doc.add(new KnnFloatVectorField("field2", new float[] {1, 2, 3}));
iw.addDocument(doc);
iw.commit();
}

8
lucene/core/src/test/org/apache/lucene/document/TestField.java
View File

@ -23,12 +23,12 @@ import java.nio.charset.StandardCharsets;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
@ -644,9 +644,9 @@ public class TestField extends LuceneTestCase {
assertArrayEquals(b, field.vectorValue());
expectThrows(
IllegalArgumentException.class,
() -> new KnnVectorField("bogus", new float[] {1}, (FieldType) field.fieldType()));
() -> new KnnFloatVectorField("bogus", new float[] {1}, (FieldType) field.fieldType()));
float[] vector = new float[] {1, 2};
Field field2 = new KnnVectorField("float", vector);
Field field2 = new KnnFloatVectorField("float", vector);
assertNull(field2.binaryValue());
doc.add(field);
doc.add(field2);
@ -659,7 +659,7 @@ public class TestField extends LuceneTestCase {
assertArrayEquals(b, binary.vectorValue());
assertEquals(NO_MORE_DOCS, binary.nextDoc());
VectorValues floatValues = r.leaves().get(0).reader().getVectorValues("float");
FloatVectorValues floatValues = r.leaves().get(0).reader().getFloatVectorValues("float");
assertEquals(1, floatValues.size());
assertNotEquals(NO_MORE_DOCS, floatValues.nextDoc());
assertEquals(vector.length, floatValues.vectorValue().length);

6
lucene/core/src/test/org/apache/lucene/document/TestManyKnnDocs.java
View File

@ -23,7 +23,7 @@ import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.KnnVectorQuery;
import org.apache.lucene.search.KnnFloatVectorQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@ -55,7 +55,7 @@ public class TestManyKnnDocs extends LuceneTestCase {
int numVectors = 2088992;
float[] vector = new float[1];
Document doc = new Document();
doc.add(new KnnVectorField(fieldName, vector, similarityFunction));
doc.add(new KnnFloatVectorField(fieldName, vector, similarityFunction));
for (int i = 0; i < numVectors; i++) {
vector[0] = (i % 256);
iw.addDocument(doc);
@ -65,7 +65,7 @@ public class TestManyKnnDocs extends LuceneTestCase {
iw.forceMerge(1);
iw.commit();
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(dir));
TopDocs docs = searcher.search(new KnnVectorQuery("field", new float[] {120}, 10), 5);
TopDocs docs = searcher.search(new KnnFloatVectorQuery("field", new float[] {120}, 10), 5);
assertEquals(5, docs.scoreDocs.length);
}
}

2
lucene/core/src/test/org/apache/lucene/document/TestPerFieldConsistency.java
View File

@ -94,7 +94,7 @@ public class TestPerFieldConsistency extends LuceneTestCase {
for (int i = 0; i < values.length; i++) {
values[i] = randomFloat();
}
return new KnnVectorField(fieldName, values, similarityFunction);
return new KnnFloatVectorField(fieldName, values, similarityFunction);
}
private static Field[] randomFieldsWithTheSameName(String fieldName) {

4
lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectMismatchedChecksum.java
View File

@ -24,7 +24,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
@ -69,7 +69,7 @@ public class TestAllFilesDetectMismatchedChecksum extends LuceneTestCase {
doc.add(pointNumber);
Field dvNumber = new NumericDocValuesField("long", 0L);
doc.add(dvNumber);
KnnVectorField vector = new KnnVectorField("vector", new float[16]);
KnnFloatVectorField vector = new KnnFloatVectorField("vector", new float[16]);
doc.add(vector);
for (int i = 0; i < 100; i++) {

4
lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java
View File

@ -24,7 +24,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
@ -82,7 +82,7 @@ public class TestAllFilesDetectTruncation extends LuceneTestCase {
doc.add(pointNumber);
Field dvNumber = new NumericDocValuesField("long", 0L);
doc.add(dvNumber);
KnnVectorField vector = new KnnVectorField("vector", new float[16]);
KnnFloatVectorField vector = new KnnFloatVectorField("vector", new float[16]);
doc.add(vector);
for (int i = 0; i < 100; i++) {

6
lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
View File

@ -22,7 +22,7 @@ import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
@ -96,8 +96,8 @@ public class TestCheckIndex extends BaseTestCheckIndex {
doc.add(new StoredField("field", "value" + TestUtil.randomSimpleString(random())));
// vector
doc.add(new KnnVectorField("v1", randomVector(3)));
doc.add(new KnnVectorField("v2", randomVector(3)));
doc.add(new KnnFloatVectorField("v1", randomVector(3)));
doc.add(new KnnFloatVectorField("v2", randomVector(3)));
// doc value
doc.add(new NumericDocValuesField("dv", random().nextLong()));

4
lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java
View File

@ -31,7 +31,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
@ -382,7 +382,7 @@ public class TestDocumentWriter extends LuceneTestCase {
public void testRAMUsageVector() throws IOException {
doTestRAMUsage(
field ->
new KnnVectorField(
new KnnFloatVectorField(
field, new float[] {1, 2, 3, 4}, VectorSimilarityFunction.EUCLIDEAN));
}
}

14
lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java
View File

@ -25,7 +25,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
@ -420,8 +420,8 @@ public class TestExitableDirectoryReader extends LuceneTestCase {
value[j] = random().nextFloat();
}
FieldType fieldType =
KnnVectorField.createFieldType(dimension, VectorSimilarityFunction.COSINE);
doc.add(new KnnVectorField("vector", value, fieldType));
KnnFloatVectorField.createFieldType(dimension, VectorSimilarityFunction.COSINE);
doc.add(new KnnFloatVectorField("vector", value, fieldType));
doc.add(new StringField("id", Integer.toString(i), Field.Store.YES));
writer.addDocument(doc);
@ -455,7 +455,7 @@ public class TestExitableDirectoryReader extends LuceneTestCase {
expectThrows(
ExitingReaderException.class,
() -> {
DocIdSetIterator iter = leaf.getVectorValues("vector");
DocIdSetIterator iter = leaf.getFloatVectorValues("vector");
scanAndRetrieve(leaf, iter);
});
@ -465,7 +465,7 @@ public class TestExitableDirectoryReader extends LuceneTestCase {
leaf.searchNearestVectors(
"vector", new float[dimension], 5, leaf.getLiveDocs(), Integer.MAX_VALUE));
} else {
DocIdSetIterator iter = leaf.getVectorValues("vector");
DocIdSetIterator iter = leaf.getFloatVectorValues("vector");
scanAndRetrieve(leaf, iter);
leaf.searchNearestVectors(
@ -488,8 +488,8 @@ public class TestExitableDirectoryReader extends LuceneTestCase {
if (random().nextBoolean()
&& iter.docID() != DocIdSetIterator.NO_MORE_DOCS
&& iter instanceof VectorValues) {
((VectorValues) iter).vectorValue();
&& iter instanceof FloatVectorValues) {
((FloatVectorValues) iter).vectorValue();
}
}
}

17
lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
View File

@ -37,11 +37,11 @@ import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.KnnVectorQuery;
import org.apache.lucene.search.KnnFloatVectorQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.SearcherManager;
@ -204,7 +204,7 @@ public class TestKnnGraph extends LuceneTestCase {
for (int field = 0; field < numVectorFields; field++) {
dims[field] = atLeast(3);
values[field] = randomVectors(numDoc, dims[field]);
fieldTypes[field] = KnnVectorField.createFieldType(dims[field], similarityFunction);
fieldTypes[field] = KnnFloatVectorField.createFieldType(dims[field], similarityFunction);
}
try (Directory dir = newDirectory();
@ -214,7 +214,7 @@ public class TestKnnGraph extends LuceneTestCase {
for (int field = 0; field < numVectorFields; field++) {
float[] vector = values[field][docID];
if (vector != null) {
doc.add(new KnnVectorField(KNN_GRAPH_FIELD + field, vector, fieldTypes[field]));
doc.add(new KnnFloatVectorField(KNN_GRAPH_FIELD + field, vector, fieldTypes[field]));
}
}
String idString = Integer.toString(docID);
@ -395,7 +395,8 @@ public class TestKnnGraph extends LuceneTestCase {
latch.await();
IndexSearcher searcher = manager.acquire();
try {
KnnVectorQuery query = new KnnVectorQuery("vector", new float[] {0f, 0.1f}, 5);
KnnFloatVectorQuery query =
new KnnFloatVectorQuery("vector", new float[] {0f, 0.1f}, 5);
TopDocs results = searcher.search(query, 5);
StoredFields storedFields = searcher.storedFields();
for (ScoreDoc doc : results.scoreDocs) {
@ -481,7 +482,7 @@ public class TestKnnGraph extends LuceneTestCase {
continue;
}
HnswGraph graphValues = vectorReader.getGraph(vectorField);
VectorValues vectorValues = reader.getVectorValues(vectorField);
FloatVectorValues vectorValues = reader.getFloatVectorValues(vectorField);
if (vectorValues == null) {
assert graphValues == null;
continue;
@ -632,8 +633,8 @@ public class TestKnnGraph extends LuceneTestCase {
throws IOException {
Document doc = new Document();
if (vector != null) {
FieldType fieldType = KnnVectorField.createFieldType(vector.length, similarityFunction);
doc.add(new KnnVectorField(KNN_GRAPH_FIELD, vector, fieldType));
FieldType fieldType = KnnFloatVectorField.createFieldType(vector.length, similarityFunction);
doc.add(new KnnFloatVectorField(KNN_GRAPH_FIELD, vector, fieldType));
}
String idString = Integer.toString(id);
doc.add(new StringField("id", idString, Field.Store.YES));

2
lucene/core/src/test/org/apache/lucene/index/TestSegmentToThreadMapping.java
View File

@ -107,7 +107,7 @@ public class TestSegmentToThreadMapping extends LuceneTestCase {
}
@Override
public VectorValues getVectorValues(String field) {
public FloatVectorValues getFloatVectorValues(String field) {
return null;
}

8
lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.java
View File

@ -30,7 +30,7 @@ import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
@ -168,7 +168,7 @@ public class TestSortingCodecReader extends LuceneTestCase {
doc.add(new BinaryDocValuesField("binary_dv", new BytesRef(Integer.toString(docId))));
doc.add(
new SortedSetDocValuesField("sorted_set_dv", new BytesRef(Integer.toString(docId))));
doc.add(new KnnVectorField("vector", new float[] {(float) docId}));
doc.add(new KnnFloatVectorField("vector", new float[] {(float) docId}));
doc.add(new NumericDocValuesField("foo", random().nextInt(20)));
FieldType ft = new FieldType(StringField.TYPE_NOT_STORED);
@ -238,7 +238,7 @@ public class TestSortingCodecReader extends LuceneTestCase {
leaf.getSortedNumericDocValues("sorted_numeric_dv");
SortedSetDocValues sorted_set_dv = leaf.getSortedSetDocValues("sorted_set_dv");
SortedDocValues binary_sorted_dv = leaf.getSortedDocValues("binary_sorted_dv");
VectorValues vectorValues = leaf.getVectorValues("vector");
FloatVectorValues vectorValues = leaf.getFloatVectorValues("vector");
NumericDocValues ids = leaf.getNumericDocValues("id");
long prevValue = -1;
boolean usingAltIds = false;
@ -253,7 +253,7 @@ public class TestSortingCodecReader extends LuceneTestCase {
sorted_numeric_dv = leaf.getSortedNumericDocValues("sorted_numeric_dv");
sorted_set_dv = leaf.getSortedSetDocValues("sorted_set_dv");
binary_sorted_dv = leaf.getSortedDocValues("binary_sorted_dv");
vectorValues = leaf.getVectorValues("vector");
vectorValues = leaf.getFloatVectorValues("vector");
prevValue = -1;
}
assertTrue(prevValue + " < " + ids.longValue(), prevValue < ids.longValue());

10
lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java
View File

@ -23,7 +23,7 @@ import org.apache.lucene.document.DoubleDocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
@ -583,7 +583,7 @@ public class TestFieldExistsQuery extends LuceneTestCase {
Document doc = new Document();
boolean hasValue = random().nextBoolean();
if (hasValue) {
doc.add(new KnnVectorField("vector", randomVector(5)));
doc.add(new KnnFloatVectorField("vector", randomVector(5)));
doc.add(new StringField("has_value", "yes", Store.NO));
}
doc.add(new StringField("field", "value", Store.NO));
@ -632,7 +632,7 @@ public class TestFieldExistsQuery extends LuceneTestCase {
RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
for (int i = 0; i < 100; ++i) {
Document doc = new Document();
doc.add(new KnnVectorField("vector", randomVector(5)));
doc.add(new KnnFloatVectorField("vector", randomVector(5)));
iw.addDocument(doc);
}
iw.commit();
@ -656,7 +656,7 @@ public class TestFieldExistsQuery extends LuceneTestCase {
for (int i = 0; i < numDocs; ++i) {
Document doc = new Document();
if (allDocsHaveVector || random().nextBoolean()) {
doc.add(new KnnVectorField("vector", randomVector(5)));
doc.add(new KnnFloatVectorField("vector", randomVector(5)));
numVectors++;
}
doc.add(new StringField("field", "value" + (i % 2), Store.NO));
@ -685,7 +685,7 @@ public class TestFieldExistsQuery extends LuceneTestCase {
RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
// 1st segment has the field, but 2nd one does not
Document doc = new Document();
doc.add(new KnnVectorField("vector", randomVector(3)));
doc.add(new KnnFloatVectorField("vector", randomVector(3)));
iw.addDocument(doc);
iw.commit();
iw.addDocument(new Document());

18
lucene/core/src/test/org/apache/lucene/search/TestKnnVectorQuery.java → lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java
View File

@ -22,7 +22,7 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
@ -33,10 +33,10 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.util.TestVectorUtil;
import org.apache.lucene.util.VectorUtil;
public class TestKnnVectorQuery extends BaseKnnVectorQueryTestCase {
public class TestKnnFloatVectorQuery extends BaseKnnVectorQueryTestCase {
@Override
KnnVectorQuery getKnnVectorQuery(String field, float[] query, int k, Query queryFilter) {
return new KnnVectorQuery(field, query, k, queryFilter);
KnnFloatVectorQuery getKnnVectorQuery(String field, float[] query, int k, Query queryFilter) {
return new KnnFloatVectorQuery(field, query, k, queryFilter);
}
@Override
@ -52,22 +52,22 @@ public class TestKnnVectorQuery extends BaseKnnVectorQueryTestCase {
@Override
Field getKnnVectorField(
String name, float[] vector, VectorSimilarityFunction similarityFunction) {
return new KnnVectorField(name, vector, similarityFunction);
return new KnnFloatVectorField(name, vector, similarityFunction);
}
@Override
Field getKnnVectorField(String name, float[] vector) {
return new KnnVectorField(name, vector);
return new KnnFloatVectorField(name, vector);
}
public void testToString() {
AbstractKnnVectorQuery q1 = getKnnVectorQuery("f1", new float[] {0, 1}, 10);
assertEquals("KnnVectorQuery:f1[0.0,...][10]", q1.toString("ignored"));
assertEquals("KnnFloatVectorQuery:f1[0.0,...][10]", q1.toString("ignored"));
}
public void testGetTarget() {
float[] queryVector = new float[] {0, 1};
KnnVectorQuery q1 = new KnnVectorQuery("f1", queryVector, 10);
KnnFloatVectorQuery q1 = new KnnFloatVectorQuery("f1", queryVector, 10);
assertArrayEquals(queryVector, q1.getTargetCopy(), 0);
assertNotEquals(queryVector, q1.getTargetCopy());
@ -159,7 +159,7 @@ public class TestKnnVectorQuery extends BaseKnnVectorQueryTestCase {
}
}
private static class ThrowingKnnVectorQuery extends KnnVectorQuery {
private static class ThrowingKnnVectorQuery extends KnnFloatVectorQuery {
public ThrowingKnnVectorQuery(String field, float[] target, int k, Query filter) {
super(field, target, k, filter);

4
lucene/core/src/test/org/apache/lucene/search/TestVectorScorer.java
View File

@ -23,7 +23,7 @@ import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnByteVectorField;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo;
@ -81,7 +81,7 @@ public class TestVectorScorer extends LuceneTestCase {
}
doc.add(new KnnByteVectorField(field, v, EUCLIDEAN));
} else {
doc.add(new KnnVectorField(field, contents[i]));
doc.add(new KnnFloatVectorField(field, contents[i]));
}
doc.add(new StringField("id", "id" + i, Field.Store.YES));
writer.addDocument(doc);

10
lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java
View File

@ -41,6 +41,7 @@ import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@ -49,7 +50,6 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
@ -728,21 +728,21 @@ abstract class HnswGraphTestCase<T> extends LuceneTestCase {
}
/** Returns vectors evenly distributed around the upper unit semicircle. */
static class CircularVectorValues extends VectorValues
static class CircularFloatVectorValues extends FloatVectorValues
implements RandomAccessVectorValues<float[]> {
private final int size;
private final float[] value;
int doc = -1;
CircularVectorValues(int size) {
CircularFloatVectorValues(int size) {
this.size = size;
value = new float[2];
}
@Override
public CircularVectorValues copy() {
return new CircularVectorValues(size);
public CircularFloatVectorValues copy() {
return new CircularFloatVectorValues(size);
}
@Override

11
lucene/core/src/test/org/apache/lucene/util/hnsw/KnnGraphTester.java
View File

@ -46,7 +46,7 @@ import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KnnByteVectorField;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DirectoryReader;
@ -60,7 +60,7 @@ import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.KnnVectorQuery;
import org.apache.lucene.search.KnnFloatVectorQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.ScoreDoc;
@ -540,7 +540,7 @@ public class KnnGraphTester {
private static TopDocs doKnnVectorQuery(
IndexSearcher searcher, String field, float[] vector, int k, int fanout, Query filter)
throws IOException {
return searcher.search(new KnnVectorQuery(field, vector, k + fanout, filter), k);
return searcher.search(new KnnFloatVectorQuery(field, vector, k + fanout, filter), k);
}
private float checkResults(TopDocs[] results, int[][] nn) {
@ -704,7 +704,7 @@ public class KnnGraphTester {
FieldType fieldType =
switch (vectorEncoding) {
case BYTE -> KnnByteVectorField.createFieldType(dim, similarityFunction);
case FLOAT32 -> KnnVectorField.createFieldType(dim, similarityFunction);
case FLOAT32 -> KnnFloatVectorField.createFieldType(dim, similarityFunction);
};
if (quiet == false) {
iwc.setInfoStream(new PrintStreamInfoStream(System.out));
@ -721,7 +721,8 @@ public class KnnGraphTester {
case BYTE -> doc.add(
new KnnByteVectorField(
KNN_FIELD, ((VectorReaderByte) vectorReader).nextBytes(), fieldType));
case FLOAT32 -> doc.add(new KnnVectorField(KNN_FIELD, vectorReader.next(), fieldType));
case FLOAT32 -> doc.add(
new KnnFloatVectorField(KNN_FIELD, vectorReader.next(), fieldType));
}
doc.add(new StoredField(ID_FIELD, i));
iw.addDocument(doc);

14
lucene/core/src/test/org/apache/lucene/util/hnsw/TestHnswFloatVectorGraph.java
View File

@ -22,12 +22,12 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import java.io.IOException;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.KnnVectorQuery;
import org.apache.lucene.search.KnnFloatVectorQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.FixedBitSet;
@ -48,7 +48,7 @@ public class TestHnswFloatVectorGraph extends HnswGraphTestCase<float[]> {
@Override
Query knnQuery(String field, float[] vector, int k) {
return new KnnVectorQuery(field, vector, k);
return new KnnFloatVectorQuery(field, vector, k);
}
@Override
@ -69,7 +69,7 @@ public class TestHnswFloatVectorGraph extends HnswGraphTestCase<float[]> {
@Override
AbstractMockVectorValues<float[]> vectorValues(LeafReader reader, String fieldName)
throws IOException {
VectorValues vectorValues = reader.getVectorValues(fieldName);
FloatVectorValues vectorValues = reader.getFloatVectorValues(fieldName);
float[][] vectors = new float[reader.maxDoc()][];
while (vectorValues.nextDoc() != NO_MORE_DOCS) {
vectors[vectorValues.docID()] =
@ -81,12 +81,12 @@ public class TestHnswFloatVectorGraph extends HnswGraphTestCase<float[]> {
@Override
Field knnVectorField(String name, float[] vector, VectorSimilarityFunction similarityFunction) {
return new KnnVectorField(name, vector, similarityFunction);
return new KnnFloatVectorField(name, vector, similarityFunction);
}
@Override
RandomAccessVectorValues<float[]> circularVectorValues(int nDoc) {
return new CircularVectorValues(nDoc);
return new CircularFloatVectorValues(nDoc);
}
@Override

5
lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java
View File

@ -34,7 +34,7 @@ import org.apache.lucene.demo.knn.DemoEmbeddings;
import org.apache.lucene.demo.knn.KnnVectorDict;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
@ -262,7 +262,8 @@ public class IndexFiles implements AutoCloseable {
demoEmbeddings.computeEmbedding(
new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)));
doc.add(
new KnnVectorField("contents-vector", vector, VectorSimilarityFunction.DOT_PRODUCT));
new KnnFloatVectorField(
"contents-vector", vector, VectorSimilarityFunction.DOT_PRODUCT));
}
}

6
lucene/demo/src/java/org/apache/lucene/demo/SearchFiles.java
View File

@ -37,7 +37,7 @@ import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.KnnVectorQuery;
import org.apache.lucene.search.KnnFloatVectorQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.ScoreDoc;
@ -280,8 +280,8 @@ public class SearchFiles {
semanticQueryText.append(term).append(' ');
}
if (semanticQueryText.length() > 0) {
KnnVectorQuery knnQuery =
new KnnVectorQuery(
KnnFloatVectorQuery knnQuery =
new KnnFloatVectorQuery(
"contents-vector",
new DemoEmbeddings(vectorDict).computeEmbedding(semanticQueryText.toString()),
k);

4
lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java
View File

@ -25,6 +25,7 @@ import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.LeafMetaData;
import org.apache.lucene.index.LeafReader;
@ -39,7 +40,6 @@ import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.Version;
@ -161,7 +161,7 @@ public class TermVectorLeafReader extends LeafReader {
}
@Override
public VectorValues getVectorValues(String fieldName) {
public FloatVectorValues getFloatVectorValues(String fieldName) {
return null;
}

2
lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
View File

@ -1391,7 +1391,7 @@ public class MemoryIndex {
}
@Override
public VectorValues getVectorValues(String fieldName) {
public FloatVectorValues getFloatVectorValues(String fieldName) {
return null;
}

16
lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java
View File

@ -25,12 +25,12 @@ import org.apache.lucene.codecs.KnnVectorsWriter;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Sorter;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.Bits;
@ -111,17 +111,17 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
FieldInfo fi = fis.fieldInfo(field);
assert fi != null
&& fi.getVectorDimension() > 0
&& fi.getVectorEncoding() == VectorEncoding.FLOAT32;
VectorValues values = delegate.getVectorValues(field);
assert values != null;
assert values.docID() == -1;
assert values.size() >= 0;
assert values.dimension() > 0;
return values;
FloatVectorValues floatValues = delegate.getFloatVectorValues(field);
assert floatValues != null;
assert floatValues.docID() == -1;
assert floatValues.size() >= 0;
assert floatValues.dimension() > 0;
return floatValues;
}
@Override

4
lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java
View File

@ -32,13 +32,13 @@ import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.internal.tests.IndexPackageAccess;
import org.apache.lucene.internal.tests.TestSecrets;
import org.apache.lucene.store.Directory;
@ -352,7 +352,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
}
if (r.nextBoolean()) {
int dimension = 1 + r.nextInt(VectorValues.MAX_DIMENSIONS);
int dimension = 1 + r.nextInt(FloatVectorValues.MAX_DIMENSIONS);
VectorSimilarityFunction similarityFunction =
RandomPicks.randomFrom(r, VectorSimilarityFunction.values());
VectorEncoding encoding = RandomPicks.randomFrom(r, VectorEncoding.values());

177
lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
View File

@ -29,7 +29,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KnnByteVectorField;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
@ -37,6 +37,7 @@ import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@ -46,7 +47,6 @@ import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
@ -82,36 +82,37 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
protected void addRandomFields(Document doc) {
switch (vectorEncoding) {
case BYTE -> doc.add(new KnnByteVectorField("v2", randomVector8(30), similarityFunction));
case FLOAT32 -> doc.add(new KnnVectorField("v2", randomVector(30), similarityFunction));
case FLOAT32 -> doc.add(new KnnFloatVectorField("v2", randomVector(30), similarityFunction));
}
}
public void testFieldConstructor() {
float[] v = new float[1];
KnnVectorField field = new KnnVectorField("f", v);
KnnFloatVectorField field = new KnnFloatVectorField("f", v);
assertEquals(1, field.fieldType().vectorDimension());
assertEquals(VectorSimilarityFunction.EUCLIDEAN, field.fieldType().vectorSimilarityFunction());
assertSame(v, field.vectorValue());
}
public void testFieldConstructorExceptions() {
expectThrows(IllegalArgumentException.class, () -> new KnnVectorField(null, new float[1]));
expectThrows(IllegalArgumentException.class, () -> new KnnVectorField("f", null));
expectThrows(IllegalArgumentException.class, () -> new KnnFloatVectorField(null, new float[1]));
expectThrows(IllegalArgumentException.class, () -> new KnnFloatVectorField("f", null));
expectThrows(
IllegalArgumentException.class,
() -> new KnnVectorField("f", new float[1], (VectorSimilarityFunction) null));
expectThrows(IllegalArgumentException.class, () -> new KnnVectorField("f", new float[0]));
() -> new KnnFloatVectorField("f", new float[1], (VectorSimilarityFunction) null));
expectThrows(IllegalArgumentException.class, () -> new KnnFloatVectorField("f", new float[0]));
expectThrows(
IllegalArgumentException.class,
() -> new KnnVectorField("f", new float[VectorValues.MAX_DIMENSIONS + 1]));
() -> new KnnFloatVectorField("f", new float[FloatVectorValues.MAX_DIMENSIONS + 1]));
expectThrows(
IllegalArgumentException.class,
() ->
new KnnVectorField("f", new float[VectorValues.MAX_DIMENSIONS + 1], (FieldType) null));
new KnnFloatVectorField(
"f", new float[FloatVectorValues.MAX_DIMENSIONS + 1], (FieldType) null));
}
public void testFieldSetValue() {
KnnVectorField field = new KnnVectorField("f", new float[1]);
KnnFloatVectorField field = new KnnFloatVectorField("f", new float[1]);
float[] v1 = new float[1];
field.setVectorValue(v1);
assertSame(v1, field.vectorValue());
@ -125,11 +126,11 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
Document doc2 = new Document();
doc2.add(new KnnVectorField("f", new float[3], VectorSimilarityFunction.DOT_PRODUCT));
doc2.add(new KnnFloatVectorField("f", new float[3], VectorSimilarityFunction.DOT_PRODUCT));
IllegalArgumentException expected =
expectThrows(IllegalArgumentException.class, () -> w.addDocument(doc2));
String errMsg =
@ -142,12 +143,12 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
w.commit();
Document doc2 = new Document();
doc2.add(new KnnVectorField("f", new float[3], VectorSimilarityFunction.DOT_PRODUCT));
doc2.add(new KnnFloatVectorField("f", new float[3], VectorSimilarityFunction.DOT_PRODUCT));
IllegalArgumentException expected =
expectThrows(IllegalArgumentException.class, () -> w.addDocument(doc2));
String errMsg =
@ -162,11 +163,11 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
Document doc2 = new Document();
doc2.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
doc2.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
IllegalArgumentException expected =
expectThrows(IllegalArgumentException.class, () -> w.addDocument(doc2));
String errMsg =
@ -179,12 +180,12 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
w.commit();
Document doc2 = new Document();
doc2.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
doc2.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
IllegalArgumentException expected =
expectThrows(IllegalArgumentException.class, () -> w.addDocument(doc2));
String errMsg =
@ -198,13 +199,13 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (Directory dir = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
}
try (IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc2 = new Document();
doc2.add(new KnnVectorField("f", new float[1], VectorSimilarityFunction.DOT_PRODUCT));
doc2.add(new KnnFloatVectorField("f", new float[1], VectorSimilarityFunction.DOT_PRODUCT));
IllegalArgumentException expected =
expectThrows(IllegalArgumentException.class, () -> w2.addDocument(doc2));
assertEquals(
@ -219,13 +220,13 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (Directory dir = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
}
try (IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc2 = new Document();
doc2.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
doc2.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
IllegalArgumentException expected =
expectThrows(IllegalArgumentException.class, () -> w2.addDocument(doc2));
assertEquals(
@ -239,7 +240,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
public void testAddIndexesDirectory0() throws Exception {
String fieldName = "field";
Document doc = new Document();
doc.add(new KnnVectorField(fieldName, new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField(fieldName, new float[4], VectorSimilarityFunction.DOT_PRODUCT));
try (Directory dir = newDirectory();
Directory dir2 = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
@ -250,7 +251,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
w2.forceMerge(1);
try (IndexReader reader = DirectoryReader.open(w2)) {
LeafReader r = getOnlyLeafReader(reader);
VectorValues vectorValues = r.getVectorValues(fieldName);
FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName);
assertEquals(0, vectorValues.nextDoc());
assertEquals(0, vectorValues.vectorValue()[0], 0);
assertEquals(NO_MORE_DOCS, vectorValues.nextDoc());
@ -267,14 +268,15 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
w.addDocument(doc);
}
doc.add(new KnnVectorField(fieldName, new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(
new KnnFloatVectorField(fieldName, new float[4], VectorSimilarityFunction.DOT_PRODUCT));
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
w2.addDocument(doc);
w2.addIndexes(dir);
w2.forceMerge(1);
try (IndexReader reader = DirectoryReader.open(w2)) {
LeafReader r = getOnlyLeafReader(reader);
VectorValues vectorValues = r.getVectorValues(fieldName);
FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName);
assertNotEquals(NO_MORE_DOCS, vectorValues.nextDoc());
assertEquals(0, vectorValues.vectorValue()[0], 0);
assertEquals(NO_MORE_DOCS, vectorValues.nextDoc());
@ -287,7 +289,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
String fieldName = "field";
float[] vector = new float[1];
Document doc = new Document();
doc.add(new KnnVectorField(fieldName, vector, VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField(fieldName, vector, VectorSimilarityFunction.DOT_PRODUCT));
try (Directory dir = newDirectory();
Directory dir2 = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
@ -300,7 +302,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
w2.forceMerge(1);
try (IndexReader reader = DirectoryReader.open(w2)) {
LeafReader r = getOnlyLeafReader(reader);
VectorValues vectorValues = r.getVectorValues(fieldName);
FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName);
assertEquals(0, vectorValues.nextDoc());
// The merge order is randomized, we might get 0 first, or 1
float value = vectorValues.vectorValue()[0];
@ -318,12 +320,12 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
Directory dir2 = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
}
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[5], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[5], VectorSimilarityFunction.DOT_PRODUCT));
w2.addDocument(doc);
IllegalArgumentException expected =
expectThrows(
@ -341,12 +343,12 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
Directory dir2 = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
}
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
w2.addDocument(doc);
IllegalArgumentException expected =
expectThrows(IllegalArgumentException.class, () -> w2.addIndexes(dir));
@ -363,12 +365,12 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
Directory dir2 = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
}
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[5], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[5], VectorSimilarityFunction.DOT_PRODUCT));
w2.addDocument(doc);
try (DirectoryReader r = DirectoryReader.open(dir)) {
IllegalArgumentException expected =
@ -389,12 +391,12 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
Directory dir2 = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
}
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
w2.addDocument(doc);
try (DirectoryReader r = DirectoryReader.open(dir)) {
IllegalArgumentException expected =
@ -415,12 +417,12 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
Directory dir2 = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
}
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[5], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[5], VectorSimilarityFunction.DOT_PRODUCT));
w2.addDocument(doc);
try (DirectoryReader r = DirectoryReader.open(dir)) {
IllegalArgumentException expected =
@ -439,12 +441,12 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
Directory dir2 = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
}
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.EUCLIDEAN));
w2.addDocument(doc);
try (DirectoryReader r = DirectoryReader.open(dir)) {
IllegalArgumentException expected =
@ -462,8 +464,8 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
IllegalArgumentException expected =
expectThrows(IllegalArgumentException.class, () -> w.addDocument(doc));
assertEquals(
@ -480,13 +482,13 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
IllegalArgumentException.class,
() ->
doc.add(
new KnnVectorField(
new KnnFloatVectorField(
"f",
new float[VectorValues.MAX_DIMENSIONS + 1],
new float[FloatVectorValues.MAX_DIMENSIONS + 1],
VectorSimilarityFunction.DOT_PRODUCT)));
Document doc2 = new Document();
doc2.add(new KnnVectorField("f", new float[1], VectorSimilarityFunction.EUCLIDEAN));
doc2.add(new KnnFloatVectorField("f", new float[1], VectorSimilarityFunction.EUCLIDEAN));
w.addDocument(doc2);
}
}
@ -500,11 +502,12 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
IllegalArgumentException.class,
() ->
doc.add(
new KnnVectorField("f", new float[0], VectorSimilarityFunction.EUCLIDEAN)));
new KnnFloatVectorField(
"f", new float[0], VectorSimilarityFunction.EUCLIDEAN)));
assertEquals("cannot index an empty vector", e.getMessage());
Document doc2 = new Document();
doc2.add(new KnnVectorField("f", new float[1], VectorSimilarityFunction.EUCLIDEAN));
doc2.add(new KnnFloatVectorField("f", new float[1], VectorSimilarityFunction.EUCLIDEAN));
w.addDocument(doc2);
}
}
@ -514,14 +517,14 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (Directory dir = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
}
IndexWriterConfig iwc = newIndexWriterConfig();
iwc.setCodec(Codec.forName("SimpleText"));
try (IndexWriter w = new IndexWriter(dir, iwc)) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
w.forceMerge(1);
}
@ -535,12 +538,12 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (Directory dir = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, iwc)) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
}
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
doc.add(new KnnFloatVectorField("f", new float[4], VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
w.forceMerge(1);
}
@ -548,8 +551,8 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
}
public void testInvalidKnnVectorFieldUsage() {
KnnVectorField field =
new KnnVectorField("field", new float[2], VectorSimilarityFunction.EUCLIDEAN);
KnnFloatVectorField field =
new KnnFloatVectorField("field", new float[2], VectorSimilarityFunction.EUCLIDEAN);
expectThrows(IllegalArgumentException.class, () -> field.setIntValue(14));
@ -563,13 +566,15 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new StringField("id", "0", Field.Store.NO));
doc.add(new KnnVectorField("v", new float[] {2, 3, 5}, VectorSimilarityFunction.DOT_PRODUCT));
doc.add(
new KnnFloatVectorField(
"v", new float[] {2, 3, 5}, VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
w.addDocument(new Document());
w.commit();
try (DirectoryReader r = DirectoryReader.open(w)) {
VectorValues values = getOnlyLeafReader(r).getVectorValues("v");
FloatVectorValues values = getOnlyLeafReader(r).getFloatVectorValues("v");
assertNotNull(values);
assertEquals(1, values.size());
}
@ -577,7 +582,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
w.forceMerge(1);
try (DirectoryReader r = DirectoryReader.open(w)) {
LeafReader leafReader = getOnlyLeafReader(r);
VectorValues values = leafReader.getVectorValues("v");
FloatVectorValues values = leafReader.getFloatVectorValues("v");
assertNotNull(values);
assertEquals(0, values.size());
@ -596,13 +601,15 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
Document doc = new Document();
doc.add(new StringField("id", "0", Field.Store.NO));
doc.add(
new KnnVectorField("v0", new float[] {2, 3, 5}, VectorSimilarityFunction.DOT_PRODUCT));
new KnnFloatVectorField(
"v0", new float[] {2, 3, 5}, VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
w.commit();
doc = new Document();
doc.add(
new KnnVectorField("v1", new float[] {2, 3, 5}, VectorSimilarityFunction.DOT_PRODUCT));
new KnnFloatVectorField(
"v1", new float[] {2, 3, 5}, VectorSimilarityFunction.DOT_PRODUCT));
w.addDocument(doc);
w.forceMerge(1);
}
@ -636,7 +643,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
}
case FLOAT32 -> {
float[] v = randomVector(fieldDims[field]);
doc.add(new KnnVectorField(fieldName, v, fieldSimilarityFunctions[field]));
doc.add(new KnnFloatVectorField(fieldName, v, fieldSimilarityFunctions[field]));
fieldTotals[field] += v[0];
}
}
@ -664,7 +671,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
}
case FLOAT32 -> {
for (LeafReaderContext ctx : r.leaves()) {
VectorValues vectorValues = ctx.reader().getVectorValues(fieldName);
FloatVectorValues vectorValues = ctx.reader().getFloatVectorValues(fieldName);
if (vectorValues != null) {
docCount += vectorValues.size();
while (vectorValues.nextDoc() != NO_MORE_DOCS) {
@ -704,20 +711,20 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc1 = new Document();
doc1.add(new KnnVectorField(fieldName, v, VectorSimilarityFunction.EUCLIDEAN));
doc1.add(new KnnFloatVectorField(fieldName, v, VectorSimilarityFunction.EUCLIDEAN));
v[0] = 1;
Document doc2 = new Document();
doc2.add(new KnnVectorField(fieldName, v, VectorSimilarityFunction.EUCLIDEAN));
doc2.add(new KnnFloatVectorField(fieldName, v, VectorSimilarityFunction.EUCLIDEAN));
iw.addDocument(doc1);
iw.addDocument(doc2);
v[0] = 2;
Document doc3 = new Document();
doc3.add(new KnnVectorField(fieldName, v, VectorSimilarityFunction.EUCLIDEAN));
doc3.add(new KnnFloatVectorField(fieldName, v, VectorSimilarityFunction.EUCLIDEAN));
iw.addDocument(doc3);
iw.forceMerge(1);
try (IndexReader reader = DirectoryReader.open(iw)) {
LeafReader r = getOnlyLeafReader(reader);
VectorValues vectorValues = r.getVectorValues(fieldName);
FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName);
vectorValues.nextDoc();
assertEquals(1, vectorValues.vectorValue()[0], 0);
vectorValues.nextDoc();
@ -743,7 +750,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
LeafReader leaf = getOnlyLeafReader(reader);
StoredFields storedFields = leaf.storedFields();
VectorValues vectorValues = leaf.getVectorValues(fieldName);
FloatVectorValues vectorValues = leaf.getFloatVectorValues(fieldName);
assertEquals(2, vectorValues.dimension());
assertEquals(3, vectorValues.size());
assertEquals("1", storedFields.document(vectorValues.nextDoc()).get("id"));
@ -791,22 +798,23 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
float[] v = new float[] {1};
doc.add(new KnnVectorField("field1", v, VectorSimilarityFunction.EUCLIDEAN));
doc.add(new KnnFloatVectorField("field1", v, VectorSimilarityFunction.EUCLIDEAN));
doc.add(
new KnnVectorField("field2", new float[] {1, 2, 3}, VectorSimilarityFunction.EUCLIDEAN));
new KnnFloatVectorField(
"field2", new float[] {1, 2, 3}, VectorSimilarityFunction.EUCLIDEAN));
iw.addDocument(doc);
v[0] = 2;
iw.addDocument(doc);
doc = new Document();
doc.add(
new KnnVectorField(
new KnnFloatVectorField(
"field3", new float[] {1, 2, 3}, VectorSimilarityFunction.DOT_PRODUCT));
iw.addDocument(doc);
iw.forceMerge(1);
try (IndexReader reader = DirectoryReader.open(iw)) {
LeafReader leaf = reader.leaves().get(0).reader();
VectorValues vectorValues = leaf.getVectorValues("field1");
FloatVectorValues vectorValues = leaf.getFloatVectorValues("field1");
assertEquals(1, vectorValues.dimension());
assertEquals(2, vectorValues.size());
vectorValues.nextDoc();
@ -815,7 +823,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
assertEquals(2f, vectorValues.vectorValue()[0], 0);
assertEquals(NO_MORE_DOCS, vectorValues.nextDoc());
VectorValues vectorValues2 = leaf.getVectorValues("field2");
FloatVectorValues vectorValues2 = leaf.getFloatVectorValues("field2");
assertEquals(3, vectorValues2.dimension());
assertEquals(2, vectorValues2.size());
vectorValues2.nextDoc();
@ -824,7 +832,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
assertEquals(2f, vectorValues2.vectorValue()[1], 0);
assertEquals(NO_MORE_DOCS, vectorValues2.nextDoc());
VectorValues vectorValues3 = leaf.getVectorValues("field3");
FloatVectorValues vectorValues3 = leaf.getFloatVectorValues("field3");
assertEquals(3, vectorValues3.dimension());
assertEquals(1, vectorValues3.size());
vectorValues3.nextDoc();
@ -883,7 +891,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (IndexReader reader = DirectoryReader.open(iw)) {
int valueCount = 0, totalSize = 0;
for (LeafReaderContext ctx : reader.leaves()) {
VectorValues vectorValues = ctx.reader().getVectorValues(fieldName);
FloatVectorValues vectorValues = ctx.reader().getFloatVectorValues(fieldName);
if (vectorValues == null) {
continue;
}
@ -1020,7 +1028,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (IndexReader reader = DirectoryReader.open(iw)) {
for (LeafReaderContext ctx : reader.leaves()) {
Bits liveDocs = ctx.reader().getLiveDocs();
VectorValues vectorValues = ctx.reader().getVectorValues(fieldName);
FloatVectorValues vectorValues = ctx.reader().getFloatVectorValues(fieldName);
if (vectorValues == null) {
continue;
}
@ -1077,7 +1085,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (IndexReader reader = DirectoryReader.open(iw)) {
for (LeafReaderContext ctx : reader.leaves()) {
Bits liveDocs = ctx.reader().getLiveDocs();
VectorValues vectorValues = ctx.reader().getVectorValues(fieldName);
FloatVectorValues vectorValues = ctx.reader().getFloatVectorValues(fieldName);
if (vectorValues == null) {
continue;
}
@ -1109,7 +1117,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
// assert that searchNearestVectors returns the expected number of documents,
// in descending score order
int size = ctx.reader().getVectorValues(fieldName).size();
int size = ctx.reader().getFloatVectorValues(fieldName).size();
int k = random().nextInt(size / 10 + 1) + 1;
if (k > numLiveDocsWithVectors) {
k = numLiveDocsWithVectors;
@ -1182,7 +1190,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
throws IOException {
Document doc = new Document();
if (vector != null) {
doc.add(new KnnVectorField(field, vector, similarityFunction));
doc.add(new KnnFloatVectorField(field, vector, similarityFunction));
}
doc.add(new NumericDocValuesField("sortkey", sortkey));
String idString = Integer.toString(id);
@ -1213,10 +1221,10 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
try (Directory dir = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
Document doc = new Document();
doc.add(new KnnVectorField("v1", randomVector(3), VectorSimilarityFunction.EUCLIDEAN));
doc.add(new KnnFloatVectorField("v1", randomVector(3), VectorSimilarityFunction.EUCLIDEAN));
w.addDocument(doc);
doc.add(new KnnVectorField("v2", randomVector(3), VectorSimilarityFunction.EUCLIDEAN));
doc.add(new KnnFloatVectorField("v2", randomVector(3), VectorSimilarityFunction.EUCLIDEAN));
w.addDocument(doc);
}
@ -1261,14 +1269,15 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
// randomly add a vector field
if (random().nextInt(4) == 3) {
doc.add(
new KnnVectorField(fieldName, new float[4], VectorSimilarityFunction.EUCLIDEAN));
new KnnFloatVectorField(
fieldName, new float[4], VectorSimilarityFunction.EUCLIDEAN));
}
w.addDocument(doc);
}
w.forceMerge(1);
try (IndexReader reader = DirectoryReader.open(w)) {
LeafReader r = getOnlyLeafReader(reader);
VectorValues vectorValues = r.getVectorValues(fieldName);
FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName);
int[] vectorDocs = new int[vectorValues.size() + 1];
int cur = -1;
while (++cur < vectorValues.size() + 1) {
@ -1277,7 +1286,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
assertTrue(vectorDocs[cur] > vectorDocs[cur - 1]);
}
}
vectorValues = r.getVectorValues(fieldName);
vectorValues = r.getFloatVectorValues(fieldName);
cur = -1;
for (int i = 0; i < numdocs; i++) {
// randomly advance to i
@ -1321,7 +1330,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
case FLOAT32 -> {
float[] v = randomVector(dim);
fieldValuesCheckSum += v[0];
doc.add(new KnnVectorField("knn_vector", v, similarityFunction));
doc.add(new KnnFloatVectorField("knn_vector", v, similarityFunction));
}
}
fieldDocCount++;
@ -1355,7 +1364,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
}
case FLOAT32 -> {
for (LeafReaderContext ctx : r.leaves()) {
VectorValues vectorValues = ctx.reader().getVectorValues("knn_vector");
FloatVectorValues vectorValues = ctx.reader().getFloatVectorValues("knn_vector");
if (vectorValues != null) {
docCount += vectorValues.size();
StoredFields storedFields = ctx.reader().storedFields();

6
lucene/test-framework/src/java/org/apache/lucene/tests/index/MergeReaderWrapper.java
View File

@ -29,6 +29,7 @@ import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.LeafMetaData;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
@ -39,7 +40,6 @@ import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.Bits;
@ -218,8 +218,8 @@ class MergeReaderWrapper extends LeafReader {
}
@Override
public VectorValues getVectorValues(String fieldName) throws IOException {
return in.getVectorValues(fieldName);
public FloatVectorValues getFloatVectorValues(String fieldName) throws IOException {
return in.getFloatVectorValues(fieldName);
}
@Override

4
lucene/test-framework/src/java/org/apache/lucene/tests/search/QueryUtils.java
View File

@ -26,6 +26,7 @@ import java.util.Random;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafMetaData;
import org.apache.lucene.index.LeafReader;
@ -40,7 +41,6 @@ import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.BulkScorer;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
@ -225,7 +225,7 @@ public class QueryUtils {
}
@Override
public VectorValues getVectorValues(String field) throws IOException {
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
return null;
}