From 972dbfb17440f8f6a0708b2353913b7c822e94e9 Mon Sep 17 00:00:00 2001 From: Anh Dung Bui Date: Fri, 8 Nov 2024 17:31:28 +0900 Subject: [PATCH] Fix build error --- .../lucene95/Lucene95HnswVectorsWriter.java | 4 ++-- .../benchmark/jmh/VectorScorerBenchmark.java | 2 +- .../lucene99/Lucene99FlatVectorsWriter.java | 4 ++-- .../Lucene99ScalarQuantizedVectorsWriter.java | 2 +- .../org/apache/lucene/store/IndexInput.java | 17 +++++++++++++++++ .../Lucene99MemorySegmentByteVectorScorer.java | 9 ++++++++- ...99MemorySegmentByteVectorScorerSupplier.java | 6 +++++- .../lucene/store/MemorySegmentIndexInput.java | 6 ++++++ .../codecs/hnsw/TestFlatVectorScorer.java | 9 ++------- .../vectorization/TestVectorScorer.java | 4 ++-- 10 files changed, 46 insertions(+), 17 deletions(-) diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java index c855d8f5e07..67ebed95bcd 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java @@ -450,7 +450,7 @@ public final class Lucene95HnswVectorsWriter extends KnnVectorsWriter { new OffHeapByteVectorValues.DenseOffHeapVectorValues( fieldInfo.getVectorDimension(), docsWithField.cardinality(), - vectorDataInput, + vectorDataInput.toRandomAccessInput(), byteSize, defaultFlatVectorScorer, fieldInfo.getVectorSimilarityFunction())); @@ -462,7 +462,7 @@ public final class Lucene95HnswVectorsWriter extends KnnVectorsWriter { new OffHeapFloatVectorValues.DenseOffHeapVectorValues( fieldInfo.getVectorDimension(), docsWithField.cardinality(), - vectorDataInput, + vectorDataInput.toRandomAccessInput(), byteSize, defaultFlatVectorScorer, fieldInfo.getVectorSimilarityFunction())); diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerBenchmark.java index 0a4da1f4886..512908345c4 100644 --- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerBenchmark.java +++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/VectorScorerBenchmark.java @@ -98,7 +98,7 @@ public class VectorScorerBenchmark { static KnnVectorValues vectorValues( int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException { return new OffHeapByteVectorValues.DenseOffHeapVectorValues( - dims, size, in.slice("test", 0, in.length()), dims, new ThrowingFlatVectorScorer(), sim); + dims, size, in.toRandomAccessInput(), dims, new ThrowingFlatVectorScorer(), sim); } static final class ThrowingFlatVectorScorer implements FlatVectorsScorer { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java index b731e758b7a..9a323bc7ba6 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java @@ -303,7 +303,7 @@ public final class Lucene99FlatVectorsWriter extends FlatVectorsWriter { new OffHeapByteVectorValues.DenseOffHeapVectorValues( fieldInfo.getVectorDimension(), docsWithField.cardinality(), - finalVectorDataInput, + finalVectorDataInput.toRandomAccessInput(), fieldInfo.getVectorDimension() * Byte.BYTES, vectorsScorer, fieldInfo.getVectorSimilarityFunction())); @@ -313,7 +313,7 @@ public final class Lucene99FlatVectorsWriter extends FlatVectorsWriter { new OffHeapFloatVectorValues.DenseOffHeapVectorValues( fieldInfo.getVectorDimension(), docsWithField.cardinality(), - finalVectorDataInput, + finalVectorDataInput.toRandomAccessInput(), fieldInfo.getVectorDimension() * Float.BYTES, vectorsScorer, fieldInfo.getVectorSimilarityFunction())); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java index 1a30b5271cd..583cafcd8b2 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java @@ -524,7 +524,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite compress, fieldInfo.getVectorSimilarityFunction(), vectorsScorer, - quantizationDataInput))); + quantizationDataInput.toRandomAccessInput()))); } finally { if (success == false) { IOUtils.closeWhileHandlingException(tempQuantizedVectorData, quantizationDataInput); diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java index 5eaeeac6213..26947542b60 100644 --- a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java @@ -152,6 +152,14 @@ public abstract class IndexInput extends DataInput implements Closeable { } } + /** Convert this IndexInput a RandomAccessInput. */ + public RandomAccessInput toRandomAccessInput() throws IOException { + if (this instanceof RandomAccessInput) { + return (RandomAccessInput) this; + } + return randomAccessSlice(0, length()); + } + /** * Creates a random-access slice of this index input, with the given offset and length. * @@ -214,6 +222,15 @@ public abstract class IndexInput extends DataInput implements Closeable { slice.prefetch(offset, length); } + @Override + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException e) { + throw new Error("This cannot happen: Failing to clone RandomAccessInput", e); + } + } + @Override public String toString() { return "RandomAccessInput(" + IndexInput.this.toString() + ")"; diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java index b65f1e57092..9a8008cd86b 100644 --- a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java +++ b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.FilterIndexInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.MemorySegmentAccessInput; +import org.apache.lucene.store.RandomAccessInput; import org.apache.lucene.util.hnsw.RandomVectorScorer; abstract sealed class Lucene99MemorySegmentByteVectorScorer @@ -40,8 +41,14 @@ abstract sealed class Lucene99MemorySegmentByteVectorScorer * returned. */ public static Optional create( - VectorSimilarityFunction type, IndexInput input, KnnVectorValues values, byte[] queryVector) { + VectorSimilarityFunction type, + RandomAccessInput slice, + KnnVectorValues values, + byte[] queryVector) { assert values instanceof ByteVectorValues; + if (!(slice instanceof IndexInput input)) { + return Optional.empty(); + } input = FilterIndexInput.unwrapOnlyTest(input); if (!(input instanceof MemorySegmentAccessInput msInput)) { return Optional.empty(); diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java index 02c71561122..22012d89ac1 100644 --- a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java +++ b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.FilterIndexInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.MemorySegmentAccessInput; +import org.apache.lucene.store.RandomAccessInput; import org.apache.lucene.util.hnsw.RandomVectorScorer; import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier; @@ -42,8 +43,11 @@ public abstract sealed class Lucene99MemorySegmentByteVectorScorerSupplier * optional is returned. */ static Optional create( - VectorSimilarityFunction type, IndexInput input, KnnVectorValues values) { + VectorSimilarityFunction type, RandomAccessInput slice, KnnVectorValues values) { assert values instanceof ByteVectorValues; + if (!(slice instanceof IndexInput input)) { + return Optional.empty(); + } input = FilterIndexInput.unwrapOnlyTest(input); if (!(input instanceof MemorySegmentAccessInput msInput)) { return Optional.empty(); diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java index 832fa5f98e6..a1e605a51df 100644 --- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java +++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java @@ -232,6 +232,12 @@ abstract class MemorySegmentIndexInput extends IndexInput } } + @Override + public void readFloats(long pos, float[] dst, int offset, int len) throws IOException { + seek(pos); + readFloats(dst, offset, len); + } + @Override public void readFloats(float[] dst, int offset, int length) throws IOException { try { diff --git a/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java index 6fe9a685e1b..4e951ad7539 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/hnsw/TestFlatVectorScorer.java @@ -178,18 +178,13 @@ public class TestFlatVectorScorer extends LuceneTestCase { ByteVectorValues byteVectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException { return new OffHeapByteVectorValues.DenseOffHeapVectorValues( - dims, size, in.slice("byteValues", 0, in.length()), dims, flatVectorsScorer, sim); + dims, size, in.toRandomAccessInput(), dims, flatVectorsScorer, sim); } FloatVectorValues floatVectorValues( int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException { return new OffHeapFloatVectorValues.DenseOffHeapVectorValues( - dims, - size, - in.slice("floatValues", 0, in.length()), - dims * Float.BYTES, - flatVectorsScorer, - sim); + dims, size, in.toRandomAccessInput(), dims * Float.BYTES, flatVectorsScorer, sim); } /** Concatenates float arrays as byte[]. */ diff --git a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorScorer.java b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorScorer.java index bc3b6813a5b..122c71afbe5 100644 --- a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorScorer.java +++ b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestVectorScorer.java @@ -381,13 +381,13 @@ public class TestVectorScorer extends LuceneTestCase { KnnVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException { return new OffHeapByteVectorValues.DenseOffHeapVectorValues( - dims, size, in.slice("byteValues", 0, in.length()), dims, MEMSEG_SCORER, sim); + dims, size, in.toRandomAccessInput(), dims, MEMSEG_SCORER, sim); } KnnVectorValues floatVectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException { return new OffHeapFloatVectorValues.DenseOffHeapVectorValues( - dims, size, in.slice("floatValues", 0, in.length()), dims, MEMSEG_SCORER, sim); + dims, size, in.toRandomAccessInput(), dims, MEMSEG_SCORER, sim); } // creates the vector based on the given ordinal, which is reproducible given the ord and dims