mirror of https://github.com/apache/lucene.git
Fix file leak in Lucene90VectorWriter (#2331)
Adds a new test class based on BaseIndexFileFormatTestCase
This commit is contained in:
Ignacio Vera
GitHub
parent
aea29464f1
commit
73d4f8cd03
|
|
@ -54,23 +54,25 @@ public final class Lucene90VectorWriter extends VectorWriter {
|
|||
String metaFileName =
|
||||
IndexFileNames.segmentFileName(
|
||||
state.segmentInfo.name, state.segmentSuffix, Lucene90VectorFormat.META_EXTENSION);
|
||||
meta = state.directory.createOutput(metaFileName, state.context);
|
||||
|
||||
String vectorDataFileName =
|
||||
IndexFileNames.segmentFileName(
|
||||
state.segmentInfo.name,
|
||||
state.segmentSuffix,
|
||||
Lucene90VectorFormat.VECTOR_DATA_EXTENSION);
|
||||
vectorData = state.directory.createOutput(vectorDataFileName, state.context);
|
||||
|
||||
String indexDataFileName =
|
||||
IndexFileNames.segmentFileName(
|
||||
state.segmentInfo.name,
|
||||
state.segmentSuffix,
|
||||
Lucene90VectorFormat.VECTOR_INDEX_EXTENSION);
|
||||
vectorIndex = state.directory.createOutput(indexDataFileName, state.context);
|
||||
|
||||
boolean success = false;
|
||||
try {
|
||||
meta = state.directory.createOutput(metaFileName, state.context);
|
||||
vectorData = state.directory.createOutput(vectorDataFileName, state.context);
|
||||
vectorIndex = state.directory.createOutput(indexDataFileName, state.context);
|
||||
|
||||
CodecUtil.writeIndexHeader(
|
||||
meta,
|
||||
Lucene90VectorFormat.META_CODEC_NAME,
|
||||
|
|
@ -89,8 +91,11 @@ public final class Lucene90VectorWriter extends VectorWriter {
|
|||
Lucene90VectorFormat.VERSION_CURRENT,
|
||||
state.segmentInfo.getId(),
|
||||
state.segmentSuffix);
|
||||
} catch (IOException e) {
|
||||
IOUtils.closeWhileHandlingException(this);
|
||||
success = true;
|
||||
} finally {
|
||||
if (success == false) {
|
||||
IOUtils.closeWhileHandlingException(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.codecs.lucene90;
|
||||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.index.BaseVectorFormatTestCase;
|
||||
import org.apache.lucene.util.TestUtil;
|
||||
|
||||
public class TestLucene90VectorFormat extends BaseVectorFormatTestCase {
|
||||
|
||||
@Override
|
||||
protected Codec getCodec() {
|
||||
return TestUtil.getDefaultCodec();
|
||||
}
|
||||
}
|
||||
|
|
@ -22,42 +22,45 @@ import java.io.ByteArrayOutputStream;
|
|||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.VectorFormat;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.Field.Store;
|
||||
import org.apache.lucene.document.FieldType;
|
||||
import org.apache.lucene.document.NumericDocValuesField;
|
||||
import org.apache.lucene.document.StringField;
|
||||
import org.apache.lucene.document.VectorField;
|
||||
import org.apache.lucene.index.VectorValues.SearchStrategy;
|
||||
import org.apache.lucene.search.Sort;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.FSDirectory;
|
||||
import org.apache.lucene.util.Bits;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.lucene.util.TestUtil;
|
||||
import org.apache.lucene.util.VectorUtil;
|
||||
|
||||
/** Test Indexing/IndexWriter with vectors */
|
||||
public class TestVectorValues extends LuceneTestCase {
|
||||
/**
|
||||
* Base class aiming at testing {@link VectorFormat vectors formats}. To test a new format, all you
|
||||
* need is to register a new {@link Codec} which uses it and extend this class and override {@link
|
||||
* #getCodec()}.
|
||||
*
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public abstract class BaseVectorFormatTestCase extends BaseIndexFileFormatTestCase {
|
||||
|
||||
private IndexWriterConfig createIndexWriterConfig() {
|
||||
IndexWriterConfig iwc = newIndexWriterConfig();
|
||||
iwc.setCodec(Codec.forName("Lucene90"));
|
||||
return iwc;
|
||||
@Override
|
||||
protected void addRandomFields(Document doc) {
|
||||
doc.add(new VectorField("v2", randomVector(30), VectorValues.SearchStrategy.NONE));
|
||||
}
|
||||
|
||||
// Suddenly add vectors to an existing field:
|
||||
public void testUpgradeFieldToVectors() throws Exception {
|
||||
try (Directory dir = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(newStringField("f", "foo", Store.NO));
|
||||
doc.add(newStringField("f", "foo", Field.Store.NO));
|
||||
w.addDocument(doc);
|
||||
}
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
|
|
@ -79,7 +82,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
expectThrows(IllegalArgumentException.class, () -> new VectorField("f", null));
|
||||
expectThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> new VectorField("f", new float[1], (SearchStrategy) null));
|
||||
() -> new VectorField("f", new float[1], (VectorValues.SearchStrategy) null));
|
||||
expectThrows(IllegalArgumentException.class, () -> new VectorField("f", new float[0]));
|
||||
expectThrows(
|
||||
IllegalArgumentException.class,
|
||||
|
|
@ -101,12 +104,15 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
public void testFieldCreateFieldType() {
|
||||
expectThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> VectorField.createHnswType(0, SearchStrategy.EUCLIDEAN_HNSW, 16, 16));
|
||||
() -> VectorField.createHnswType(0, VectorValues.SearchStrategy.EUCLIDEAN_HNSW, 16, 16));
|
||||
expectThrows(
|
||||
IllegalArgumentException.class,
|
||||
() ->
|
||||
VectorField.createHnswType(
|
||||
VectorValues.MAX_DIMENSIONS + 1, SearchStrategy.EUCLIDEAN_HNSW, 16, 16));
|
||||
VectorValues.MAX_DIMENSIONS + 1,
|
||||
VectorValues.SearchStrategy.EUCLIDEAN_HNSW,
|
||||
16,
|
||||
16));
|
||||
expectThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> VectorField.createHnswType(VectorValues.MAX_DIMENSIONS + 1, null, 16, 16));
|
||||
|
|
@ -114,14 +120,14 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
IllegalArgumentException.class,
|
||||
() ->
|
||||
VectorField.createHnswType(
|
||||
VectorValues.MAX_DIMENSIONS + 1, SearchStrategy.NONE, 16, 16));
|
||||
VectorValues.MAX_DIMENSIONS + 1, VectorValues.SearchStrategy.NONE, 16, 16));
|
||||
}
|
||||
|
||||
// Illegal schema change tests:
|
||||
|
||||
public void testIllegalDimChangeTwoDocs() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
|
|
@ -141,7 +147,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
|
||||
public void testIllegalSearchStrategyChange() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
|
|
@ -162,13 +168,13 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
|
||||
public void testIllegalDimChangeTwoWriters() throws Exception {
|
||||
try (Directory dir = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
}
|
||||
|
||||
try (IndexWriter w2 = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc2 = new Document();
|
||||
doc2.add(new VectorField("f", new float[1], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
IllegalArgumentException expected =
|
||||
|
|
@ -181,13 +187,13 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
|
||||
public void testIllegalSearchStrategyChangeTwoWriters() throws Exception {
|
||||
try (Directory dir = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
}
|
||||
|
||||
try (IndexWriter w2 = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc2 = new Document();
|
||||
doc2.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.EUCLIDEAN_HNSW));
|
||||
IllegalArgumentException expected =
|
||||
|
|
@ -205,10 +211,10 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
doc.add(new VectorField(fieldName, new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
try (Directory dir = newDirectory();
|
||||
Directory dir2 = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
w.addDocument(doc);
|
||||
}
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, createIndexWriterConfig())) {
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
|
||||
w2.addIndexes(dir);
|
||||
w2.forceMerge(1);
|
||||
try (IndexReader reader = w2.getReader()) {
|
||||
|
|
@ -227,12 +233,12 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
Document doc = new Document();
|
||||
try (Directory dir = newDirectory();
|
||||
Directory dir2 = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
w.addDocument(doc);
|
||||
}
|
||||
doc.add(
|
||||
new VectorField(fieldName, new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, createIndexWriterConfig())) {
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
|
||||
w2.addDocument(doc);
|
||||
w2.addIndexes(dir);
|
||||
w2.forceMerge(1);
|
||||
|
|
@ -254,10 +260,10 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
doc.add(new VectorField(fieldName, vector, VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
try (Directory dir = newDirectory();
|
||||
Directory dir2 = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
w.addDocument(doc);
|
||||
}
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, createIndexWriterConfig())) {
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
|
||||
vector[0] = 1;
|
||||
w2.addDocument(doc);
|
||||
w2.addIndexes(dir);
|
||||
|
|
@ -280,12 +286,12 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
public void testIllegalDimChangeViaAddIndexesDirectory() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
Directory dir2 = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
}
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, createIndexWriterConfig())) {
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[5], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w2.addDocument(doc);
|
||||
|
|
@ -301,12 +307,12 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
public void testIllegalSearchStrategyChangeViaAddIndexesDirectory() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
Directory dir2 = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
}
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, createIndexWriterConfig())) {
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.EUCLIDEAN_HNSW));
|
||||
w2.addDocument(doc);
|
||||
|
|
@ -322,12 +328,12 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
public void testIllegalDimChangeViaAddIndexesCodecReader() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
Directory dir2 = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
}
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, createIndexWriterConfig())) {
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[5], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w2.addDocument(doc);
|
||||
|
|
@ -346,12 +352,12 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
public void testIllegalSearchStrategyChangeViaAddIndexesCodecReader() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
Directory dir2 = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
}
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, createIndexWriterConfig())) {
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.EUCLIDEAN_HNSW));
|
||||
w2.addDocument(doc);
|
||||
|
|
@ -371,12 +377,12 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
public void testIllegalDimChangeViaAddIndexesSlowCodecReader() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
Directory dir2 = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
}
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, createIndexWriterConfig())) {
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[5], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w2.addDocument(doc);
|
||||
|
|
@ -393,14 +399,14 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
public void testIllegalSearchStrategyChangeViaAddIndexesSlowCodecReader() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
Directory dir2 = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
}
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, createIndexWriterConfig())) {
|
||||
try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], SearchStrategy.EUCLIDEAN_HNSW));
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.EUCLIDEAN_HNSW));
|
||||
w2.addDocument(doc);
|
||||
try (DirectoryReader r = DirectoryReader.open(dir)) {
|
||||
IllegalArgumentException expected =
|
||||
|
|
@ -415,7 +421,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
|
||||
public void testIllegalMultipleValues() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
|
|
@ -429,7 +435,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
|
||||
public void testIllegalDimensionTooLarge() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
expectThrows(
|
||||
IllegalArgumentException.class,
|
||||
|
|
@ -448,12 +454,12 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
|
||||
public void testIllegalEmptyVector() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
Exception e =
|
||||
expectThrows(
|
||||
IllegalArgumentException.class,
|
||||
() -> doc.add(new VectorField("f", new float[0], SearchStrategy.NONE)));
|
||||
() -> doc.add(new VectorField("f", new float[0], VectorValues.SearchStrategy.NONE)));
|
||||
assertEquals("cannot index an empty vector", e.getMessage());
|
||||
|
||||
Document doc2 = new Document();
|
||||
|
|
@ -465,7 +471,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
// Write vectors, one segment with default codec, another with SimpleText, then forceMerge
|
||||
public void testDifferentCodecs1() throws Exception {
|
||||
try (Directory dir = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
|
|
@ -491,7 +497,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
}
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("f", new float[4], VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
w.addDocument(doc);
|
||||
|
|
@ -512,9 +518,9 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
|
||||
public void testDeleteAllVectorDocs() throws Exception {
|
||||
try (Directory dir = newDirectory();
|
||||
IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("id", "0", Store.NO));
|
||||
doc.add(new StringField("id", "0", Field.Store.NO));
|
||||
doc.add(
|
||||
new VectorField(
|
||||
"v", new float[] {2, 3, 5}, VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
|
|
@ -535,9 +541,9 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
|
||||
public void testVectorFieldMissingFromOneSegment() throws Exception {
|
||||
try (Directory dir = FSDirectory.open(createTempDir());
|
||||
IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("id", "0", Store.NO));
|
||||
doc.add(new StringField("id", "0", Field.Store.NO));
|
||||
doc.add(
|
||||
new VectorField(
|
||||
"v0", new float[] {2, 3, 5}, VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
|
|
@ -568,7 +574,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
random().nextInt(VectorValues.SearchStrategy.values().length)];
|
||||
}
|
||||
try (Directory dir = newDirectory();
|
||||
RandomIndexWriter w = new RandomIndexWriter(random(), dir, createIndexWriterConfig())) {
|
||||
RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig())) {
|
||||
for (int i = 0; i < numDocs; i++) {
|
||||
Document doc = new Document();
|
||||
for (int field = 0; field < numFields; field++) {
|
||||
|
|
@ -610,7 +616,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
String fieldName = "field";
|
||||
float[] v = {0};
|
||||
try (Directory dir = newDirectory();
|
||||
IndexWriter iw = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc1 = new Document();
|
||||
doc1.add(new VectorField(fieldName, v, VectorValues.SearchStrategy.EUCLIDEAN_HNSW));
|
||||
v[0] = 1;
|
||||
|
|
@ -637,7 +643,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
}
|
||||
|
||||
public void testSortedIndex() throws Exception {
|
||||
IndexWriterConfig iwc = createIndexWriterConfig();
|
||||
IndexWriterConfig iwc = newIndexWriterConfig();
|
||||
iwc.setIndexSort(new Sort(new SortField("sortkey", SortField.Type.INT)));
|
||||
String fieldName = "field";
|
||||
try (Directory dir = newDirectory();
|
||||
|
|
@ -675,13 +681,15 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
float[] v = new float[] {1};
|
||||
doc.add(new VectorField("field1", v, SearchStrategy.EUCLIDEAN_HNSW));
|
||||
doc.add(new VectorField("field2", new float[] {1, 2, 3}, SearchStrategy.NONE));
|
||||
doc.add(new VectorField("field1", v, VectorValues.SearchStrategy.EUCLIDEAN_HNSW));
|
||||
doc.add(new VectorField("field2", new float[] {1, 2, 3}, VectorValues.SearchStrategy.NONE));
|
||||
iw.addDocument(doc);
|
||||
v[0] = 2;
|
||||
iw.addDocument(doc);
|
||||
doc = new Document();
|
||||
doc.add(new VectorField("field3", new float[] {1, 2, 3}, SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
doc.add(
|
||||
new VectorField(
|
||||
"field3", new float[] {1, 2, 3}, VectorValues.SearchStrategy.DOT_PRODUCT_HNSW));
|
||||
iw.addDocument(doc);
|
||||
iw.forceMerge(1);
|
||||
try (IndexReader reader = iw.getReader()) {
|
||||
|
|
@ -721,7 +729,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
* consistently.
|
||||
*/
|
||||
public void testRandom() throws Exception {
|
||||
IndexWriterConfig iwc = createIndexWriterConfig();
|
||||
IndexWriterConfig iwc = newIndexWriterConfig();
|
||||
if (random().nextBoolean()) {
|
||||
iwc.setIndexSort(new Sort(new SortField("sortkey", SortField.Type.INT)));
|
||||
}
|
||||
|
|
@ -742,9 +750,9 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
if (random().nextBoolean() && values[i] != null) {
|
||||
// sometimes use a shared scratch array
|
||||
System.arraycopy(values[i], 0, scratch, 0, scratch.length);
|
||||
add(iw, fieldName, i, scratch, SearchStrategy.NONE);
|
||||
add(iw, fieldName, i, scratch, VectorValues.SearchStrategy.NONE);
|
||||
} else {
|
||||
add(iw, fieldName, i, values[i], SearchStrategy.NONE);
|
||||
add(iw, fieldName, i, values[i], VectorValues.SearchStrategy.NONE);
|
||||
}
|
||||
if (random().nextInt(10) == 2) {
|
||||
// sometimes delete a random document
|
||||
|
|
@ -817,7 +825,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
values[i] = value;
|
||||
id2value[id] = value;
|
||||
id2ord[id] = i;
|
||||
add(iw, fieldName, id, value, SearchStrategy.EUCLIDEAN_HNSW);
|
||||
add(iw, fieldName, id, value, VectorValues.SearchStrategy.EUCLIDEAN_HNSW);
|
||||
}
|
||||
try (IndexReader reader = iw.getReader()) {
|
||||
for (LeafReaderContext ctx : reader.leaves()) {
|
||||
|
|
@ -850,14 +858,18 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
}
|
||||
|
||||
private void add(
|
||||
IndexWriter iw, String field, int id, float[] vector, SearchStrategy searchStrategy)
|
||||
IndexWriter iw,
|
||||
String field,
|
||||
int id,
|
||||
float[] vector,
|
||||
VectorValues.SearchStrategy searchStrategy)
|
||||
throws IOException {
|
||||
add(iw, field, id, random().nextInt(100), vector, searchStrategy);
|
||||
}
|
||||
|
||||
private void add(IndexWriter iw, String field, int id, int sortkey, float[] vector)
|
||||
throws IOException {
|
||||
add(iw, field, id, sortkey, vector, SearchStrategy.NONE);
|
||||
add(iw, field, id, sortkey, vector, VectorValues.SearchStrategy.NONE);
|
||||
}
|
||||
|
||||
private void add(
|
||||
|
|
@ -866,7 +878,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
int id,
|
||||
int sortkey,
|
||||
float[] vector,
|
||||
SearchStrategy searchStrategy)
|
||||
VectorValues.SearchStrategy searchStrategy)
|
||||
throws IOException {
|
||||
Document doc = new Document();
|
||||
if (vector != null) {
|
||||
|
|
@ -890,7 +902,7 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
|
||||
public void testCheckIndexIncludesVectors() throws Exception {
|
||||
try (Directory dir = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
Document doc = new Document();
|
||||
doc.add(new VectorField("v1", randomVector(3), VectorValues.SearchStrategy.NONE));
|
||||
w.addDocument(doc);
|
||||
|
|
@ -924,14 +936,14 @@ public class TestVectorValues extends LuceneTestCase {
|
|||
|
||||
public void testAdvance() throws Exception {
|
||||
try (Directory dir = newDirectory()) {
|
||||
try (IndexWriter w = new IndexWriter(dir, createIndexWriterConfig())) {
|
||||
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
|
||||
int numdocs = atLeast(1500);
|
||||
String fieldName = "field";
|
||||
for (int i = 0; i < numdocs; i++) {
|
||||
Document doc = new Document();
|
||||
// randomly add a vector field
|
||||
if (random().nextInt(4) == 3) {
|
||||
doc.add(new VectorField(fieldName, new float[4], SearchStrategy.NONE));
|
||||
doc.add(new VectorField(fieldName, new float[4], VectorValues.SearchStrategy.NONE));
|
||||
}
|
||||
w.addDocument(doc);
|
||||
}
|
||||
Loading…
Reference in New Issue