Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GITHUB#11911: improve checkindex to be more thorough for vectors (#11916) 

search every N docs to get close to 64 tests
This commit is contained in:
Benjamin Trent 2022-11-10 16:45:47 -05:00 committed by GitHub
parent e9ef61ba39
commit 3a506ec87a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 31 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/CHANGES.txt
View File

@ -187,6 +187,10 @@ Build
======================== Lucene 9.4.2 =======================
Improvements
---------------------
* GITHUB#11916: improve checkindex to be more thorough for vectors. (Ben Trent)
Bug Fixes
---------------------
* GITHUB#11905: Fix integer overflow when seeking the vector index for connections in a single segment.

33
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -55,11 +55,7 @@ import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldExistsQuery;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.*;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@ -2593,8 +2589,33 @@ public final class CheckIndex implements Closeable {
status.totalKnnVectorFields++;
int docCount = 0;
final Bits bits = reader.getLiveDocs();
int everyNdoc = Math.max(values.size() / 64, 1);
while (values.nextDoc() != NO_MORE_DOCS) {
int valueLength = values.vectorValue().length;
float[] vectorValue = values.vectorValue();
// search the first maxNumSearches vectors to exercise the graph
if (values.docID() % everyNdoc == 0) {
TopDocs docs =
reader
.getVectorReader()
.search(fieldInfo.name, vectorValue, 10, bits, Integer.MAX_VALUE);
if (docs.scoreDocs.length == 0) {
throw new CheckIndexException(
"Field \"" + fieldInfo.name + "\" failed to search k nearest neighbors");
}
if (bits != null) {
for (ScoreDoc doc : docs.scoreDocs) {
if (bits.get(doc.doc) == false) {
throw new CheckIndexException(
"Searching Field \""
+ fieldInfo.name
+ "\" matched deleted doc="
+ doc.doc);
}
}
}
}
int valueLength = vectorValue.length;
if (valueLength != dimension) {
throw new CheckIndexException(
"Field \""