Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename FieldInfo(s)#hasVectors to hasTermVectors. (#13759) 

This will help remove confusion between dense numeric vectors and term
vectors.
This commit is contained in:
Adrien Grand 2024-09-11 13:19:56 +02:00 committed by GitHub
parent 02b37670e4
commit 3da48d0403
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
33 changed files with 52 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lucene/MIGRATE.md
View File

@ -262,6 +262,11 @@ ConcurrentMergeScheduler now disables auto I/O throttling by default. There is s
happening at the CPU level, since ConcurrentMergeScheduler has a maximum number of threads it can
use, which is only a fraction of the total number of threads of the host by default.
### FieldInfos#hasVectors and FieldInfo#hasVectors renamed to hasTermVectors
To reduce confusion between term vectors and numeric vectors, `hasVectors` has been renamed to
`hasTermVectors`.
## Migration from Lucene 9.0 to Lucene 9.1
### Test framework package migration and module (LUCENE-10301)
@ -860,4 +865,4 @@ method now supports an additional 4th and last argument to optionally enable cre
`TotalHitCountCollectorManager` now requires that an array of `LeafSlice`s, retrieved via `IndexSearcher#getSlices`,
is provided to its constructor. Depending on whether segment partitions are present among slices, the manager can
optimize the type of collectors it creates and exposes via `newCollector`.
optimize the type of collectors it creates and exposes via `newCollector`.

2
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java
View File

@ -347,7 +347,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
output.writeVInt(fi.number);
byte bits = 0x0;
if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
if (fi.hasTermVectors()) bits |= STORE_TERMVECTOR;
if (fi.omitsNorms()) bits |= OMIT_NORMS;
if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;

2
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java
View File

@ -333,7 +333,7 @@ public final class Lucene90FieldInfosFormat extends FieldInfosFormat {
output.writeVInt(fi.number);
byte bits = 0x0;
if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
if (fi.hasTermVectors()) bits |= STORE_TERMVECTOR;
if (fi.omitsNorms()) bits |= OMIT_NORMS;
if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;

2
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
View File

@ -268,7 +268,7 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, STORETV);
SimpleTextUtil.write(out, Boolean.toString(fi.hasVectors()), scratch);
SimpleTextUtil.write(out, Boolean.toString(fi.hasTermVectors()), scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, PAYLOADS);

2
lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java
View File

@ -399,7 +399,7 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {
output.writeVInt(fi.number);
byte bits = 0x0;
if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
if (fi.hasTermVectors()) bits |= STORE_TERMVECTOR;
if (fi.omitsNorms()) bits |= OMIT_NORMS;
if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;

4
lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java
View File

@ -107,7 +107,7 @@ final class PerFieldMergeState {
for (FieldInfo fi : src) {
if (this.filteredNames.contains(fi.name)) {
this.filtered.add(fi);
hasVectors |= fi.hasVectors();
hasVectors |= fi.hasTermVectors();
hasPostings |= fi.getIndexOptions() != IndexOptions.NONE;
hasProx |= fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
hasFreq |= fi.getIndexOptions() != IndexOptions.DOCS;
@ -172,7 +172,7 @@ final class PerFieldMergeState {
}
@Override
public boolean hasVectors() {
public boolean hasTermVectors() {
return filteredHasVectors;
}

2
lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java
View File

@ -66,7 +66,7 @@ public class DocumentStoredFieldVisitor extends StoredFieldVisitor {
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
final FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.setStoreTermVectors(fieldInfo.hasVectors());
ft.setStoreTermVectors(fieldInfo.hasTermVectors());
ft.setOmitNorms(fieldInfo.omitsNorms());
ft.setIndexOptions(fieldInfo.getIndexOptions());
doc.add(

2
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -3836,7 +3836,7 @@ public final class CheckIndex implements Closeable {
// Make sure FieldInfo thinks this field is vector'd:
final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
if (fieldInfo.hasVectors() == false) {
if (fieldInfo.hasTermVectors() == false) {
throw new CheckIndexException(
"docID="
+ j

2
lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
View File

@ -487,7 +487,7 @@ final class DocumentsWriterPerThread implements Accountable, Lock {
infoStream.message(
"DWPT",
"new segment has "
+ (flushState.fieldInfos.hasVectors() ? "vectors" : "no vectors")
+ (flushState.fieldInfos.hasTermVectors() ? "vectors" : "no vectors")
+ "; "
+ (flushState.fieldInfos.hasNorms() ? "norms" : "no norms")
+ "; "

2
lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
View File

@ -641,7 +641,7 @@ public final class FieldInfo {
}
/** Returns true if any term vectors exist for this field. */
public boolean hasVectors() {
public boolean hasTermVectors() {
return storeTermVector;
}

20
lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
View File

@ -52,7 +52,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
private final boolean hasProx;
private final boolean hasPayloads;
private final boolean hasOffsets;
private final boolean hasVectors;
private final boolean hasTermVectors;
private final boolean hasNorms;
private final boolean hasDocValues;
private final boolean hasPointValues;
@ -73,7 +73,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
* as the backing structure.
*/
public FieldInfos(FieldInfo[] infos) {
boolean hasVectors = false;
boolean hasTermVectors = false;
boolean hasPostings = false;
boolean hasProx = false;
boolean hasPayloads = false;
@ -111,7 +111,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
+ info.name);
}
hasVectors |= info.hasVectors();
hasTermVectors |= info.hasTermVectors();
hasPostings |= info.getIndexOptions() != IndexOptions.NONE;
hasProx |= info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
hasFreq |= info.getIndexOptions() != IndexOptions.DOCS;
@ -139,7 +139,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
}
this.hasVectors = hasVectors;
this.hasTermVectors = hasTermVectors;
this.hasPostings = hasPostings;
this.hasProx = hasProx;
this.hasPayloads = hasPayloads;
@ -275,9 +275,9 @@ public class FieldInfos implements Iterable<FieldInfo> {
return hasOffsets;
}
/** Returns true if any fields have vectors */
public boolean hasVectors() {
return hasVectors;
/** Returns true if any fields have term vectors */
public boolean hasTermVectors() {
return hasTermVectors;
}
/** Returns true if any fields have norms */
@ -441,7 +441,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
fieldNumber,
fi.getIndexOptions(),
fi.getIndexOptions() != IndexOptions.NONE
? new IndexOptionsProperties(fi.hasVectors(), fi.omitsNorms())
? new IndexOptionsProperties(fi.hasTermVectors(), fi.omitsNorms())
: null,
fi.getDocValuesType(),
fi.hasDocValuesSkipIndex(),
@ -517,7 +517,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
verifySameIndexOptions(fieldName, currentOpts, fi.getIndexOptions());
if (currentOpts != IndexOptions.NONE) {
boolean curStoreTermVector = fieldProperties.indexOptionsProperties.storeTermVectors;
verifySameStoreTermVectors(fieldName, curStoreTermVector, fi.hasVectors());
verifySameStoreTermVectors(fieldName, curStoreTermVector, fi.hasTermVectors());
boolean curOmitNorms = fieldProperties.indexOptionsProperties.omitNorms;
verifySameOmitNorms(fieldName, curOmitNorms, fi.omitsNorms());
}
@ -792,7 +792,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
new FieldInfo(
fi.getName(),
fieldNumber,
fi.hasVectors(),
fi.hasTermVectors(),
fi.omitsNorms(),
fi.hasPayloads(),
fi.getIndexOptions(),

2
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -5306,7 +5306,7 @@ public class IndexWriter
("merge codec=" + codec)
+ (" maxDoc=" + merge.info.info.maxDoc())
+ ("; merged segment has "
+ (mergeState.mergeFieldInfos.hasVectors() ? "vectors" : "no vectors"))
+ (mergeState.mergeFieldInfos.hasTermVectors() ? "vectors" : "no vectors"))
+ ("; " + (mergeState.mergeFieldInfos.hasNorms() ? "norms" : "no norms"))
+ ("; "
+ (mergeState.mergeFieldInfos.hasDocValues() ? "docValues" : "no docValues"))

4
lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
View File

@ -1144,7 +1144,7 @@ final class IndexingChain implements Accountable {
// segment
norms = new NormValuesWriter(fieldInfo, bytesUsed);
}
if (fieldInfo.hasVectors()) {
if (fieldInfo.hasTermVectors()) {
termVectorsWriter.setHasVectors();
}
}
@ -1558,7 +1558,7 @@ final class IndexingChain implements Accountable {
void assertSameSchema(FieldInfo fi) {
assertSame("index options", fi.getIndexOptions(), indexOptions);
assertSame("omit norms", fi.omitsNorms(), omitNorms);
assertSame("store term vector", fi.hasVectors(), storeTermVector);
assertSame("store term vector", fi.hasTermVectors(), storeTermVector);
assertSame("doc values type", fi.getDocValuesType(), docValuesType);
assertSame("doc values skip index", fi.hasDocValuesSkipIndex(), hasDocValuesSkipIndex);
assertSame(

2
lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
View File

@ -158,7 +158,7 @@ public class ParallelLeafReader extends LeafReader {
// only add these if the reader responsible for that field name is the current:
// TODO consider populating 1st leaf with vectors even if the field name has been seen on
// a previous leaf
if (fieldInfo.hasVectors()) {
if (fieldInfo.hasTermVectors()) {
tvFieldToReader.put(fieldInfo.name, reader);
}
// TODO consider populating 1st leaf with terms even if the field name has been seen on a

2
lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
View File

@ -708,7 +708,7 @@ final class ReadersAndUpdates {
return new FieldInfo(
fi.name,
fieldNumber,
fi.hasVectors(),
fi.hasTermVectors(),
fi.omitsNorms(),
fi.hasPayloads(),
fi.getIndexOptions(),

2
lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
View File

@ -117,7 +117,7 @@ final class SegmentCoreReaders {
.storedFieldsFormat()
.fieldsReader(cfsDir, si.info, coreFieldInfos, context);
if (coreFieldInfos.hasVectors()) { // open term vector files only as needed
if (coreFieldInfos.hasTermVectors()) { // open term vector files only as needed
termVectorsReaderOrig =
si.info
.getCodec()

2
lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
View File

@ -176,7 +176,7 @@ final class SegmentMerger {
numMerged);
}
if (mergeState.mergeFieldInfos.hasVectors()) {
if (mergeState.mergeFieldInfos.hasTermVectors()) {
mergingTasks.add(
() -> {
mergeWithLogging(this::mergeTermVectors, "term vectors");

2
lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
View File

@ -290,7 +290,7 @@ public class TestConsistentFieldNumbers extends LuceneTestCase {
for (FieldInfo fi : fis) {
Field expected = getField(Integer.parseInt(fi.name));
assertEquals(expected.fieldType().indexOptions(), fi.getIndexOptions());
assertEquals(expected.fieldType().storeTermVectors(), fi.hasVectors());
assertEquals(expected.fieldType().storeTermVectors(), fi.hasTermVectors());
}
}

2
lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
View File

@ -269,7 +269,7 @@ public class TestDirectoryReader extends LuceneTestCase {
} else {
notIndexedFieldNames.add(name);
}
if (fieldInfo.hasVectors()) {
if (fieldInfo.hasTermVectors()) {
tvFieldNames.add(name);
}
}

2
lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
View File

@ -1503,7 +1503,7 @@ public class TestIndexWriter extends LuceneTestCase {
DirectoryReader r0 = DirectoryReader.open(dir);
for (LeafReaderContext ctx : r0.leaves()) {
SegmentReader sr = (SegmentReader) ctx.reader();
assertFalse(sr.getFieldInfos().hasVectors());
assertFalse(sr.getFieldInfos().hasTermVectors());
}
r0.close();

2
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
View File

@ -1447,7 +1447,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
assertTrue(reader.numDocs() > 0);
SegmentInfos.readLatestCommit(dir);
for (LeafReaderContext context : reader.leaves()) {
assertFalse(context.reader().getFieldInfos().hasVectors());
assertFalse(context.reader().getFieldInfos().hasTermVectors());
}
reader.close();
dir.close();

2
lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
View File

@ -138,7 +138,7 @@ public class TestSegmentMerger extends LuceneTestCase {
int tvCount = 0;
for (FieldInfo fieldInfo : mergedReader.getFieldInfos()) {
if (fieldInfo.hasVectors()) {
if (fieldInfo.hasTermVectors()) {
tvCount++;
}
}

2
lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
View File

@ -90,7 +90,7 @@ public class TestSegmentReader extends LuceneTestCase {
} else {
notIndexedFieldNames.add(name);
}
if (fieldInfo.hasVectors()) {
if (fieldInfo.hasTermVectors()) {
tvFieldNames.add(name);
} else if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
noTVFieldNames.add(name);

2
lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java
View File

@ -190,7 +190,7 @@ public class TestTermVectorsReader extends LuceneTestCase {
DirectoryReader reader = DirectoryReader.open(dir);
for (LeafReaderContext ctx : reader.leaves()) {
SegmentReader sr = (SegmentReader) ctx.reader();
assertTrue(sr.getFieldInfos().hasVectors());
assertTrue(sr.getFieldInfos().hasTermVectors());
}
reader.close();
}

6
lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java
View File

@ -684,7 +684,7 @@ public class UnifiedHighlighter {
* <li>If there's a field info it has {@link
* IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS} then {@link OffsetSource#POSTINGS}
* is returned.
* <li>If there's a field info and {@link FieldInfo#hasVectors()} then {@link
* <li>If there's a field info and {@link FieldInfo#hasTermVectors()} then {@link
* OffsetSource#TERM_VECTORS} is returned (note we can't check here if the TV has offsets;
* if there isn't then an exception will get thrown down the line).
* <li>Fall-back: {@link OffsetSource#ANALYSIS} is returned.
@ -698,11 +698,11 @@ public class UnifiedHighlighter {
FieldInfo fieldInfo = getFieldInfo(field);
if (fieldInfo != null) {
if (fieldInfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
return fieldInfo.hasVectors()
return fieldInfo.hasTermVectors()
? OffsetSource.POSTINGS_WITH_TERM_VECTORS
: OffsetSource.POSTINGS;
}
if (fieldInfo.hasVectors()) { // unfortunately we can't also check if the TV has offsets
if (fieldInfo.hasTermVectors()) { // unfortunately we can't also check if the TV has offsets
return OffsetSource.TERM_VECTORS;
}
}

2
lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
View File

@ -187,7 +187,7 @@ public abstract class BaseFragmentsBuilder implements FragmentsBuilder {
public void stringField(FieldInfo fieldInfo, String value) {
Objects.requireNonNull(value, "String value should not be null");
FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.setStoreTermVectors(fieldInfo.hasVectors());
ft.setStoreTermVectors(fieldInfo.hasTermVectors());
fields.add(new Field(fieldInfo.name, value, ft));
}

2
lucene/luke/src/java/org/apache/lucene/luke/models/documents/DocumentField.java
View File

@ -73,7 +73,7 @@ public final class DocumentField {
dfield.name = finfo.name;
dfield.idxOptions = finfo.getIndexOptions();
dfield.hasTermVectors = finfo.hasVectors();
dfield.hasTermVectors = finfo.hasTermVectors();
dfield.hasPayloads = finfo.hasPayloads();
dfield.hasNorms = finfo.hasNorms();

2
lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
View File

@ -836,7 +836,7 @@ public class MemoryIndex {
new FieldInfo(
info.fieldInfo.name,
info.fieldInfo.number,
info.fieldInfo.hasVectors(),
info.fieldInfo.hasTermVectors(),
info.fieldInfo.hasPayloads(),
info.fieldInfo.hasPayloads(),
info.fieldInfo.getIndexOptions(),

2
lucene/misc/src/test/org/apache/lucene/misc/document/TestLazyDocument.java
View File

@ -210,7 +210,7 @@ public class TestLazyDocument extends LuceneTestCase {
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
final FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.setStoreTermVectors(fieldInfo.hasVectors());
ft.setStoreTermVectors(fieldInfo.hasTermVectors());
ft.setOmitNorms(fieldInfo.omitsNorms());
ft.setIndexOptions(fieldInfo.getIndexOptions());
Objects.requireNonNull(value, "String value should not be null");

2
lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/IDVersionPostingsWriter.java
View File

@ -71,7 +71,7 @@ final class IDVersionPostingsWriter extends PushPostingsWriterBase {
// LUCENE-5693: because CheckIndex cross-checks term vectors with postings even for deleted
// docs, and because our PF only indexes the
// non-deleted documents on flush, CheckIndex will see this as corruption:
if (fieldInfo.hasVectors()) {
if (fieldInfo.hasTermVectors()) {
throw new IllegalArgumentException(
"field cannot index term vectors: CheckIndex will report this as index corruption");
}

4
lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java
View File

@ -87,7 +87,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
assertFalse(infos2.fieldInfo("field").getDocValuesType() != DocValuesType.NONE);
assertFalse(infos2.fieldInfo("field").omitsNorms());
assertFalse(infos2.fieldInfo("field").hasPayloads());
assertFalse(infos2.fieldInfo("field").hasVectors());
assertFalse(infos2.fieldInfo("field").hasTermVectors());
assertEquals(0, infos2.fieldInfo("field").getPointDimensionCount());
assertEquals(0, infos2.fieldInfo("field").getVectorDimension());
assertFalse(infos2.fieldInfo("field").isSoftDeletesField());
@ -418,7 +418,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
assertEquals(expected.getIndexOptions(), actual.getIndexOptions());
assertEquals(expected.hasNorms(), actual.hasNorms());
assertEquals(expected.hasPayloads(), actual.hasPayloads());
assertEquals(expected.hasVectors(), actual.hasVectors());
assertEquals(expected.hasTermVectors(), actual.hasTermVectors());
assertEquals(expected.omitsNorms(), actual.omitsNorms());
assertEquals(expected.getDocValuesGen(), actual.getDocValuesGen());
}

2
lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java
View File

@ -352,7 +352,7 @@ public abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
new FieldInfo(
proto.name,
proto.number,
proto.hasVectors(),
proto.hasTermVectors(),
proto.omitsNorms(),
proto.hasPayloads(),
proto.getIndexOptions(),

2
lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java
View File

@ -98,7 +98,7 @@ public class MismatchedLeafReader extends FilterLeafReader {
new FieldInfo(
oldInfo.name, // name
i, // number
oldInfo.hasVectors(), // storeTermVector
oldInfo.hasTermVectors(), // storeTermVector
oldInfo.omitsNorms(), // omitNorms
oldInfo.hasPayloads(), // storePayloads
oldInfo.getIndexOptions(), // indexOptions