Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8045: ParallelLeafReader should correctly report FieldInfo.dvGen

This commit is contained in:
Alan Woodward 2017-11-09 13:31:20 +00:00
parent b03c7249d3
commit 77ea211096
4 changed files with 39 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -58,6 +58,9 @@ Bug Fixes
Previously bogus numbers were used, and many similarities would
completely degrade. (Robert Muir, Adrien Grand)
* LUCENE-8045: ParallelLeafReader did not correctly report FieldInfo.dvGen
(Alan Woodward)
Optimizations
* LUCENE-8018: Smaller FieldInfos memory footprint by not retaining unnecessary

12
lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
View File

@ -403,7 +403,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber,
boolean storeTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions,
DocValuesType docValues, long dvGen,
int dimensionCount, int dimensionNumBytes) {
assert assertNotFinished();
if (docValues == null) {
@ -417,7 +418,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
// before then we'll get the same name and number,
// else we'll allocate a new one:
final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues, dimensionCount, dimensionNumBytes);
fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, new HashMap<>(), dimensionCount, dimensionNumBytes);
fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dimensionCount, dimensionNumBytes);
assert !byName.containsKey(fi.name);
globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
byName.put(fi.name, fi);
@ -435,16 +436,21 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
fi.setDocValuesType(docValues); // this will also perform the consistency check.
fi.setDocValuesGen(dvGen);
}
}
return fi;
}
public FieldInfo add(FieldInfo fi) {
return add(fi, -1);
}
public FieldInfo add(FieldInfo fi, long dvGen) {
// IMPORTANT - reuse the field number if possible for consistent field numbers across segments
return addOrUpdateInternal(fi.name, fi.number, fi.hasVectors(),
fi.omitsNorms(), fi.hasPayloads(),
fi.getIndexOptions(), fi.getDocValuesType(),
fi.getIndexOptions(), fi.getDocValuesType(), dvGen,
fi.getPointDimensionCount(), fi.getPointNumBytes());
}

2
lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java
View File

@ -129,7 +129,7 @@ public class ParallelLeafReader extends LeafReader {
for (FieldInfo fieldInfo : readerFieldInfos) {
// NOTE: first reader having a given field "wins":
if (!fieldToReader.containsKey(fieldInfo.name)) {
builder.add(fieldInfo);
builder.add(fieldInfo, fieldInfo.getDocValuesGen());
fieldToReader.put(fieldInfo.name, reader);
// only add these if the reader responsible for that field name is the current:
// TODO consider populating 1st leaf with vectors even if the field name has been seen on a previous leaf

26
lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java
View File

@ -23,6 +23,7 @@ import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.AlreadyClosedException;
@ -371,4 +372,29 @@ public class TestParallelLeafReader extends LuceneTestCase {
new ParallelLeafReader(false, getOnlyLeafReader(r2), getOnlyLeafReader(r1)).close();
IOUtils.close(r1, dir1, r2, dir2);
}
public void testWithDocValuesUpdates() throws Exception {
Directory dir1 = newDirectory();
IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random()));
IndexWriter w1 = new IndexWriter(dir1, iwc1);
Document d = new Document();
d.add(newTextField("name", "billy", Field.Store.NO));
d.add(new NumericDocValuesField("age", 21));
w1.addDocument(d);
w1.commit();
w1.updateNumericDocValue(new Term("name", "billy"), "age", 22);
w1.close();
IndexReader r1 = DirectoryReader.open(dir1);
LeafReader lr = new ParallelLeafReader(false, getOnlyLeafReader(r1));
NumericDocValues dv = lr.getNumericDocValues("age");
assertEquals(0, dv.nextDoc());
assertEquals(22, dv.longValue());
assertEquals(1, lr.getFieldInfos().fieldInfo("age").getDocValuesGen());
IOUtils.close(lr, r1, dir1);
}
}