From 77ea2110965a05e91cbd04257ccd394e1fd3fad3 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 9 Nov 2017 13:31:20 +0000 Subject: [PATCH] LUCENE-8045: ParallelLeafReader should correctly report FieldInfo.dvGen --- lucene/CHANGES.txt | 3 +++ .../org/apache/lucene/index/FieldInfos.java | 12 ++++++--- .../lucene/index/ParallelLeafReader.java | 2 +- .../lucene/index/TestParallelLeafReader.java | 26 +++++++++++++++++++ 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 867843ef2e7..9477a7991a2 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -58,6 +58,9 @@ Bug Fixes Previously bogus numbers were used, and many similarities would completely degrade. (Robert Muir, Adrien Grand) +* LUCENE-8045: ParallelLeafReader did not correctly report FieldInfo.dvGen + (Alan Woodward) + Optimizations * LUCENE-8018: Smaller FieldInfos memory footprint by not retaining unnecessary diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java index a7dc0b86277..3e68693253a 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java @@ -403,7 +403,8 @@ public class FieldInfos implements Iterable { private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean storeTermVector, - boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, + boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, + DocValuesType docValues, long dvGen, int dimensionCount, int dimensionNumBytes) { assert assertNotFinished(); if (docValues == null) { @@ -417,7 +418,7 @@ public class FieldInfos implements Iterable { // before then we'll get the same name and number, // else we'll allocate a new one: final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues, dimensionCount, dimensionNumBytes); - fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, new HashMap<>(), dimensionCount, dimensionNumBytes); + fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dimensionCount, dimensionNumBytes); assert !byName.containsKey(fi.name); globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType()); byName.put(fi.name, fi); @@ -435,16 +436,21 @@ public class FieldInfos implements Iterable { } fi.setDocValuesType(docValues); // this will also perform the consistency check. + fi.setDocValuesGen(dvGen); } } return fi; } public FieldInfo add(FieldInfo fi) { + return add(fi, -1); + } + + public FieldInfo add(FieldInfo fi, long dvGen) { // IMPORTANT - reuse the field number if possible for consistent field numbers across segments return addOrUpdateInternal(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(), fi.hasPayloads(), - fi.getIndexOptions(), fi.getDocValuesType(), + fi.getIndexOptions(), fi.getDocValuesType(), dvGen, fi.getPointDimensionCount(), fi.getPointNumBytes()); } diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java index 74b39622ebe..492b6e7bc97 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java @@ -129,7 +129,7 @@ public class ParallelLeafReader extends LeafReader { for (FieldInfo fieldInfo : readerFieldInfos) { // NOTE: first reader having a given field "wins": if (!fieldToReader.containsKey(fieldInfo.name)) { - builder.add(fieldInfo); + builder.add(fieldInfo, fieldInfo.getDocValuesGen()); fieldToReader.put(fieldInfo.name, reader); // only add these if the reader responsible for that field name is the current: // TODO consider populating 1st leaf with vectors even if the field name has been seen on a previous leaf diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java index 35523f352a7..71110d3b31e 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelLeafReader.java @@ -23,6 +23,7 @@ import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.store.AlreadyClosedException; @@ -371,4 +372,29 @@ public class TestParallelLeafReader extends LuceneTestCase { new ParallelLeafReader(false, getOnlyLeafReader(r2), getOnlyLeafReader(r1)).close(); IOUtils.close(r1, dir1, r2, dir2); } + + public void testWithDocValuesUpdates() throws Exception { + Directory dir1 = newDirectory(); + IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random())); + IndexWriter w1 = new IndexWriter(dir1, iwc1); + Document d = new Document(); + d.add(newTextField("name", "billy", Field.Store.NO)); + d.add(new NumericDocValuesField("age", 21)); + w1.addDocument(d); + w1.commit(); + w1.updateNumericDocValue(new Term("name", "billy"), "age", 22); + w1.close(); + + IndexReader r1 = DirectoryReader.open(dir1); + LeafReader lr = new ParallelLeafReader(false, getOnlyLeafReader(r1)); + + NumericDocValues dv = lr.getNumericDocValues("age"); + assertEquals(0, dv.nextDoc()); + assertEquals(22, dv.longValue()); + + assertEquals(1, lr.getFieldInfos().fieldInfo("age").getDocValuesGen()); + + IOUtils.close(lr, r1, dir1); + + } }