Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-9257: Always keep FST off-heap. Remove SegmentReadState.openedFromWriter.

This commit is contained in:
Bruno Roustant 2020-03-06 13:45:24 +01:00
parent 4cf37ade35
commit c73d2c15ba
No known key found for this signature in database
GPG Key ID: CD28DABB95360525
22 changed files with 39 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/CHANGES.txt
View File

@ -126,7 +126,8 @@ Bug Fixes
Other
---------------------
(No changes)
* LUCENE-9257: Always keep FST off-heap. FSTLoadMode, Reader attributes and openedFromWriter removed. (Bruno Roustant)
======================= Lucene 8.5.0 =======================
@ -269,8 +270,6 @@ Other
* LUCENE-9225: Rectangle extends LatLonGeometry so it can be used in a geometry collection. (Ignacio Vera)
* LUCENE-9257: Always keep FST off-heap. FSTLoadMode and Reader attributes removed. (Bruno Roustant)
======================= Lucene 8.4.1 =======================
Bug Fixes

2
lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java
View File

@ -203,7 +203,7 @@ public final class BlockTreeTermsReader extends FieldsProducer {
final long indexStartFP = indexIn.readVLong();
FieldReader previous = fieldMap.put(fieldInfo.name,
new FieldReader(this, fieldInfo, numTerms, rootCode, sumTotalTermFreq, sumDocFreq, docCount,
indexStartFP, indexIn, minTerm, maxTerm, state.openedFromWriter));
indexStartFP, indexIn, minTerm, maxTerm));
if (previous != null) {
throw new CorruptIndexException("duplicate field: " + fieldInfo.name, termsIn);
}

2
lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java
View File

@ -63,7 +63,7 @@ public final class FieldReader extends Terms implements Accountable {
//private boolean DEBUG;
FieldReader(BlockTreeTermsReader parent, FieldInfo fieldInfo, long numTerms, BytesRef rootCode, long sumTotalTermFreq, long sumDocFreq, int docCount,
long indexStartFP, IndexInput indexIn, BytesRef minTerm, BytesRef maxTerm, boolean openedFromWriter) throws IOException {
long indexStartFP, IndexInput indexIn, BytesRef minTerm, BytesRef maxTerm) throws IOException {
assert numTerms > 0;
this.fieldInfo = fieldInfo;
//DEBUG = BlockTreeTermsReader.DEBUG && fieldInfo.name.equals("id");

2
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -672,7 +672,7 @@ public final class CheckIndex implements Closeable {
long startOpenReaderNS = System.nanoTime();
if (infoStream != null)
infoStream.print(" test: open reader.........");
reader = new SegmentReader(info, sis.getIndexCreatedVersionMajor(), false, IOContext.DEFAULT);
reader = new SegmentReader(info, sis.getIndexCreatedVersionMajor(), IOContext.DEFAULT);
msg(infoStream, String.format(Locale.ROOT, "OK [took %.3f sec]", nsToSec(System.nanoTime()-startOpenReaderNS)));
segInfoStat.openReaderPassed = true;

2
lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
View File

@ -131,7 +131,7 @@ final class DefaultIndexingChain extends DocConsumer {
if (docState.infoStream.isEnabled("IW")) {
docState.infoStream.message("IW", ((System.nanoTime()-t0)/1000000) + " msec to write norms");
}
SegmentReadState readState = new SegmentReadState(state.directory, state.segmentInfo, state.fieldInfos, true, IOContext.READ, state.segmentSuffix);
SegmentReadState readState = new SegmentReadState(state.directory, state.segmentInfo, state.fieldInfos, IOContext.READ, state.segmentSuffix);
t0 = System.nanoTime();
writeDocValues(state, sortMap);

4
lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
View File

@ -168,7 +168,7 @@ final class ReadersAndUpdates {
public synchronized SegmentReader getReader(IOContext context) throws IOException {
if (reader == null) {
// We steal returned ref:
reader = new SegmentReader(info, indexCreatedVersionMajor, true, context);
reader = new SegmentReader(info, indexCreatedVersionMajor, context);
pendingDeletes.onNewReader(reader, info);
}
@ -535,7 +535,7 @@ final class ReadersAndUpdates {
// IndexWriter.commitMergedDeletes).
final SegmentReader reader;
if (this.reader == null) {
reader = new SegmentReader(info, indexCreatedVersionMajor, true, IOContext.READONCE);
reader = new SegmentReader(info, indexCreatedVersionMajor, IOContext.READONCE);
pendingDeletes.onNewReader(reader, info);
} else {
reader = this.reader;

4
lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
View File

@ -89,7 +89,7 @@ final class SegmentCoreReaders {
private final Set<IndexReader.ClosedListener> coreClosedListeners =
Collections.synchronizedSet(new LinkedHashSet<IndexReader.ClosedListener>());
SegmentCoreReaders(Directory dir, SegmentCommitInfo si, boolean openedFromWriter, IOContext context) throws IOException {
SegmentCoreReaders(Directory dir, SegmentCommitInfo si, IOContext context) throws IOException {
final Codec codec = si.info.getCodec();
final Directory cfsDir; // confusing name: if (cfs) it's the cfsdir, otherwise it's the segment's directory.
@ -107,7 +107,7 @@ final class SegmentCoreReaders {
coreFieldInfos = codec.fieldInfosFormat().read(cfsDir, si.info, "", context);
final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si.info, coreFieldInfos, openedFromWriter, context);
final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si.info, coreFieldInfos, context);
final PostingsFormat format = codec.postingsFormat();
// Ask codec for its Fields
fields = format.fieldsProducer(segmentReadState);

2
lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java
View File

@ -46,7 +46,7 @@ final class SegmentDocValues {
}
// set SegmentReadState to list only the fields that are relevant to that gen
SegmentReadState srs = new SegmentReadState(dvDir, si.info, infos, false, IOContext.READ, segmentSuffix);
SegmentReadState srs = new SegmentReadState(dvDir, si.info, infos, IOContext.READ, segmentSuffix);
DocValuesFormat dvFormat = si.info.getCodec().docValuesFormat();
return new RefCount<DocValuesProducer>(dvFormat.fieldsProducer(srs)) {
@SuppressWarnings("synthetic-access")

2
lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
View File

@ -112,7 +112,7 @@ final class SegmentMerger {
final SegmentWriteState segmentWriteState = new SegmentWriteState(mergeState.infoStream, directory, mergeState.segmentInfo,
mergeState.mergeFieldInfos, null, context);
final SegmentReadState segmentReadState = new SegmentReadState(directory, mergeState.segmentInfo, mergeState.mergeFieldInfos,
true, IOContext.READ, segmentWriteState.segmentSuffix);
IOContext.READ, segmentWriteState.segmentSuffix);
if (mergeState.mergeFieldInfos.hasNorms()) {
if (mergeState.infoStream.isEnabled("SM")) {

13
lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java
View File

@ -48,29 +48,23 @@ public class SegmentReadState {
* {@link IndexFileNames#segmentFileName(String,String,String)}). */
public final String segmentSuffix;
/**
* True iff this SegmentReadState is opened from an IndexWriter.
*/
public final boolean openedFromWriter;
/** Create a {@code SegmentReadState}. */
public SegmentReadState(Directory dir, SegmentInfo info,
FieldInfos fieldInfos, boolean openedFromWriter, IOContext context) {
this(dir, info, fieldInfos, openedFromWriter, context, "");
FieldInfos fieldInfos, IOContext context) {
this(dir, info, fieldInfos, context, "");
}
/** Create a {@code SegmentReadState}. */
public SegmentReadState(Directory dir,
SegmentInfo info,
FieldInfos fieldInfos,
boolean openedFromWriter, IOContext context,
IOContext context,
String segmentSuffix) {
this.directory = dir;
this.segmentInfo = info;
this.fieldInfos = fieldInfos;
this.context = context;
this.segmentSuffix = segmentSuffix;
this.openedFromWriter = openedFromWriter;
}
/** Create a {@code SegmentReadState}. */
@ -80,7 +74,6 @@ public class SegmentReadState {
this.segmentInfo = other.segmentInfo;
this.fieldInfos = other.fieldInfos;
this.context = other.context;
this.openedFromWriter = other.openedFromWriter;
this.segmentSuffix = newSegmentSuffix;
}
}

4
lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
View File

@ -72,7 +72,7 @@ public final class SegmentReader extends CodecReader {
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
SegmentReader(SegmentCommitInfo si, int createdVersionMajor, boolean openedFromWriter, IOContext context) throws IOException {
SegmentReader(SegmentCommitInfo si, int createdVersionMajor, IOContext context) throws IOException {
this.si = si.clone();
this.originalSi = si;
this.metaData = new LeafMetaData(createdVersionMajor, si.info.getMinVersion(), si.info.getIndexSort());
@ -80,7 +80,7 @@ public final class SegmentReader extends CodecReader {
// We pull liveDocs/DV updates from disk:
this.isNRT = false;
core = new SegmentCoreReaders(si.info.dir, si, openedFromWriter, context);
core = new SegmentCoreReaders(si.info.dir, si, context);
segDocValues = new SegmentDocValues();
boolean success = false;

4
lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
View File

@ -63,7 +63,7 @@ public final class StandardDirectoryReader extends DirectoryReader {
boolean success = false;
try {
for (int i = sis.size()-1; i >= 0; i--) {
readers[i] = new SegmentReader(sis.info(i), sis.getIndexCreatedVersionMajor(), false, IOContext.READ);
readers[i] = new SegmentReader(sis.info(i), sis.getIndexCreatedVersionMajor(), IOContext.READ);
}
// This may throw CorruptIndexException if there are too many docs, so
@ -176,7 +176,7 @@ public final class StandardDirectoryReader extends DirectoryReader {
SegmentReader newReader;
if (oldReader == null || commitInfo.info.getUseCompoundFile() != oldReader.getSegmentInfo().info.getUseCompoundFile()) {
// this is a new reader; in case we hit an exception we can decRef it safely
newReader = new SegmentReader(commitInfo, infos.getIndexCreatedVersionMajor(), false, IOContext.READ);
newReader = new SegmentReader(commitInfo, infos.getIndexCreatedVersionMajor(), IOContext.READ);
newReaders[i] = newReader;
} else {
if (oldReader.isNRT) {

4
lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
View File

@ -222,7 +222,7 @@ public class TestCodecs extends LuceneTestCase {
final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, SEGMENT, 10000, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
this.write(si, fieldInfos, dir, fields);
final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, false, newIOContext(random())));
final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random())));
final Iterator<String> fieldsEnum = reader.iterator();
String fieldName = fieldsEnum.next();
@ -282,7 +282,7 @@ public class TestCodecs extends LuceneTestCase {
if (VERBOSE) {
System.out.println("TEST: now read postings");
}
final FieldsProducer terms = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, false, newIOContext(random())));
final FieldsProducer terms = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random())));
final Verify[] threads = new Verify[NUM_TEST_THREADS-1];
for(int i=0;i<NUM_TEST_THREADS-1;i++) {

2
lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
View File

@ -416,7 +416,7 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
assert infos.size() == 1;
final LeafReader parLeafReader = new SegmentReader(infos.info(0), Version.LATEST.major, false, IOContext.DEFAULT);
final LeafReader parLeafReader = new SegmentReader(infos.info(0), Version.LATEST.major, IOContext.DEFAULT);
//checkParallelReader(leaf, parLeafReader, schemaGen);

6
lucene/core/src/test/org/apache/lucene/index/TestDoc.java
View File

@ -213,8 +213,8 @@ public class TestDoc extends LuceneTestCase {
private SegmentCommitInfo merge(Directory dir, SegmentCommitInfo si1, SegmentCommitInfo si2, String merged, boolean useCompoundFile)
throws Exception {
IOContext context = newIOContext(random(), new IOContext(new MergeInfo(-1, -1, false, -1)));
SegmentReader r1 = new SegmentReader(si1, Version.LATEST.major, false, context);
SegmentReader r2 = new SegmentReader(si2, Version.LATEST.major, false, context);
SegmentReader r1 = new SegmentReader(si1, Version.LATEST.major, context);
SegmentReader r2 = new SegmentReader(si2, Version.LATEST.major, context);
final Codec codec = Codec.getDefault();
TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(si1.info.dir);
@ -244,7 +244,7 @@ public class TestDoc extends LuceneTestCase {
private void printSegment(PrintWriter out, SegmentCommitInfo si)
throws Exception {
SegmentReader reader = new SegmentReader(si, Version.LATEST.major, false, newIOContext(random()));
SegmentReader reader = new SegmentReader(si, Version.LATEST.major, newIOContext(random()));
for (int i = 0; i < reader.numDocs(); i++)
out.println(reader.document(i));

8
lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java
View File

@ -63,7 +63,7 @@ public class TestDocumentWriter extends LuceneTestCase {
SegmentCommitInfo info = writer.newestSegment();
writer.close();
//After adding the document, we should be able to read it back in
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, false, newIOContext(random()));
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, newIOContext(random()));
assertTrue(reader != null);
Document doc = reader.document(0);
assertTrue(doc != null);
@ -124,7 +124,7 @@ public class TestDocumentWriter extends LuceneTestCase {
writer.commit();
SegmentCommitInfo info = writer.newestSegment();
writer.close();
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, false, newIOContext(random()));
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, newIOContext(random()));
PostingsEnum termPositions = MultiTerms.getTermPostingsEnum(reader, "repeated", new BytesRef("repeated"));
assertTrue(termPositions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
@ -195,7 +195,7 @@ public class TestDocumentWriter extends LuceneTestCase {
writer.commit();
SegmentCommitInfo info = writer.newestSegment();
writer.close();
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, false, newIOContext(random()));
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, newIOContext(random()));
PostingsEnum termPositions = MultiTerms.getTermPostingsEnum(reader, "f1", new BytesRef("a"));
assertTrue(termPositions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
@ -237,7 +237,7 @@ public class TestDocumentWriter extends LuceneTestCase {
writer.commit();
SegmentCommitInfo info = writer.newestSegment();
writer.close();
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, false, newIOContext(random()));
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, newIOContext(random()));
PostingsEnum termPositions = reader.postings(new Term("preanalyzed", "term1"), PostingsEnum.ALL);
assertTrue(termPositions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);

2
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java
View File

@ -333,7 +333,7 @@ public class TestIndexWriterThreadsToSegments extends LuceneTestCase {
SegmentInfo si = TestUtil.getDefaultCodec().segmentInfoFormat().read(dir, segName, id, IOContext.DEFAULT);
si.setCodec(codec);
SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, 0, -1, -1, -1);
SegmentReader sr = new SegmentReader(sci, Version.LATEST.major, false, IOContext.DEFAULT);
SegmentReader sr = new SegmentReader(sci, Version.LATEST.major, IOContext.DEFAULT);
try {
thread0Count += sr.docFreq(new Term("field", "threadID0"));
thread1Count += sr.docFreq(new Term("field", "threadID1"));

6
lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
View File

@ -60,8 +60,8 @@ public class TestSegmentMerger extends LuceneTestCase {
SegmentCommitInfo info1 = DocHelper.writeDoc(random(), merge1Dir, doc1);
DocHelper.setupDoc(doc2);
SegmentCommitInfo info2 = DocHelper.writeDoc(random(), merge2Dir, doc2);
reader1 = new SegmentReader(info1, Version.LATEST.major, false, newIOContext(random()));
reader2 = new SegmentReader(info2, Version.LATEST.major, false, newIOContext(random()));
reader1 = new SegmentReader(info1, Version.LATEST.major, newIOContext(random()));
reader2 = new SegmentReader(info2, Version.LATEST.major, newIOContext(random()));
}
@Override
@ -98,7 +98,7 @@ public class TestSegmentMerger extends LuceneTestCase {
mergeState.segmentInfo,
0, 0, -1L, -1L, -1L),
Version.LATEST.major,
false, newIOContext(random()));
newIOContext(random()));
assertTrue(mergedReader != null);
assertTrue(mergedReader.numDocs() == 2);
Document newDoc1 = mergedReader.document(0);

2
lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
View File

@ -43,7 +43,7 @@ public class TestSegmentReader extends LuceneTestCase {
dir = newDirectory();
DocHelper.setupDoc(testDoc);
SegmentCommitInfo info = DocHelper.writeDoc(random(), dir, testDoc);
reader = new SegmentReader(info, Version.LATEST.major, false, IOContext.READ);
reader = new SegmentReader(info, Version.LATEST.major, IOContext.READ);
}
@Override

6
lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java
View File

@ -54,7 +54,7 @@ public class TestSegmentTermDocs extends LuceneTestCase {
public void testTermDocs() throws IOException {
//After adding the document, we should be able to read it back in
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, false, newIOContext(random()));
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, newIOContext(random()));
assertTrue(reader != null);
TermsEnum terms = reader.terms(DocHelper.TEXT_FIELD_2_KEY).iterator();
@ -72,7 +72,7 @@ public class TestSegmentTermDocs extends LuceneTestCase {
public void testBadSeek() throws IOException {
{
//After adding the document, we should be able to read it back in
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, false, newIOContext(random()));
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, newIOContext(random()));
assertTrue(reader != null);
PostingsEnum termDocs = TestUtil.docs(random(), reader,
"textField2",
@ -85,7 +85,7 @@ public class TestSegmentTermDocs extends LuceneTestCase {
}
{
//After adding the document, we should be able to read it back in
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, false, newIOContext(random()));
SegmentReader reader = new SegmentReader(info, Version.LATEST.major, newIOContext(random()));
assertTrue(reader != null);
PostingsEnum termDocs = TestUtil.docs(random(), reader,
"junk",

2
lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
View File

@ -362,7 +362,7 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
segmentInfo, fieldInfos,
null, new IOContext(new FlushInfo(1, 20)));
SegmentReadState readState = new SegmentReadState(dir, segmentInfo, fieldInfos, false, IOContext.READ);
SegmentReadState readState = new SegmentReadState(dir, segmentInfo, fieldInfos, IOContext.READ);
// PostingsFormat
NormsProducer fakeNorms = new NormsProducer() {

2
lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
View File

@ -748,7 +748,7 @@ public class RandomPostingsTester {
currentFieldInfos = newFieldInfos;
SegmentReadState readState = new SegmentReadState(dir, segmentInfo, newFieldInfos, false, IOContext.READ);
SegmentReadState readState = new SegmentReadState(dir, segmentInfo, newFieldInfos, IOContext.READ);
return codec.postingsFormat().fieldsProducer(readState);
}