Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make FSTPostingsFormat load FSTs off-heap (#12552) 

* Make FSTPostingsFormat load FSTs off-heap
This commit is contained in:
Tony-X 2023-09-19 13:29:23 -07:00 committed by GitHub
parent fe348de619
commit ca69ae6d98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 15 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/CHANGES.txt
View File

@ -91,6 +91,8 @@ Optimizations
* GITHUB#12408: Lazy initialization improvements for Facets implementations when there are segments with no hits
to count. (Greg Miller)
* GITHUB#12552: Make FSTPostingsFormat load FSTs off-heap. (Tony X)
Bug Fixes
---------------------

22
lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
View File

@ -39,6 +39,7 @@ import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@ -50,6 +51,7 @@ import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.fst.BytesRefFSTEnum;
import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.OffHeapFSTStore;
import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.fst.Util;
@ -61,9 +63,9 @@ import org.apache.lucene.util.fst.Util;
* @lucene.experimental
*/
public class FSTTermsReader extends FieldsProducer {
final TreeMap<String, TermsReader> fields = new TreeMap<>();
final PostingsReaderBase postingsReader;
// static boolean TEST = false;
private final TreeMap<String, TermsReader> fields = new TreeMap<>();
private final PostingsReaderBase postingsReader;
private final IndexInput fstTermsInput;
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader)
throws IOException {
@ -72,7 +74,9 @@ public class FSTTermsReader extends FieldsProducer {
state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
this.postingsReader = postingsReader;
final IndexInput in = state.directory.openInput(termsFileName, state.context);
this.fstTermsInput = state.directory.openInput(termsFileName, IOContext.LOAD);
IndexInput in = this.fstTermsInput;
boolean success = false;
try {
@ -105,9 +109,7 @@ public class FSTTermsReader extends FieldsProducer {
}
success = true;
} finally {
if (success) {
IOUtils.close(in);
} else {
if (success == false) {
IOUtils.closeWhileHandlingException(in);
}
}
@ -163,7 +165,7 @@ public class FSTTermsReader extends FieldsProducer {
@Override
public void close() throws IOException {
try {
IOUtils.close(postingsReader);
IOUtils.close(postingsReader, fstTermsInput);
} finally {
fields.clear();
}
@ -191,7 +193,9 @@ public class FSTTermsReader extends FieldsProducer {
this.sumTotalTermFreq = sumTotalTermFreq;
this.sumDocFreq = sumDocFreq;
this.docCount = docCount;
this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo));
OffHeapFSTStore offHeapFSTStore = new OffHeapFSTStore();
this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo), offHeapFSTStore);
in.skipBytes(offHeapFSTStore.size());
}
@Override