diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 2a0f19f833b..38c8127f07a 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -126,7 +126,8 @@ Bug Fixes Other --------------------- -(No changes) + +* LUCENE-9257: Always keep FST off-heap. FSTLoadMode, Reader attributes and openedFromWriter removed. (Bruno Roustant) ======================= Lucene 8.5.0 ======================= @@ -269,8 +270,6 @@ Other * LUCENE-9225: Rectangle extends LatLonGeometry so it can be used in a geometry collection. (Ignacio Vera) -* LUCENE-9257: Always keep FST off-heap. FSTLoadMode and Reader attributes removed. (Bruno Roustant) - ======================= Lucene 8.4.1 ======================= Bug Fixes diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java index f369e80f9ba..c9e17033f59 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsReader.java @@ -203,7 +203,7 @@ public final class BlockTreeTermsReader extends FieldsProducer { final long indexStartFP = indexIn.readVLong(); FieldReader previous = fieldMap.put(fieldInfo.name, new FieldReader(this, fieldInfo, numTerms, rootCode, sumTotalTermFreq, sumDocFreq, docCount, - indexStartFP, indexIn, minTerm, maxTerm, state.openedFromWriter)); + indexStartFP, indexIn, minTerm, maxTerm)); if (previous != null) { throw new CorruptIndexException("duplicate field: " + fieldInfo.name, termsIn); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java index 79816f2862d..01b9fa86dbd 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/FieldReader.java @@ -63,7 +63,7 @@ public final class FieldReader extends Terms implements Accountable { //private boolean DEBUG; FieldReader(BlockTreeTermsReader parent, FieldInfo fieldInfo, long numTerms, BytesRef rootCode, long sumTotalTermFreq, long sumDocFreq, int docCount, - long indexStartFP, IndexInput indexIn, BytesRef minTerm, BytesRef maxTerm, boolean openedFromWriter) throws IOException { + long indexStartFP, IndexInput indexIn, BytesRef minTerm, BytesRef maxTerm) throws IOException { assert numTerms > 0; this.fieldInfo = fieldInfo; //DEBUG = BlockTreeTermsReader.DEBUG && fieldInfo.name.equals("id"); diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java index 347f2ff2c38..79d68856f87 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java @@ -672,7 +672,7 @@ public final class CheckIndex implements Closeable { long startOpenReaderNS = System.nanoTime(); if (infoStream != null) infoStream.print(" test: open reader........."); - reader = new SegmentReader(info, sis.getIndexCreatedVersionMajor(), false, IOContext.DEFAULT); + reader = new SegmentReader(info, sis.getIndexCreatedVersionMajor(), IOContext.DEFAULT); msg(infoStream, String.format(Locale.ROOT, "OK [took %.3f sec]", nsToSec(System.nanoTime()-startOpenReaderNS))); segInfoStat.openReaderPassed = true; diff --git a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java index c2dae6063b3..42307dbc71a 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java +++ b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java @@ -131,7 +131,7 @@ final class DefaultIndexingChain extends DocConsumer { if (docState.infoStream.isEnabled("IW")) { docState.infoStream.message("IW", ((System.nanoTime()-t0)/1000000) + " msec to write norms"); } - SegmentReadState readState = new SegmentReadState(state.directory, state.segmentInfo, state.fieldInfos, true, IOContext.READ, state.segmentSuffix); + SegmentReadState readState = new SegmentReadState(state.directory, state.segmentInfo, state.fieldInfos, IOContext.READ, state.segmentSuffix); t0 = System.nanoTime(); writeDocValues(state, sortMap); diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java index 2baa0d0d7cd..17ce1e8c58a 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java +++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java @@ -168,7 +168,7 @@ final class ReadersAndUpdates { public synchronized SegmentReader getReader(IOContext context) throws IOException { if (reader == null) { // We steal returned ref: - reader = new SegmentReader(info, indexCreatedVersionMajor, true, context); + reader = new SegmentReader(info, indexCreatedVersionMajor, context); pendingDeletes.onNewReader(reader, info); } @@ -535,7 +535,7 @@ final class ReadersAndUpdates { // IndexWriter.commitMergedDeletes). final SegmentReader reader; if (this.reader == null) { - reader = new SegmentReader(info, indexCreatedVersionMajor, true, IOContext.READONCE); + reader = new SegmentReader(info, indexCreatedVersionMajor, IOContext.READONCE); pendingDeletes.onNewReader(reader, info); } else { reader = this.reader; diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java b/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java index 21e6c82cb60..0e7b9e43ca0 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java @@ -89,7 +89,7 @@ final class SegmentCoreReaders { private final Set coreClosedListeners = Collections.synchronizedSet(new LinkedHashSet()); - SegmentCoreReaders(Directory dir, SegmentCommitInfo si, boolean openedFromWriter, IOContext context) throws IOException { + SegmentCoreReaders(Directory dir, SegmentCommitInfo si, IOContext context) throws IOException { final Codec codec = si.info.getCodec(); final Directory cfsDir; // confusing name: if (cfs) it's the cfsdir, otherwise it's the segment's directory. @@ -107,7 +107,7 @@ final class SegmentCoreReaders { coreFieldInfos = codec.fieldInfosFormat().read(cfsDir, si.info, "", context); - final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si.info, coreFieldInfos, openedFromWriter, context); + final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si.info, coreFieldInfos, context); final PostingsFormat format = codec.postingsFormat(); // Ask codec for its Fields fields = format.fieldsProducer(segmentReadState); diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java index fee0050886d..aa44f8a43eb 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java @@ -46,7 +46,7 @@ final class SegmentDocValues { } // set SegmentReadState to list only the fields that are relevant to that gen - SegmentReadState srs = new SegmentReadState(dvDir, si.info, infos, false, IOContext.READ, segmentSuffix); + SegmentReadState srs = new SegmentReadState(dvDir, si.info, infos, IOContext.READ, segmentSuffix); DocValuesFormat dvFormat = si.info.getCodec().docValuesFormat(); return new RefCount(dvFormat.fieldsProducer(srs)) { @SuppressWarnings("synthetic-access") diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java index a91cea2a66c..a0780202dc9 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java @@ -112,7 +112,7 @@ final class SegmentMerger { final SegmentWriteState segmentWriteState = new SegmentWriteState(mergeState.infoStream, directory, mergeState.segmentInfo, mergeState.mergeFieldInfos, null, context); final SegmentReadState segmentReadState = new SegmentReadState(directory, mergeState.segmentInfo, mergeState.mergeFieldInfos, - true, IOContext.READ, segmentWriteState.segmentSuffix); + IOContext.READ, segmentWriteState.segmentSuffix); if (mergeState.mergeFieldInfos.hasNorms()) { if (mergeState.infoStream.isEnabled("SM")) { diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java b/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java index 107d754ed4f..5f5784be9e2 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java @@ -48,29 +48,23 @@ public class SegmentReadState { * {@link IndexFileNames#segmentFileName(String,String,String)}). */ public final String segmentSuffix; - /** - * True iff this SegmentReadState is opened from an IndexWriter. - */ - public final boolean openedFromWriter; - /** Create a {@code SegmentReadState}. */ public SegmentReadState(Directory dir, SegmentInfo info, - FieldInfos fieldInfos, boolean openedFromWriter, IOContext context) { - this(dir, info, fieldInfos, openedFromWriter, context, ""); + FieldInfos fieldInfos, IOContext context) { + this(dir, info, fieldInfos, context, ""); } /** Create a {@code SegmentReadState}. */ public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, - boolean openedFromWriter, IOContext context, + IOContext context, String segmentSuffix) { this.directory = dir; this.segmentInfo = info; this.fieldInfos = fieldInfos; this.context = context; this.segmentSuffix = segmentSuffix; - this.openedFromWriter = openedFromWriter; } /** Create a {@code SegmentReadState}. */ @@ -80,7 +74,6 @@ public class SegmentReadState { this.segmentInfo = other.segmentInfo; this.fieldInfos = other.fieldInfos; this.context = other.context; - this.openedFromWriter = other.openedFromWriter; this.segmentSuffix = newSegmentSuffix; } } diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java b/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java index c0ae337e5fc..b368b964b7a 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java @@ -72,7 +72,7 @@ public final class SegmentReader extends CodecReader { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - SegmentReader(SegmentCommitInfo si, int createdVersionMajor, boolean openedFromWriter, IOContext context) throws IOException { + SegmentReader(SegmentCommitInfo si, int createdVersionMajor, IOContext context) throws IOException { this.si = si.clone(); this.originalSi = si; this.metaData = new LeafMetaData(createdVersionMajor, si.info.getMinVersion(), si.info.getIndexSort()); @@ -80,7 +80,7 @@ public final class SegmentReader extends CodecReader { // We pull liveDocs/DV updates from disk: this.isNRT = false; - core = new SegmentCoreReaders(si.info.dir, si, openedFromWriter, context); + core = new SegmentCoreReaders(si.info.dir, si, context); segDocValues = new SegmentDocValues(); boolean success = false; diff --git a/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java index 82b7bad6049..8904eef8d8a 100644 --- a/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java @@ -63,7 +63,7 @@ public final class StandardDirectoryReader extends DirectoryReader { boolean success = false; try { for (int i = sis.size()-1; i >= 0; i--) { - readers[i] = new SegmentReader(sis.info(i), sis.getIndexCreatedVersionMajor(), false, IOContext.READ); + readers[i] = new SegmentReader(sis.info(i), sis.getIndexCreatedVersionMajor(), IOContext.READ); } // This may throw CorruptIndexException if there are too many docs, so @@ -176,7 +176,7 @@ public final class StandardDirectoryReader extends DirectoryReader { SegmentReader newReader; if (oldReader == null || commitInfo.info.getUseCompoundFile() != oldReader.getSegmentInfo().info.getUseCompoundFile()) { // this is a new reader; in case we hit an exception we can decRef it safely - newReader = new SegmentReader(commitInfo, infos.getIndexCreatedVersionMajor(), false, IOContext.READ); + newReader = new SegmentReader(commitInfo, infos.getIndexCreatedVersionMajor(), IOContext.READ); newReaders[i] = newReader; } else { if (oldReader.isNRT) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java index f48ae6e43de..6094ce67cde 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java @@ -222,7 +222,7 @@ public class TestCodecs extends LuceneTestCase { final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, SEGMENT, 10000, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null); this.write(si, fieldInfos, dir, fields); - final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, false, newIOContext(random()))); + final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random()))); final Iterator fieldsEnum = reader.iterator(); String fieldName = fieldsEnum.next(); @@ -282,7 +282,7 @@ public class TestCodecs extends LuceneTestCase { if (VERBOSE) { System.out.println("TEST: now read postings"); } - final FieldsProducer terms = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, false, newIOContext(random()))); + final FieldsProducer terms = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random()))); final Verify[] threads = new Verify[NUM_TEST_THREADS-1]; for(int i=0;i