Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Run merge-on-full-flush even though no changes got flushed. (#12549) 

Currently, merge-on-full-flush only checks if merges need to run if changes
have been flushed to disk. This prevents from having different merging logic
for refreshes and commits, since the merge policy would not be checked upon
commit if no new documents got indexed since the previous refresh.
This commit is contained in:
Adrien Grand 2023-10-24 16:57:05 +02:00 committed by GitHub
parent 5461d1a160
commit 71c4ea74ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 14 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -3680,7 +3680,7 @@ public class IndexWriter
// merge completes which would otherwise have
// removed the files we are now syncing.
deleter.incRef(toCommit.files(false));
if (anyChanges && maxCommitMergeWaitMillis > 0) {
if (maxCommitMergeWaitMillis > 0) {
// we can safely call preparePointInTimeMerge since writeReaderPool(true) above
// wrote all
// necessary files to disk and checkpointed them.
@ -4232,7 +4232,6 @@ public class IndexWriter
flushSuccess = true;
} finally {
assert Thread.holdsLock(fullFlushLock);
;
docWriter.finishFullFlush(flushSuccess);
processEvents(false);
}

3
lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
View File

@ -654,7 +654,8 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
public void testOverDecRefDuringReopen() throws Exception {
MockDirectoryWrapper dir = newMockDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
IndexWriterConfig iwc =
new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setCodec(TestUtil.getDefaultCodec());
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();

5
lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
View File

@ -519,11 +519,10 @@ public class TestIndexWriter extends LuceneTestCase {
doc.add(newField("field", "aaa", customType));
for (int i = 0; i < 19; i++) writer.addDocument(doc);
writer.flush(false, true);
writer.close();
SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
// Since we flushed w/o allowing merging we should now
// have 10 segments
assertEquals(10, sis.size());
assertEquals(10, writer.getSegmentCount());
writer.close();
dir.close();
}

2
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
View File

@ -1314,7 +1314,7 @@ public class TestIndexWriterDelete extends LuceneTestCase {
w.addDocument(doc);
w.close();
iwc = new IndexWriterConfig(new MockAnalyzer(random()));
iwc = new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
w = new IndexWriter(d, iwc);
IndexReader r = DirectoryReader.open(w, false, false);

15
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
View File

@ -394,14 +394,13 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
.setMaxFullFlushMergeWaitMillis(Integer.MAX_VALUE);
IndexWriter writerWithMergePolicy = new IndexWriter(dir, iwc);
writerWithMergePolicy.commit(); // No changes. Commit doesn't trigger a merge.
// No changes. Refresh doesn't trigger a merge.
DirectoryReader unmergedReader = DirectoryReader.open(writerWithMergePolicy);
assertEquals(5, unmergedReader.leaves().size());
unmergedReader.close();
TestIndexWriter.addDoc(writerWithMergePolicy);
writerWithMergePolicy.commit(); // Doc added, do merge on commit.
writerWithMergePolicy.commit(); // Do merge on commit.
assertEquals(1, writerWithMergePolicy.getSegmentCount()); //
DirectoryReader mergedReader = DirectoryReader.open(writerWithMergePolicy);
@ -410,8 +409,8 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
try (IndexReader reader = DirectoryReader.open(writerWithMergePolicy)) {
IndexSearcher searcher = new IndexSearcher(reader);
assertEquals(6, reader.numDocs());
assertEquals(6, searcher.count(new MatchAllDocsQuery()));
assertEquals(5, reader.numDocs());
assertEquals(5, searcher.count(new MatchAllDocsQuery()));
}
writerWithMergePolicy.close();
@ -445,16 +444,14 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
.setIndexWriterEventListener(eventListener);
IndexWriter writerWithMergePolicy = new IndexWriter(dir, iwc);
writerWithMergePolicy.commit(); // No changes. Commit doesn't trigger a merge.
// No changes. Refresh doesn't trigger a merge.
DirectoryReader unmergedReader = DirectoryReader.open(writerWithMergePolicy);
assertEquals(5, unmergedReader.leaves().size());
unmergedReader.close();
TestIndexWriter.addDoc(writerWithMergePolicy);
assertFalse(eventListener.isEventsRecorded());
writerWithMergePolicy.commit(); // Doc added, do merge on commit.
writerWithMergePolicy.commit(); // Do merge on commit.
assertEquals(1, writerWithMergePolicy.getSegmentCount()); //
assertTrue(eventListener.isEventsRecorded());

2
lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
View File

@ -263,6 +263,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
newIndexWriterConfig()
.setSoftDeletesField("_soft_deletes")
.setMaxBufferedDocs(3) // make sure we write one segment
.setMergePolicy(NoMergePolicy.INSTANCE) // prevent deletes from triggering merges
.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH));
Document doc = new Document();
doc.add(new StringField("id", "1", Field.Store.YES));
@ -331,6 +332,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
newIndexWriterConfig()
.setSoftDeletesField("_soft_deletes")
.setMaxBufferedDocs(3) // make sure we write one segment
.setMergePolicy(NoMergePolicy.INSTANCE) // prevent deletes from triggering merges
.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH));
Document doc = new Document();
doc.add(new StringField("id", "1", Field.Store.YES));