From d966adcb6239d969d88dcc9198fa3df07353f854 Mon Sep 17 00:00:00 2001 From: Marc D'Mello Date: Wed, 5 Oct 2022 15:33:12 -0700 Subject: [PATCH] GITHUB-11761: Move minimum TieredMergePolicy delete percentage and change default value (#11831) Move minimum TieredMergePolicy delete percentage from 20% to 5% and change deletePctAllowed default to 20% Co-authored-by: Marc D'Mello --- lucene/CHANGES.txt | 3 +++ .../org/apache/lucene/index/TieredMergePolicy.java | 13 +++++++++---- .../apache/lucene/index/TestIndexWriterDelete.java | 1 + .../apache/lucene/index/TestTieredMergePolicy.java | 1 + 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index f1e64372168..46347a09c58 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -98,6 +98,9 @@ API Changes * GITHUB#11804: FacetsCollector#collect is no longer final, allowing extension. (Greg Miller) +* GITHUB#11761: TieredMergePolicy now allowed a maximum allowable deletes percentage of down to 5%, and the default + maximum allowable deletes percentage is changed from 33% to 20%. (Marc D'Mello) + Improvements --------------------- * GITHUB#11778: Detailed part-of-speech information for particle(조사) and ending(어미) on Nori diff --git a/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java index eff23656dad..cc1a89e10ae 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java +++ b/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java @@ -92,7 +92,7 @@ public class TieredMergePolicy extends MergePolicy { private long floorSegmentBytes = 2 * 1024 * 1024L; private double segsPerTier = 10.0; private double forceMergeDeletesPctAllowed = 10.0; - private double deletesPctAllowed = 33.0; + private double deletesPctAllowed = 20.0; /** Sole constructor, setting all settings to their defaults. */ public TieredMergePolicy() { @@ -151,12 +151,17 @@ public class TieredMergePolicy extends MergePolicy { /** * Controls the maximum percentage of deleted documents that is tolerated in the index. Lower * values make the index more space efficient at the expense of increased CPU and I/O activity. - * Values must be between 20 and 50. Default value is 33. + * Values must be between 5 and 50. Default value is 20. + * + *

When the maximum delete percentage is lowered, the indexing thread will call for merges more + * often, meaning that write amplification factor will be increased. Write amplification factor + * measures the number of times each document in the index is written. A higher write + * amplification factor will lead to higher CPU and I/O activity as indicated above. */ public TieredMergePolicy setDeletesPctAllowed(double v) { - if (v < 20 || v > 50) { + if (v < 5 || v > 50) { throw new IllegalArgumentException( - "indexPctDeletedTarget must be >= 20.0 and <= 50 (got " + v + ")"); + "indexPctDeletedTarget must be >= 5.0 and <= 50 (got " + v + ")"); } deletesPctAllowed = v; return this; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java index 2115dc0cb72..9f8413a76bd 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java @@ -1345,6 +1345,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { w.addDocument(doc); w.close(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); + ((TieredMergePolicy) iwc.getMergePolicy()).setDeletesPctAllowed(33.0); iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND); w = new IndexWriter(d, iwc); IndexReader r = DirectoryReader.open(w, false, false); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java index 0523d9db21b..7d52107b900 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java @@ -310,6 +310,7 @@ public class TestTieredMergePolicy extends BaseMergePolicyTestCase { ((1024.0 * 1024.0)); // fudge it up, we're trying to catch egregious errors and segbytes // don't really reflect the number for original merges. tmp.setMaxMergedSegmentMB(mbSize); + tmp.setDeletesPctAllowed(33.0); conf.setMaxBufferedDocs(100); conf.setMergePolicy(tmp);