Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8392: Avoir corner cases when maxMergeAtOnce is greater than segmentsPerTier.

This commit is contained in:
Adrien Grand 2018-07-10 08:42:59 +02:00
parent cd08c7ef13
commit 9433ab79e2
1 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
View File

@ -90,7 +90,9 @@ public class TieredMergePolicy extends MergePolicy {
* the index, then we disable compound file for it. * the index, then we disable compound file for it.
* @see MergePolicy#setNoCFSRatio */ * @see MergePolicy#setNoCFSRatio */
public static final double DEFAULT_NO_CFS_RATIO = 0.1; public static final double DEFAULT_NO_CFS_RATIO = 0.1;
// User-specified maxMergeAtOnce. In practice we always take the min of its
// value and segsPerTier to avoid suboptimal merging.
private int maxMergeAtOnce = 10; private int maxMergeAtOnce = 10;
private long maxMergedSegmentBytes = 5*1024*1024*1024L; private long maxMergedSegmentBytes = 5*1024*1024*1024L;
private int maxMergeAtOnceExplicit = 30; private int maxMergeAtOnceExplicit = 30;
@ -255,10 +257,6 @@ public class TieredMergePolicy extends MergePolicy {
/** Sets the allowed number of segments per tier. Smaller /** Sets the allowed number of segments per tier. Smaller
* values mean more merging but fewer segments. * values mean more merging but fewer segments.
* *
* <p><b>NOTE</b>: this value should be {@code >=} the {@link
* #setMaxMergeAtOnce} otherwise you'll force too much
* merging to occur.</p>
*
* <p>Default is 10.0.</p> */ * <p>Default is 10.0.</p> */
public TieredMergePolicy setSegmentsPerTier(double v) { public TieredMergePolicy setSegmentsPerTier(double v) {
if (v < 2.0) { if (v < 2.0) {
@ -397,6 +395,7 @@ public class TieredMergePolicy extends MergePolicy {
} }
} }
final int mergeFactor = (int) Math.min(maxMergeAtOnce, segsPerTier);
// Compute max allowed segments in the index // Compute max allowed segments in the index
long levelSize = Math.max(minSegmentBytes, floorSegmentBytes); long levelSize = Math.max(minSegmentBytes, floorSegmentBytes);
long bytesLeft = totIndexBytes; long bytesLeft = totIndexBytes;
@ -409,20 +408,20 @@ public class TieredMergePolicy extends MergePolicy {
} }
allowedSegCount += segsPerTier; allowedSegCount += segsPerTier;
bytesLeft -= segsPerTier * levelSize; bytesLeft -= segsPerTier * levelSize;
levelSize *= maxMergeAtOnce; levelSize *= mergeFactor;
} }
if (verbose(mergeContext) && tooBigCount > 0) { if (verbose(mergeContext) && tooBigCount > 0) {
message(" allowedSegmentCount=" + allowedSegCount + " vs count=" + infos.size() + message(" allowedSegmentCount=" + allowedSegCount + " vs count=" + infos.size() +
" (eligible count=" + sortedInfos.size() + ") tooBigCount= " + tooBigCount, mergeContext); " (eligible count=" + sortedInfos.size() + ") tooBigCount= " + tooBigCount, mergeContext);
} }
return doFindMerges(sortedInfos, maxMergedSegmentBytes, maxMergeAtOnce, (int) allowedSegCount, MERGE_TYPE.NATURAL, return doFindMerges(sortedInfos, maxMergedSegmentBytes, mergeFactor, (int) allowedSegCount, MERGE_TYPE.NATURAL,
mergeContext, mergingBytes >= maxMergedSegmentBytes); mergeContext, mergingBytes >= maxMergedSegmentBytes);
} }
private MergeSpecification doFindMerges(List<SegmentSizeAndDocs> sortedEligibleInfos, private MergeSpecification doFindMerges(List<SegmentSizeAndDocs> sortedEligibleInfos,
final long maxMergedSegmentBytes, final long maxMergedSegmentBytes,
final int maxMergeAtOnce, final int allowedSegCount, final int mergeFactor, final int allowedSegCount,
final MERGE_TYPE mergeType, final MERGE_TYPE mergeType,
MergeContext mergeContext, MergeContext mergeContext,
boolean maxMergeIsRunning) throws IOException { boolean maxMergeIsRunning) throws IOException {
@ -486,7 +485,7 @@ public class TieredMergePolicy extends MergePolicy {
long bestMergeBytes = 0; long bestMergeBytes = 0;
// Consider all merge starts. // Consider all merge starts.
int lim = sortedEligible.size() - maxMergeAtOnce; // assume the usual case of background merging. int lim = sortedEligible.size() - mergeFactor; // assume the usual case of background merging.
if (mergeType != MERGE_TYPE.NATURAL) { // The unusual case of forceMerge or expungeDeletes. if (mergeType != MERGE_TYPE.NATURAL) { // The unusual case of forceMerge or expungeDeletes.
// The incoming eligible list will have only segments with > forceMergeDeletesPctAllowed in the case of // The incoming eligible list will have only segments with > forceMergeDeletesPctAllowed in the case of
@ -502,7 +501,7 @@ public class TieredMergePolicy extends MergePolicy {
final List<SegmentCommitInfo> candidate = new ArrayList<>(); final List<SegmentCommitInfo> candidate = new ArrayList<>();
boolean hitTooLarge = false; boolean hitTooLarge = false;
long bytesThisMerge = 0; long bytesThisMerge = 0;
for (int idx = startIdx; idx < sortedEligible.size() && candidate.size() < maxMergeAtOnce && bytesThisMerge < maxMergedSegmentBytes; idx++) { for (int idx = startIdx; idx < sortedEligible.size() && candidate.size() < mergeFactor && bytesThisMerge < maxMergedSegmentBytes; idx++) {
final SegmentSizeAndDocs segSizeDocs = sortedEligible.get(idx); final SegmentSizeAndDocs segSizeDocs = sortedEligible.get(idx);
final long segBytes = segSizeDocs.sizeInBytes; final long segBytes = segSizeDocs.sizeInBytes;
@ -601,7 +600,8 @@ public class TieredMergePolicy extends MergePolicy {
// matter in this case because this merge will not // matter in this case because this merge will not
// "cascade" and so it cannot lead to N^2 merge cost // "cascade" and so it cannot lead to N^2 merge cost
// over time: // over time:
skew = 1.0/maxMergeAtOnce; final int mergeFactor = (int) Math.min(maxMergeAtOnce, segsPerTier);
skew = 1.0/mergeFactor;
} else { } else {
skew = ((double) floorSize(segmentsSizes.get(candidate.get(0)).sizeInBytes)) / totAfterMergeBytesFloored; skew = ((double) floorSize(segmentsSizes.get(candidate.get(0)).sizeInBytes)) / totAfterMergeBytesFloored;
} }