honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make sure _forcemerge respects `max_num_segments`. (#32291) 

An upcoming [Lucene change](https://issues.apache.org/jira/browse/LUCENE-7976)
will make TieredMergePolicy respect the maximum merged segment size all the
time, meaning it will possibly not respect the `max_num_segments` parameter
anymore if the shard is larger than the maximum segment size.

This change makes sure that `max_num_segments` is respected for now in order
to give us time to think about how to integrate this change, and also to delay
it until 7.0 as this might be a big-enough change for us to wait for a new
major version.
This commit is contained in:
Adrien Grand 2018-07-26 09:02:58 +02:00 committed by GitHub
parent 48885d2b7e
commit f7ba14d5f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 222 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java Normal file
View File

@ -0,0 +1,120 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index;
import org.apache.lucene.index.FilterMergePolicy;
import org.apache.lucene.index.SegmentCommitInfo;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.TieredMergePolicy;
import java.io.IOException;
import java.util.Map;
/**
* Wrapper around {@link TieredMergePolicy} which doesn't respect
* {@link TieredMergePolicy#setMaxMergedSegmentMB(double)} on forced merges.
* See https://issues.apache.org/jira/browse/LUCENE-7976.
*/
final class EsTieredMergePolicy extends FilterMergePolicy {
final TieredMergePolicy regularMergePolicy;
final TieredMergePolicy forcedMergePolicy;
EsTieredMergePolicy() {
super(new TieredMergePolicy());
regularMergePolicy = (TieredMergePolicy) in;
forcedMergePolicy = new TieredMergePolicy();
forcedMergePolicy.setMaxMergedSegmentMB(Double.POSITIVE_INFINITY); // unlimited
}
@Override
public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount,
Map<SegmentCommitInfo, Boolean> segmentsToMerge, MergeContext mergeContext) throws IOException {
return forcedMergePolicy.findForcedMerges(infos, maxSegmentCount, segmentsToMerge, mergeContext);
}
@Override
public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException {
return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);
}
public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) {
regularMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
forcedMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
}
public double getForceMergeDeletesPctAllowed() {
return forcedMergePolicy.getForceMergeDeletesPctAllowed();
}
public void setFloorSegmentMB(double mbFrac) {
regularMergePolicy.setFloorSegmentMB(mbFrac);
forcedMergePolicy.setFloorSegmentMB(mbFrac);
}
public double getFloorSegmentMB() {
return regularMergePolicy.getFloorSegmentMB();
}
public void setMaxMergeAtOnce(int maxMergeAtOnce) {
regularMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
forcedMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
}
public int getMaxMergeAtOnce() {
return regularMergePolicy.getMaxMergeAtOnce();
}
public void setMaxMergeAtOnceExplicit(int maxMergeAtOnceExplicit) {
regularMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
forcedMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
}
public int getMaxMergeAtOnceExplicit() {
return forcedMergePolicy.getMaxMergeAtOnceExplicit();
}
// only setter that must NOT delegate to the forced merge policy
public void setMaxMergedSegmentMB(double mbFrac) {
regularMergePolicy.setMaxMergedSegmentMB(mbFrac);
}
public double getMaxMergedSegmentMB() {
return regularMergePolicy.getMaxMergedSegmentMB();
}
public void setSegmentsPerTier(double segmentsPerTier) {
regularMergePolicy.setSegmentsPerTier(segmentsPerTier);
forcedMergePolicy.setSegmentsPerTier(segmentsPerTier);
}
public double getSegmentsPerTier() {
return regularMergePolicy.getSegmentsPerTier();
}
public void setReclaimDeletesWeight(double reclaimDeletesWeight) {
regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
}
public double getReclaimDeletesWeight() {
return regularMergePolicy.getReclaimDeletesWeight();
}
}

2
server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
View File

@ -115,7 +115,7 @@ import org.elasticsearch.common.unit.ByteSizeValue;
*/
public final class MergePolicyConfig {
private final TieredMergePolicy mergePolicy = new TieredMergePolicy();
private final EsTieredMergePolicy mergePolicy = new EsTieredMergePolicy();
private final Logger logger;
private final boolean mergesEnabled;

80
server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java Normal file
View File

@ -0,0 +1,80 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index;
import org.apache.lucene.index.TieredMergePolicy;
import org.elasticsearch.test.ESTestCase;
public class EsTieredMergePolicyTests extends ESTestCase {
public void testDefaults() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
assertEquals(
new TieredMergePolicy().getMaxMergedSegmentMB(),
policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
// TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
}
public void testSetMaxMergedSegmentMB() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setMaxMergedSegmentMB(10 * 1024);
assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
// TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
}
public void testSetForceMergeDeletesPctAllowed() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setForceMergeDeletesPctAllowed(42);
assertEquals(42, policy.forcedMergePolicy.getForceMergeDeletesPctAllowed(), 0);
}
public void testSetFloorSegmentMB() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setFloorSegmentMB(42);
assertEquals(42, policy.regularMergePolicy.getFloorSegmentMB(), 0);
assertEquals(42, policy.forcedMergePolicy.getFloorSegmentMB(), 0);
}
public void testSetMaxMergeAtOnce() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setMaxMergeAtOnce(42);
assertEquals(42, policy.regularMergePolicy.getMaxMergeAtOnce());
}
public void testSetMaxMergeAtOnceExplicit() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setMaxMergeAtOnceExplicit(42);
assertEquals(42, policy.forcedMergePolicy.getMaxMergeAtOnceExplicit());
}
public void testSetSegmentsPerTier() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setSegmentsPerTier(42);
assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0);
}
public void testSetReclaimDeletesWeight() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setReclaimDeletesWeight(42);
assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0);
}
}

43
server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.index;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.TieredMergePolicy;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
@ -76,43 +75,43 @@ public class MergePolicySettingsTests extends ESTestCase {
public void testTieredMergePolicySettingsUpdate() throws IOException {
IndexSettings indexSettings = indexSettings(Settings.EMPTY);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING.getKey(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d).build()));
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB)).build()));
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1).build()));
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT - 1).build()));
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build()));
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build()));
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build()));
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
indexSettings.updateIndexMetaData(newIndexMeta("index", EMPTY_SETTINGS)); // see if defaults are restored
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
}
public Settings build(String value) {