honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade to Lucene-7.5.0-snapshot-608f0277b0 (#32390) 

The main highlight is the removal of the reclaim_deletes_weight in the TieredMergePolicy.
The es setting index.merge.policy.reclaim_deletes_weight is deprecated in this commit and the value is ignored. The new merge policy setting setDeletesPctAllowed should be added in a follow up.
This commit is contained in:
Jim Ferenczi 2018-07-27 08:28:51 +02:00 committed by GitHub
parent 9198bcf78e
commit 53ff06e621
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
61 changed files with 50 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/version.properties
View File

@ -1,5 +1,5 @@
elasticsearch = 7.0.0-alpha1
lucene = 7.4.0
lucene = 7.5.0-snapshot-608f0277b0
# optional dependencies
spatial4j = 0.7

4
docs/Versions.asciidoc
View File

@ -1,7 +1,7 @@
:version: 7.0.0-alpha1
:major-version: 7.x
:lucene_version: 7.4.0
:lucene_version_path: 7_4_0
:lucene_version: 7.5.0
:lucene_version_path: 7_5_0
:branch: master
:jdk: 1.8.0_131
:jdk_major: 8

1
modules/lang-expression/licenses/lucene-expressions-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
9f0a326f7ec1671ffb07f95b27f1a5812b7dc1c3

1
modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
bd7d8078a2d0ad11a24f54156cc015630c96858a

2
plugins/analysis-icu/build.gradle
View File

@ -30,7 +30,7 @@ forbiddenApis {
dependencies {
compile "org.apache.lucene:lucene-analyzers-icu:${versions.lucene}"
compile 'com.ibm.icu:icu4j:61.1'
compile 'com.ibm.icu:icu4j:62.1'
}
dependencyLicenses {

1
plugins/analysis-icu/licenses/icu4j-61.1.jar.sha1
View File

@ -1 +0,0 @@
28d33b5e44e72edcc66a5da7a34a42147f38d987

1
plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1 Normal file
View File

@ -0,0 +1 @@
7a4d00d5ec5febd252a6182e8b6e87a0a9821f81

1
plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
394e811e9d9bf0b9fba837f7ceca9e8f3e39d1c2

1
plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
7a37816def72a748416c4ae8b0f6817e30efb99f

1
plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
5cd56acfa16ba20e19b5d21d90b510eada841431

1
plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
ca7437178cdbf7b8bfe0d75c75e3c8eb93925724

1
plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
db7b56f4cf533ad9022d2312c5ee48331edccca3

1
plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
3f5dec44f380d6d58bc1c8aec51964fcb5390b60

1
plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
e8dba4d28a595eab2e8fb6095d1ac5f2d3872144

1
plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
453bf1d60df0415439095624e0b3e42492ad4716

1
plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
1243c771ee824c46a3d66ae3e4256d919fc06fbe

1
plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
70095a45257bca9f46629b5fb6cedf9eff5e2b07

1
plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
c783794b0d20d8dc1285edc7701f386b1f0e2fb8

1
plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
7199d6962d268b7877f7b5160e98e4ff21cce5c7

1
plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
9438efa504a89afb6cb4c66448c257f865164d23

1
plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
12aff508d39d206a1aead5013ecd11882062eb06

8
plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbMMapDirectoryTests.java
View File

@ -30,4 +30,12 @@ public class SmbMMapDirectoryTests extends EsBaseDirectoryTestCase {
protected Directory getDirectory(Path file) throws IOException {
return new SmbDirectoryWrapper(new MMapDirectory(file));
}
@Override
public void testCreateOutputForExistingFile() throws IOException {
/**
* This test is disabled because {@link SmbDirectoryWrapper} opens existing file
* with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
*/
}
}

8
plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java
View File

@ -30,4 +30,12 @@ public class SmbSimpleFSDirectoryTests extends EsBaseDirectoryTestCase {
protected Directory getDirectory(Path file) throws IOException {
return new SmbDirectoryWrapper(new SimpleFSDirectory(file));
}
@Override
public void testCreateOutputForExistingFile() throws IOException {
/**
* This test is disabled because {@link SmbDirectoryWrapper} opens existing file
* with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
*/
}
}

1
server/licenses/lucene-analyzers-common-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
e1afb580df500626a1c695e0fc9a7e8a8f58bcac

1
server/licenses/lucene-analyzers-common-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d27958843ca118db2ffd2c242ae3761bd5a47328

1
server/licenses/lucene-backward-codecs-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
a6ad941ef1fdad48673ed511631b7e48a9456bf7

1
server/licenses/lucene-backward-codecs-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
7ea220ba8e4accb8b04e280463042ad470e23bc0

1
server/licenses/lucene-core-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
730d9ac80436c8cbc0b2a8a749259be536b97316

1
server/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
471096d6e92338b208aa91f3a85feb2f9cfc4afd

1
server/licenses/lucene-grouping-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
56f99858a4421a517b52da36a222debcccab80c6

1
server/licenses/lucene-grouping-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
f0af947c60d24f779c22f774e81ebd7dd91cc932

1
server/licenses/lucene-highlighter-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
5266b45d7f049662817d739881765904621876d0

1
server/licenses/lucene-highlighter-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
fbc83ac5a0139ed7e7faf6c95a2718f46f28c641

1
server/licenses/lucene-join-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
c77154d18c4944ceb6ce0741060632f57d623fdc

1
server/licenses/lucene-join-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
30adfe493982b0db059dc243e269eea38d850d46

1
server/licenses/lucene-memory-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
186ff981feec1bdbf1a6236e786ec171b5fbe3e0

1
server/licenses/lucene-memory-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
656f304261d9aad05070fb68593beffafe9147e3

1
server/licenses/lucene-misc-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
bf844bb6f6d84da19e8c79ce5fbb4cf6d00f2611

1
server/licenses/lucene-misc-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
8bf22ad81a7480c255b55bada401eb131bfdb4df

1
server/licenses/lucene-queries-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
229a50e6d9d4db076f671c230d493000c6e2972c

1
server/licenses/lucene-queries-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
edb3de4d68a34c1e1ca08f79fe4d103b10e98ad1

1
server/licenses/lucene-queryparser-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
8e58add0d0c39df97d07c8e343041989bf4b3a3f

1
server/licenses/lucene-queryparser-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
7ece30d5f1e18d96f61644451c858c3d9960558f

1
server/licenses/lucene-sandbox-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
1692604fa06a945d1ee19939022ef1a912235db3

1
server/licenses/lucene-sandbox-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
ad3bd0c2ed96556193c7215bef328e689d0b157f

1
server/licenses/lucene-spatial-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
847d2f897961124e2fc7d5e55d8309635bb026bc

1
server/licenses/lucene-spatial-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
8a6bd97e39ee5af60126adbe8c8375dc41b1ea8e

1
server/licenses/lucene-spatial-extras-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
586892eefc0546643d7f5d7f83659c7db0d534ff

1
server/licenses/lucene-spatial-extras-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
07e748d2d80000a7a213f3405b82b6e26b452948

1
server/licenses/lucene-spatial3d-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
32cd2854f39ff453a5d128ce40e11eea4168abbf

1
server/licenses/lucene-spatial3d-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
fd737bd5562f3943618ee7e73a0aaffb6319fdb2

1
server/licenses/lucene-suggest-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
0cdc1a512032f8b23dd4b1add0f5cd06325addc3

1
server/licenses/lucene-suggest-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
ff3f260d1dc8c18bc67f3c33aa84a0ad290daac5

2
server/src/main/java/org/elasticsearch/Version.java
View File

@ -182,7 +182,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
public static final int V_7_0_0_alpha1_ID = 7000001;
public static final Version V_7_0_0_alpha1 =
new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
public static final Version CURRENT = V_7_0_0_alpha1;
static {

11
server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java
View File

@ -52,7 +52,7 @@ final class EsTieredMergePolicy extends FilterMergePolicy {
@Override
public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException {
return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);
return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);
}
public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) {
@ -108,13 +108,4 @@ final class EsTieredMergePolicy extends FilterMergePolicy {
public double getSegmentsPerTier() {
return regularMergePolicy.getSegmentsPerTier();
}
public void setReclaimDeletesWeight(double reclaimDeletesWeight) {
regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
}
public double getReclaimDeletesWeight() {
return regularMergePolicy.getReclaimDeletesWeight();
}
}

1
server/src/main/java/org/elasticsearch/index/IndexSettings.java
View File

@ -416,7 +416,6 @@ public final class IndexSettings {
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING, mergePolicyConfig::setMaxMergesAtOnceExplicit);
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, mergePolicyConfig::setMaxMergedSegment);
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, mergePolicyConfig::setSegmentsPerTier);
scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING, mergePolicyConfig::setReclaimDeletesWeight);
scopedSettings.addSettingsUpdateConsumer(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING,
mergeSchedulerConfig::setMaxThreadAndMergeCount);

7
server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
View File

@ -150,7 +150,7 @@ public final class MergePolicyConfig {
Property.Dynamic, Property.IndexScope);
public static final Setting<Double> INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING =
Setting.doubleSetting("index.merge.policy.reclaim_deletes_weight", DEFAULT_RECLAIM_DELETES_WEIGHT, 0.0d,
Property.Dynamic, Property.IndexScope);
Property.Dynamic, Property.IndexScope, Property.Deprecated);
public static final String INDEX_MERGE_ENABLED = "index.merge.enabled"; // don't convert to Setting<> and register... we only set this in tests and register via a plugin
@ -176,17 +176,12 @@ public final class MergePolicyConfig {
mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
mergePolicy.setSegmentsPerTier(segmentsPerTier);
mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
if (logger.isTraceEnabled()) {
logger.trace("using [tiered] merge mergePolicy with expunge_deletes_allowed[{}], floor_segment[{}], max_merge_at_once[{}], max_merge_at_once_explicit[{}], max_merged_segment[{}], segments_per_tier[{}], reclaim_deletes_weight[{}]",
forceMergeDeletesPctAllowed, floorSegment, maxMergeAtOnce, maxMergeAtOnceExplicit, maxMergedSegment, segmentsPerTier, reclaimDeletesWeight);
}
}
void setReclaimDeletesWeight(Double reclaimDeletesWeight) {
mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
}
void setSegmentsPerTier(Double segmentsPerTier) {
mergePolicy.setSegmentsPerTier(segmentsPerTier);
}

12
server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java
View File

@ -29,16 +29,14 @@ public class EsTieredMergePolicyTests extends ESTestCase {
assertEquals(
new TieredMergePolicy().getMaxMergedSegmentMB(),
policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
// TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
}
public void testSetMaxMergedSegmentMB() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setMaxMergedSegmentMB(10 * 1024);
assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
// TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
}
public void testSetForceMergeDeletesPctAllowed() {
@ -71,10 +69,4 @@ public class EsTieredMergePolicyTests extends ESTestCase {
policy.setSegmentsPerTier(42);
assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0);
}
public void testSetReclaimDeletesWeight() {
EsTieredMergePolicy policy = new EsTieredMergePolicy();
policy.setReclaimDeletesWeight(42);
assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0);
}
}

5
server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java
View File

@ -96,10 +96,6 @@ public class MergePolicySettingsTests extends ESTestCase {
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build()));
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build()));
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build()));
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
@ -110,7 +106,6 @@ public class MergePolicySettingsTests extends ESTestCase {
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
}

1
x-pack/plugin/sql/sql-action/licenses/lucene-core-7.4.0.jar.sha1
View File

@ -1 +0,0 @@
730d9ac80436c8cbc0b2a8a749259be536b97316

1
x-pack/plugin/sql/sql-action/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
471096d6e92338b208aa91f3a85feb2f9cfc4afd