From 1f1dc2e27a2f5601d0481c59db5d3ee6973d6202 Mon Sep 17 00:00:00 2001 From: Tomas Eduardo Fernandez Lobbe Date: Wed, 20 May 2015 19:32:14 +0000 Subject: [PATCH] SOLR-7463: Stop forcing MergePolicy's ''noCFSRatio'' based on the IWC ''useCompoundFile'' configuration. Use the MP's default if not specifically set git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1680663 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 14 +++++++- .../apache/solr/update/SolrIndexConfig.java | 5 --- .../solr/core/TestMergePolicyConfig.java | 35 +++++++++++++------ .../conf/solrconfig.xml | 1 + .../conf/solrconfig.xml | 1 + 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 4622e069948..87f8d7dba60 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -114,6 +114,15 @@ Upgrading from Solr 5.1 * The package structure under org.apache.solr.client.solrj.io has been changed to support the Streaming Expression Language (SOLR-7377). Any code written with the 5.1 Streaming API will have to be updated to reflect these changes. + +* Merge Policy's "noCFSRatio" is no longer set based on element in the indexConfig section + of solrconfig.xml. This means that Solr will start using Lucene's default for MP "noCFSRatio", with this + new default Solr will decide if a segment should use cfs or not based on the size of the segment in relation + the size of the complete index. For TieredMergePolicy for example (current default), segments will use cfs + if they are less than 10% of the index, otherwise cfs is disabled. Old values for this setting + (1.0 for useCompoundFile=true and 0.0 for useCompoundFile=false) as well as any other value can be set + inside the element in solrconfig.xml. will only apply to newly created + segments. See SOLR-7463. Detailed Change List @@ -409,12 +418,15 @@ Other Changes * SOLR-7541: Removed CollectionsHandler#createNodeIfNotExists. All calls made to this method now call ZkCmdExecutor#ensureExists as they were doing the same thing. Also ZkCmdExecutor#ensureExists now respects the CreateMode passed to it. (Varun Thacker) - + * SOLR-6820: Make the number of version buckets used by the UpdateLog configurable as increasing beyond the default 256 has been shown to help with high volume indexing performance in SolrCloud; helps overcome a limitation where Lucene uses the request thread to perform expensive index housekeeping work. (Mark Miller, yonik, Timothy Potter) +* SOLR-7463: Stop forcing MergePolicy's "NoCFSRatio" based on the IWC "useCompoundFile" configuration + (Tomás Fernández Löbbe) + ================== 5.1.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java index 793490ea81c..f6c4a945de4 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java @@ -259,18 +259,13 @@ public class SolrIndexConfig implements MapSerializable { if (maxMergeDocs != -1) logMergePolicy.setMaxMergeDocs(maxMergeDocs); - logMergePolicy.setNoCFSRatio(getUseCompoundFile() ? 1.0 : 0.0); - if (mergeFactor != -1) logMergePolicy.setMergeFactor(mergeFactor); - } else if (policy instanceof TieredMergePolicy) { TieredMergePolicy tieredMergePolicy = (TieredMergePolicy) policy; fixUseCFMergePolicyInitArg(TieredMergePolicy.class); - tieredMergePolicy.setNoCFSRatio(getUseCompoundFile() ? 1.0 : 0.0); - if (mergeFactor != -1) { tieredMergePolicy.setMaxMergeAtOnce(mergeFactor); tieredMergePolicy.setSegmentsPerTier(mergeFactor); diff --git a/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java b/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java index 1b9c7cbaa68..cb076616e89 100644 --- a/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java @@ -17,21 +17,21 @@ package org.apache.solr.core; * limitations under the License. */ -import org.apache.lucene.index.LeafReaderContext; -import org.apache.solr.update.SolrIndexConfigTest; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.TieredMergePolicy; -import org.apache.lucene.index.LogMergePolicy; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.LogByteSizeMergePolicy; import org.apache.lucene.index.LogDocMergePolicy; -import org.apache.solr.util.RefCounted; -import org.apache.solr.search.SolrIndexSearcher; +import org.apache.lucene.index.LogMergePolicy; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.TieredMergePolicy; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.update.SolrIndexConfigTest; +import org.apache.solr.util.RefCounted; import org.junit.After; -import java.util.concurrent.atomic.AtomicInteger; /** @see SolrIndexConfigTest */ public class TestMergePolicyConfig extends SolrTestCaseJ4 { @@ -42,6 +42,22 @@ public class TestMergePolicyConfig extends SolrTestCaseJ4 { public void after() throws Exception { deleteCore(); } + + public void testSetNoCFSMergePolicyConfig() throws Exception { + final boolean useCompoundFile = random().nextBoolean(); + System.setProperty("testSetNoCFSMergePolicyConfig.useCompoundFile", String.valueOf(useCompoundFile)); + try { + initCore("solrconfig-mergepolicy-nocfs.xml","schema-minimal.xml"); + IndexWriterConfig iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore()); + assertEquals(useCompoundFile, iwc.getUseCompoundFile()); + + TieredMergePolicy tieredMP = assertAndCast(TieredMergePolicy.class, + iwc.getMergePolicy()); + assertEquals(0.5D, tieredMP.getNoCFSRatio(), 0.0D); + } finally { + System.getProperties().remove("testSetNoCFSMergePolicyConfig.useCompoundFile"); + } + } public void testDefaultMergePolicyConfig() throws Exception { initCore("solrconfig-mergepolicy-defaults.xml","schema-minimal.xml"); @@ -50,7 +66,7 @@ public class TestMergePolicyConfig extends SolrTestCaseJ4 { TieredMergePolicy tieredMP = assertAndCast(TieredMergePolicy.class, iwc.getMergePolicy()); - assertEquals(0.0D, tieredMP.getNoCFSRatio(), 0.0D); + assertEquals(TieredMergePolicy.DEFAULT_NO_CFS_RATIO, tieredMP.getNoCFSRatio(), 0.0D); assertCommitSomeNewDocs(); assertCompoundSegments(h.getCore(), false); @@ -70,7 +86,6 @@ public class TestMergePolicyConfig extends SolrTestCaseJ4 { assertEquals(7, tieredMP.getMaxMergeAtOnce()); assertEquals(7.0D, tieredMP.getSegmentsPerTier(), 0.0D); - assertEquals(expectCFS ? 1.0D : 0.0D, tieredMP.getNoCFSRatio(), 0.0D); assertCommitSomeNewDocs(); assertCompoundSegments(h.getCore(), expectCFS); diff --git a/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml b/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml index db08b1bd8dc..ddbe51bfc84 100644 --- a/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml +++ b/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml @@ -197,6 +197,7 @@ 10 10 + 0.1 --> diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml index 76d4e0e0042..82c2f5ce541 100644 --- a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml +++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml @@ -199,6 +199,7 @@ 10 10 + 0.1 -->