Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-4941: Fix handling of <mergePolicy> init arg "useCompoundFile" -> IndexWriterConfig 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1494837 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris M. Hostetter 2013-06-20 01:36:34 +00:00
parent 31d3dc8dec
commit b3e02db3a9
4 changed files with 170 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
solr/CHANGES.txt
View File

@ -60,14 +60,21 @@ Upgrading from Solr 4.3.0
----------------------
* TieredMergePolicy and the various subtypes of LogMergePolicy no longer have
an explicit "setUseCompoundFile" method. Instead this behavior is now driven
entirely by the value of "setNoCFSRatio". If you have explicitly configured
one of these classes using <mergePolicy> and include an init arg like this...
an explicit "setUseCompoundFile" method. Instead the behavior of new
segments is determined by the IndexWriter configuration, and the MergePolicy
is only consulted to determine if merge segements should use the compound
file format (based on the value of "setNoCFSRatio"). If you have explicitly
configured one of these classes using <mergePolicy> and include an init arg
like this...
<bool name="useCompoundFile">true</bool>
...this will now be ignored and a warning will be logged. Users with an
explicitly declared <mergePolicy> are encouraged to review the current
javadocs for their MergePolicy subclass and review their configured options
carefully. See SOLR-4934 and LUCENE-5038 for more information.
...this will now be treated as if you specified...
<useCompoundFile>true</useCompoundFile>
...directly on the <indexConfig> (overriding any value already set using that
syntax) and a warning will be logged to updated your configuration. Users
with an explicitly declared <mergePolicy> are encouraged to review the
current javadocs for their MergePolicy subclass and review their configured
options carefully. See SOLR-4941, SOLR-4934 and LUCENE-5038 for more
information.
* SOLR-4778: The signature of LogWatcher.registerListener has changed, from
(ListenerConfig, CoreContainer) to (ListenerConfig). Users implementing their
@ -174,8 +181,8 @@ Bug Fixes
defined in the original. Benign since the default values are saved, but still incorrect.
(Erick Erickson, thanks Shawn Heisey for helping test!)
* SOLR-4934: Fix handling of <mergePolicy> init arg "useCompoundFile" needed
after changes in LUCENE-5038 (hossman)
* SOLR-4934, SOLR-4941: Fix handling of <mergePolicy> init arg
"useCompoundFile" needed after changes in LUCENE-5038 (hossman)
* SOLR-4456: Admin UI: Displays dashboard even if Solr is down (steffkes)

61
solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
View File

@ -43,7 +43,14 @@ public class SolrIndexConfig {
public static final String DEFAULT_MERGE_SCHEDULER_CLASSNAME = ConcurrentMergeScheduler.class.getName();
public final Version luceneVersion;
/**
* The explicit value of &lt;useCompoundFile&gt; specified on this index config
* @deprecated use {@link #getUseCompoundFile}
*/
@Deprecated
public final boolean useCompoundFile;
private boolean effectiveUseCompountFileSetting;
public final int maxBufferedDocs;
public final int maxMergeDocs;
public final int maxIndexingThreads;
@ -73,7 +80,7 @@ public class SolrIndexConfig {
@SuppressWarnings("deprecation")
private SolrIndexConfig(SolrConfig solrConfig) {
luceneVersion = solrConfig.luceneMatchVersion;
useCompoundFile = false;
useCompoundFile = effectiveUseCompountFileSetting = false;
maxBufferedDocs = -1;
maxMergeDocs = -1;
maxIndexingThreads = IndexWriterConfig.DEFAULT_MAX_THREAD_STATES;
@ -121,6 +128,7 @@ public class SolrIndexConfig {
defaultMergePolicyClassName = def.defaultMergePolicyClassName;
useCompoundFile=solrConfig.getBool(prefix+"/useCompoundFile", def.useCompoundFile);
effectiveUseCompountFileSetting = useCompoundFile;
maxBufferedDocs=solrConfig.getInt(prefix+"/maxBufferedDocs",def.maxBufferedDocs);
maxMergeDocs=solrConfig.getInt(prefix+"/maxMergeDocs",def.maxMergeDocs);
maxIndexingThreads=solrConfig.getInt(prefix+"/maxIndexingThreads",def.maxIndexingThreads);
@ -189,6 +197,10 @@ public class SolrIndexConfig {
iwc.setMergePolicy(buildMergePolicy(schema));
iwc.setMergeScheduler(buildMergeScheduler(schema));
// do this after buildMergePolicy since the backcompat logic
// there may modify the effective useCompoundFile
iwc.setUseCompoundFile(getUseCompoundFile());
if (maxIndexingThreads != -1) {
iwc.setMaxThreadStates(maxIndexingThreads);
}
@ -206,6 +218,15 @@ public class SolrIndexConfig {
return iwc;
}
/**
* Builds a MergePolicy, may also modify the value returned by
* getUseCompoundFile() for use by the IndexWriterConfig if
* "useCompoundFile" is specified as an init arg for
* an out of the box MergePolicy that no longer supports it
*
* @see #fixUseCFMergePolicyInitArg
* @see #getUseCompoundFile
*/
private MergePolicy buildMergePolicy(IndexSchema schema) {
String mpClassName = mergePolicyInfo == null ? defaultMergePolicyClassName : mergePolicyInfo.className;
@ -213,31 +234,31 @@ public class SolrIndexConfig {
if (policy instanceof LogMergePolicy) {
LogMergePolicy logMergePolicy = (LogMergePolicy) policy;
fixUseCFMergePolicyInitArg(LogMergePolicy.class);
if (maxMergeDocs != -1)
logMergePolicy.setMaxMergeDocs(maxMergeDocs);
logMergePolicy.setNoCFSRatio(useCompoundFile ? 1.0 : 0.0);
logMergePolicy.setNoCFSRatio(getUseCompoundFile() ? 1.0 : 0.0);
if (mergeFactor != -1)
logMergePolicy.setMergeFactor(mergeFactor);
fixUseCFInitArg(LogMergePolicy.class);
} else if (policy instanceof TieredMergePolicy) {
TieredMergePolicy tieredMergePolicy = (TieredMergePolicy) policy;
fixUseCFMergePolicyInitArg(TieredMergePolicy.class);
tieredMergePolicy.setNoCFSRatio(useCompoundFile ? 1.0 : 0.0);
tieredMergePolicy.setNoCFSRatio(getUseCompoundFile() ? 1.0 : 0.0);
if (mergeFactor != -1) {
tieredMergePolicy.setMaxMergeAtOnce(mergeFactor);
tieredMergePolicy.setSegmentsPerTier(mergeFactor);
}
fixUseCFInitArg(TieredMergePolicy.class);
} else if (useCompoundFile && (mergeFactor != -1)) {
log.warn("Use of <useCompoundFile> or <mergeFactor> cannot be configured if merge policy is not an instance of LogMergePolicy or TieredMergePolicy. The configured policy's defaults will be used.");
} else if (mergeFactor != -1) {
log.warn("Use of <mergeFactor> cannot be configured if merge policy is not an instance of LogMergePolicy or TieredMergePolicy. The configured policy's defaults will be used.");
}
if (mergePolicyInfo != null)
@ -256,19 +277,35 @@ public class SolrIndexConfig {
return scheduler;
}
public boolean getUseCompoundFile() {
return effectiveUseCompountFileSetting;
}
/**
* Lucene 4.4 removed the setUseCompoundFile(boolean) method from the two
* conrete MergePolicies provided with Lucene/Solr. In the event that users
* have a value explicitly configured for this setter in their mergePolicy
* init args, we remove it for them and warn about it.
* conrete MergePolicies provided with Lucene/Solr and added it to the
* IndexWRiterConfig.
* In the event that users have a value explicitly configured for this
* setter in their MergePolicy init args, we remove it from the MergePolicy
* init args, update the 'effective' useCompoundFile setting used by the
* IndexWriterConfig, and warn about discontinuing to use this init arg.
*
* @see #getUseCompoundFile
*/
private void fixUseCFInitArg(Class c) {
private void fixUseCFMergePolicyInitArg(Class c) {
if (null == mergePolicyInfo || null == mergePolicyInfo.initArgs) return;
Object useCFSArg = mergePolicyInfo.initArgs.remove("useCompoundFile");
if (null != useCFSArg) {
log.warn("Ignoring 'useCompoundFile' specified as an init arg for the <mergePolicy> since it is no longer supported by " + c.getSimpleName());
log.warn("Ignoring 'useCompoundFile' specified as an init arg for the <mergePolicy> since it is no directly longer supported by " + c.getSimpleName());
if (useCFSArg instanceof Boolean) {
boolean cfs = ((Boolean)useCFSArg).booleanValue();
log.warn("Please update your config to specify <useCompoundFile>"+cfs+"</useCompoundFile> directly in your <indexConfig> settings.");
effectiveUseCompountFileSetting = cfs;
} else {
log.error("MergePolicy's 'useCompoundFile' init arg is not a boolean, can not apply back compat logic to apply to the IndexWriterConfig: " + useCFSArg.toString());
}
}
}
}

8
solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy.xml
View File

@ -26,10 +26,12 @@
<mergePolicy class="org.apache.lucene.index.TieredMergePolicy">
<int name="maxMergeAtOnceExplicit">19</int>
<int name="segmentsPerTier">9</int>
<double name="noCFSRatio">0.6</double>
<double name="noCFSRatio">0.1</double>
<!-- Setter for this was removed in Lucene 4.4, but we should
ignore it with a warning if users have it in their configs
<!-- Setter for this was moved from the MergePolicies to IndexWriterConfig
in Lucene 4.4, so we should treat it the same as a <useCompoundFile>
setting and log a warning (instead of failing because the setter is
gone).
-->
<bool name="useCompoundFile">true</bool>

101
solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java
View File

@ -17,13 +17,24 @@ package org.apache.solr.core;
* limitations under the License.
*/
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.solr.core.SolrCore;
import org.apache.solr.util.RefCounted;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.SolrTestCaseJ4;
import org.junit.After;
import java.util.concurrent.atomic.AtomicInteger;
public class TestMergePolicyConfig extends SolrTestCaseJ4 {
private static AtomicInteger docIdCounter = new AtomicInteger(42);
@After
public void after() throws Exception {
@ -33,27 +44,39 @@ public class TestMergePolicyConfig extends SolrTestCaseJ4 {
public void testDefaultMergePolicyConfig() throws Exception {
initCore("solrconfig-mergepolicy-defaults.xml","schema-minimal.xml");
IndexWriterConfig iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore().getLatestSchema());
assertEquals(false, iwc.getUseCompoundFile());
TieredMergePolicy tieredMP = assertAndCast(TieredMergePolicy.class,
iwc.getMergePolicy());
assertEquals(0.0D, tieredMP.getNoCFSRatio(), 0.0D);
assertCommitSomeNewDocs();
assertCompoundSegments(h.getCore(), false);
}
public void testLegacyMergePolicyConfig() throws Exception {
initCore("solrconfig-mergepolicy-legacy.xml","schema-minimal.xml");
IndexWriterConfig iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore().getLatestSchema());
assertEquals(true, iwc.getUseCompoundFile());
TieredMergePolicy tieredMP = assertAndCast(TieredMergePolicy.class,
iwc.getMergePolicy());
assertEquals(7, tieredMP.getMaxMergeAtOnce());
assertEquals(7.0D, tieredMP.getSegmentsPerTier(), 0.0D);
assertEquals(1.0D, tieredMP.getNoCFSRatio(), 0.0D);
assertCommitSomeNewDocs();
assertCompoundSegments(h.getCore(), true);
}
public void testTieredMergePolicyConfig() throws Exception {
initCore("solrconfig-mergepolicy.xml","schema-minimal.xml");
IndexWriterConfig iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore().getLatestSchema());
assertEquals(true, iwc.getUseCompoundFile());
TieredMergePolicy tieredMP = assertAndCast(TieredMergePolicy.class,
iwc.getMergePolicy());
@ -62,11 +85,25 @@ public class TestMergePolicyConfig extends SolrTestCaseJ4 {
// mp-specific setters
assertEquals(19, tieredMP.getMaxMergeAtOnceExplicit());
assertEquals(0.6D, tieredMP.getNoCFSRatio(), 0.001);
assertEquals(0.1D, tieredMP.getNoCFSRatio(), 0.0D);
// make sure we overrode segmentsPerTier
// (split from maxMergeAtOnce out of mergeFactor)
assertEquals(9D, tieredMP.getSegmentsPerTier(), 0.001);
assertCommitSomeNewDocs();
// even though we have a single segment (which is 100% of the size of
// the index which is higher then our 0.6D threashold) the
// compound ratio doesn't matter because the segment was never merged
assertCompoundSegments(h.getCore(), true);
assertCommitSomeNewDocs();
assertNumSegments(h.getCore(), 2);
assertCompoundSegments(h.getCore(), true);
assertU(optimize());
assertNumSegments(h.getCore(), 1);
// we've now forced a merge, and the MP ratio should be in play
assertCompoundSegments(h.getCore(), false);
}
/**
@ -81,4 +118,66 @@ public class TestMergePolicyConfig extends SolrTestCaseJ4 {
return clazz.cast(o);
}
public static void assertCommitSomeNewDocs() {
for (int i = 0; i < 5; i++) {
int val = docIdCounter.getAndIncrement();
assertU(adoc("id", "" + val,
"a_s", val + "_" + val + "_" + val + "_" + val,
"b_s", val + "_" + val + "_" + val + "_" + val,
"c_s", val + "_" + val + "_" + val + "_" + val,
"d_s", val + "_" + val + "_" + val + "_" + val,
"e_s", val + "_" + val + "_" + val + "_" + val,
"f_s", val + "_" + val + "_" + val + "_" + val));
}
assertU(commit());
}
/**
* Given an SolrCore, asserts that the number of leave segments in
* the index reader matches the expected value.
*/
public static void assertNumSegments(SolrCore core, int expected) {
RefCounted<SolrIndexSearcher> searcherRef = core.getRegisteredSearcher();
try {
assertEquals(expected, searcherRef.get().getIndexReader().leaves().size());
} finally {
searcherRef.decref();
}
}
/**
* Given an SolrCore, asserts that each segment in the (searchable) index
* has a compound file status that matches the expected input.
*/
public static void assertCompoundSegments(SolrCore core, boolean compound) {
RefCounted<SolrIndexSearcher> searcherRef = core.getRegisteredSearcher();
try {
assertCompoundSegments(searcherRef.get().getIndexReader(), compound);
} finally {
searcherRef.decref();
}
}
/**
* Given an IndexReader, asserts that there is at least one AtomcReader leaf,
* and that all AtomicReader leaves are SegmentReader's that have a compound
* file status that matches the expected input.
*/
private static void assertCompoundSegments(IndexReader reader,
boolean compound) {
assertNotNull("Null leaves", reader.leaves());
assertTrue("no leaves", 0 < reader.leaves().size());
for (AtomicReaderContext atomic : reader.leaves()) {
assertTrue("not a segment reader: " + atomic.reader().toString(),
atomic.reader() instanceof SegmentReader);
assertEquals("Compound status incorrect for: " +
atomic.reader().toString(),
compound,
((SegmentReader)atomic.reader()).getSegmentInfo().info.getUseCompoundFile());
}
}
}