Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-9712: block when maxWarmingSearchers is exceeded instead of throwing exception, default to 1, remove from most configs

This commit is contained in:
yonik 2016-12-16 11:46:48 -05:00
parent a11cdd2fd8
commit 0f4c5f0a73
25 changed files with 62 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
solr/CHANGES.txt
View File

@ -58,6 +58,12 @@ Upgrade Notes
for a listing of highlight parameters annotated with which highlighters use them.
hl.useFastVectorHighlighter is now considered deprecated in lieu of hl.method=fastVector.
* SOLR-9712: maxWarmingSearchers now defaults to 1, and more importantly commits will now block if this
limit is exceeded instead of throwing an exception (a good thing). Consequently there is no longer a
risk in overlapping commits. Nonetheless users should continue to avoid excessive committing. Users are
advised to remove any pre-existing maxWarmingSearchers entries from their solrconfig.xml files.
New Features
----------------------
* SOLR-9293: Solrj client support for hierarchical clusters and other topics

5
solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml
View File

@ -212,11 +212,6 @@
warming. -->
<useColdSearcher>false</useColdSearcher>
<!-- Maximum number of searchers that may be warming in the background
concurrently. An error is returned if this limit is exceeded. Recommend
1-2 for read-only slaves, higher for masters w/o cache warming. -->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>
<!--

11
solr/contrib/ltr/example/solrconfig.xml
View File

@ -609,17 +609,6 @@
-->
<useColdSearcher>false</useColdSearcher>
<!-- Max Warming Searchers
Maximum number of searchers that may be warming in the
background concurrently. An error is returned if this limit
is exceeded.
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>

11
solr/contrib/morphlines-core/src/test-files/solr/solrcloud/conf/solrconfig.xml
View File

@ -606,17 +606,6 @@
-->
<useColdSearcher>false</useColdSearcher>
<!-- Max Warming Searchers
Maximum number of searchers that may be warming in the
background concurrently. An error is returned if this limit
is exceeded.
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>4</maxWarmingSearchers>
</query>

2
solr/core/src/java/org/apache/solr/core/SolrConfig.java
View File

@ -287,7 +287,7 @@ public class SolrConfig extends Config implements MapSerializable {
} else {
jmxConfig = new JmxConfiguration(false, null, null, null);
}
maxWarmingSearchers = getInt("query/maxWarmingSearchers", Integer.MAX_VALUE);
maxWarmingSearchers = getInt("query/maxWarmingSearchers", 1);
slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1);
for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin);

84
solr/core/src/java/org/apache/solr/core/SolrCore.java
View File

@ -1868,51 +1868,57 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
// if it isn't necessary.
synchronized (searcherLock) {
// see if we can return the current searcher
if (_searcher!=null && !forceNew) {
if (returnSearcher) {
_searcher.incref();
return _searcher;
} else {
return null;
for(;;) { // this loop is so w can retry in the event that we exceed maxWarmingSearchers
// see if we can return the current searcher
if (_searcher != null && !forceNew) {
if (returnSearcher) {
_searcher.incref();
return _searcher;
} else {
return null;
}
}
}
// check to see if we can wait for someone else's searcher to be set
if (onDeckSearchers>0 && !forceNew && _searcher==null) {
try {
searcherLock.wait();
} catch (InterruptedException e) {
log.info(SolrException.toStr(e));
// check to see if we can wait for someone else's searcher to be set
if (onDeckSearchers > 0 && !forceNew && _searcher == null) {
try {
searcherLock.wait();
} catch (InterruptedException e) {
log.info(SolrException.toStr(e));
}
}
}
// check again: see if we can return right now
if (_searcher!=null && !forceNew) {
if (returnSearcher) {
_searcher.incref();
return _searcher;
} else {
return null;
// check again: see if we can return right now
if (_searcher != null && !forceNew) {
if (returnSearcher) {
_searcher.incref();
return _searcher;
} else {
return null;
}
}
}
// At this point, we know we need to open a new searcher...
// first: increment count to signal other threads that we are
// opening a new searcher.
onDeckSearchers++;
if (onDeckSearchers < 1) {
// should never happen... just a sanity check
log.error(logid+"ERROR!!! onDeckSearchers is " + onDeckSearchers);
onDeckSearchers=1; // reset
} else if (onDeckSearchers > maxWarmingSearchers) {
onDeckSearchers--;
String msg="Error opening new searcher. exceeded limit of maxWarmingSearchers="+maxWarmingSearchers + ", try again later.";
log.warn(logid+""+ msg);
// HTTP 503==service unavailable, or 409==Conflict
throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,msg);
} else if (onDeckSearchers > 1) {
log.warn(logid+"PERFORMANCE WARNING: Overlapping onDeckSearchers=" + onDeckSearchers);
// At this point, we know we need to open a new searcher...
// first: increment count to signal other threads that we are
// opening a new searcher.
onDeckSearchers++;
if (onDeckSearchers < 1) {
// should never happen... just a sanity check
log.error(logid + "ERROR!!! onDeckSearchers is " + onDeckSearchers);
onDeckSearchers = 1; // reset
} else if (onDeckSearchers > maxWarmingSearchers) {
onDeckSearchers--;
try {
searcherLock.wait();
} catch (InterruptedException e) {
log.info(SolrException.toStr(e));
}
continue; // go back to the top of the loop and retry
} else if (onDeckSearchers > 1) {
log.warn(logid + "PERFORMANCE WARNING: Overlapping onDeckSearchers=" + onDeckSearchers);
}
break; // I can now exit the loop and proceed to open a searcher
}
}

1
solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml
View File

@ -57,7 +57,6 @@
<queryResultWindowSize>20</queryResultWindowSize>
<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
<useColdSearcher>false</useColdSearcher>
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>

2
solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
View File

@ -513,7 +513,7 @@ public class TestRealTimeGet extends TestRTGBase {
final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
int nWriteThreads = 5 + random().nextInt(25);
final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
final int maxConcurrentCommits = nWriteThreads; // number of committers at a time...
// query variables
final int percentRealtimeQuery = 60;

2
solr/core/src/test/org/apache/solr/search/TestStressLucene.java
View File

@ -55,7 +55,7 @@ public class TestStressLucene extends TestRTGBase {
final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
int nWriteThreads = 5 + random().nextInt(25);
final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
final int maxConcurrentCommits = nWriteThreads;
final AtomicLong operations = new AtomicLong(100000); // number of query operations to perform in total
int nReadThreads = 5 + random().nextInt(25);

2
solr/core/src/test/org/apache/solr/search/TestStressRecovery.java
View File

@ -70,7 +70,7 @@ public class TestStressRecovery extends TestRTGBase {
final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
int nWriteThreads = 2 + random().nextInt(10); // fewer write threads to give recovery thread more of a chance
final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
final int maxConcurrentCommits = nWriteThreads;
// query variables
final int percentRealtimeQuery = 75;

5
solr/core/src/test/org/apache/solr/search/TestStressReorder.java
View File

@ -68,8 +68,7 @@ public class TestStressReorder extends TestRTGBase {
final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
int nWriteThreads = 5 + random().nextInt(25);
final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
final int maxConcurrentCommits = nWriteThreads;
// query variables
final int percentRealtimeQuery = 75;
final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total
@ -84,7 +83,7 @@ public class TestStressReorder extends TestRTGBase {
final int ndocs = 1;
int nWriteThreads = 2;
final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
final int maxConcurrentCommits = nWriteThreads;
// query variables
final int percentRealtimeQuery = 101;

4
solr/core/src/test/org/apache/solr/search/TestStressUserVersions.java
View File

@ -74,7 +74,7 @@ public class TestStressUserVersions extends TestRTGBase {
final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
int nWriteThreads = 5 + random().nextInt(25);
final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
final int maxConcurrentCommits = nWriteThreads;
// query variables
final int percentRealtimeQuery = 75;
@ -90,7 +90,7 @@ public class TestStressUserVersions extends TestRTGBase {
final int ndocs = 1;
int nWriteThreads = 2;
final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
final int maxConcurrentCommits = nWriteThreads;
// query variables
final int percentRealtimeQuery = 101;

2
solr/core/src/test/org/apache/solr/search/TestStressVersions.java
View File

@ -56,7 +56,7 @@ public class TestStressVersions extends TestRTGBase {
final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
int nWriteThreads = 5 + random().nextInt(25);
final int maxConcurrentCommits = nWriteThreads; // number of committers at a time... it should be <= maxWarmingSearchers
final int maxConcurrentCommits = nWriteThreads;
// query variables
final int percentRealtimeQuery = 75;

11
solr/example/example-DIH/solr/db/conf/solrconfig.xml
View File

@ -581,17 +581,6 @@
-->
<useColdSearcher>false</useColdSearcher>
<!-- Max Warming Searchers
Maximum number of searchers that may be warming in the
background concurrently. An error is returned if this limit
is exceeded.
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>

11
solr/example/example-DIH/solr/mail/conf/solrconfig.xml
View File

@ -584,17 +584,6 @@
-->
<useColdSearcher>false</useColdSearcher>
<!-- Max Warming Searchers
Maximum number of searchers that may be warming in the
background concurrently. An error is returned if this limit
is exceeded.
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>

11
solr/example/example-DIH/solr/rss/conf/solrconfig.xml
View File

@ -581,17 +581,6 @@
-->
<useColdSearcher>false</useColdSearcher>
<!-- Max Warming Searchers
Maximum number of searchers that may be warming in the
background concurrently. An error is returned if this limit
is exceeded.
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>

11
solr/example/example-DIH/solr/solr/conf/solrconfig.xml
View File

@ -581,17 +581,6 @@
-->
<useColdSearcher>false</useColdSearcher>
<!-- Max Warming Searchers
Maximum number of searchers that may be warming in the
background concurrently. An error is returned if this limit
is exceeded.
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>

11
solr/example/example-DIH/solr/tika/conf/solrconfig.xml
View File

@ -584,17 +584,6 @@
-->
<useColdSearcher>false</useColdSearcher>
<!-- Max Warming Searchers
Maximum number of searchers that may be warming in the
background concurrently. An error is returned if this limit
is exceeded.
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>

11
solr/example/files/conf/solrconfig.xml
View File

@ -579,17 +579,6 @@
-->
<useColdSearcher>false</useColdSearcher>
<!-- Max Warming Searchers
Maximum number of searchers that may be warming in the
background concurrently. An error is returned if this limit
is exceeded.
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>

11
solr/server/solr/configsets/basic_configs/conf/solrconfig.xml
View File

@ -599,17 +599,6 @@
-->
<useColdSearcher>false</useColdSearcher>
<!-- Max Warming Searchers
Maximum number of searchers that may be warming in the
background concurrently. An error is returned if this limit
is exceeded.
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>

11
solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml
View File

@ -599,17 +599,6 @@
-->
<useColdSearcher>false</useColdSearcher>
<!-- Max Warming Searchers
Maximum number of searchers that may be warming in the
background concurrently. An error is returned if this limit
is exceeded.
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>

11
solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
View File

@ -611,17 +611,6 @@
-->
<useColdSearcher>false</useColdSearcher>
<!-- Max Warming Searchers
Maximum number of searchers that may be warming in the
background concurrently. An error is returned if this limit
is exceeded.
Recommend values of 1-2 for read-only slaves, higher for
masters w/o cache warming.
-->
<maxWarmingSearchers>2</maxWarmingSearchers>
</query>

3
solr/solrj/src/test/org/apache/solr/client/solrj/LargeVolumeTestBase.java
View File

@ -60,7 +60,8 @@ public abstract class LargeVolumeTestBase extends SolrJettyTestBase
// some of the commits could have failed because maxWarmingSearchers exceeded,
// so do a final commit to make sure everything is visible.
client.commit();
// This should no longer be true as of SOLR-9712 (Solr 6.4)
// client.commit();
query(threadCount * numdocs);
log.info("done");

1
solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTestBase.java
View File

@ -38,7 +38,6 @@ abstract public class SolrExampleTestBase extends AbstractSolrTestCase {
@Override
public void setUp() throws Exception {
ignoreException("maxWarmingSearchers");
super.setUp();
// this sets the property for jetty starting SolrDispatchFilter

2
solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java
View File

@ -109,8 +109,6 @@ abstract public class SolrJettyTestBase extends SolrTestCaseJ4
nodeProps.setProperty("coreRootDirectory", coresDir.toString());
nodeProps.setProperty("configSetBaseDir", solrHome);
ignoreException("maxWarmingSearchers");
jetty = new JettySolrRunner(solrHome, nodeProps, jettyConfig);
jetty.start();
port = jetty.getLocalPort();