mirror of https://github.com/apache/lucene.git
SOLR-6650: Add optional slow request logging at WARN level; this closes #101
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1634086 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
e7783b8928
commit
23eb9cbfcc
solr
CHANGES.txt
core/src
java/org/apache/solr
test-files/solr/collection1/conf
test/org/apache/solr
example/solr/collection1/conf
|
@ -187,6 +187,9 @@ New Features
|
|||
* SOLR-6633: /update/json/docs path can now save the underlying json doc asa string field
|
||||
and better support added to the default example (Noble Paul)
|
||||
|
||||
* SOLR-6650: Add optional slow request logging at WARN level
|
||||
(Jessica Cheng Mallet via Timothy Potter)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
|
|
|
@ -241,6 +241,7 @@ public class SolrConfig extends Config {
|
|||
jmxConfig = new JmxConfiguration(false, null, null, null);
|
||||
}
|
||||
maxWarmingSearchers = getInt("query/maxWarmingSearchers",Integer.MAX_VALUE);
|
||||
slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", 1000);
|
||||
|
||||
loadPluginInfo(SolrRequestHandler.class,"requestHandler",
|
||||
REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK);
|
||||
|
@ -400,6 +401,7 @@ public class SolrConfig extends Config {
|
|||
public final boolean useColdSearcher;
|
||||
public final Version luceneMatchVersion;
|
||||
protected String dataDir;
|
||||
public final int slowQueryThresholdMillis; // threshold above which a query is considered slow
|
||||
|
||||
//JMX configuration
|
||||
public final JmxConfiguration jmxConfig;
|
||||
|
|
|
@ -666,6 +666,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
|
|||
this.solrConfig = null;
|
||||
this.startTime = System.currentTimeMillis();
|
||||
this.maxWarmingSearchers = 2; // we don't have a config yet, just pick a number.
|
||||
this.slowQueryThresholdMillis = 1000;
|
||||
this.resourceLoader = null;
|
||||
this.updateHandler = null;
|
||||
this.isReloaded = true;
|
||||
|
@ -766,6 +767,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
|
|||
this.dataDir = dataDir;
|
||||
this.startTime = System.currentTimeMillis();
|
||||
this.maxWarmingSearchers = config.maxWarmingSearchers;
|
||||
this.slowQueryThresholdMillis = config.slowQueryThresholdMillis;
|
||||
|
||||
booleanQueryMaxClauseCount();
|
||||
|
||||
|
@ -1357,6 +1359,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
|
|||
private Object searcherLock = new Object(); // the sync object for the searcher
|
||||
private ReentrantLock openSearcherLock = new ReentrantLock(true); // used to serialize opens/reopens for absolute ordering
|
||||
private final int maxWarmingSearchers; // max number of on-deck searchers allowed
|
||||
private final int slowQueryThresholdMillis; // threshold above which a query is considered slow
|
||||
|
||||
private RefCounted<SolrIndexSearcher> realtimeSearcher;
|
||||
private Callable<DirectoryReader> newReaderCreator;
|
||||
|
@ -1983,8 +1986,15 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
|
|||
handler.handleRequest(req,rsp);
|
||||
postDecorateResponse(handler, req, rsp);
|
||||
|
||||
if (log.isInfoEnabled() && rsp.getToLog().size() > 0) {
|
||||
log.info(rsp.getToLogAsString(logid));
|
||||
if (rsp.getToLog().size() > 0) {
|
||||
if (log.isInfoEnabled()) {
|
||||
log.info(rsp.getToLogAsString(logid));
|
||||
} else if (log.isWarnEnabled()) {
|
||||
final int qtime = (int)(rsp.getEndTime() - req.getStartTime());
|
||||
if (qtime >= slowQueryThresholdMillis) {
|
||||
log.warn(rsp.getToLogAsString(logid));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -49,11 +49,13 @@ import org.slf4j.LoggerFactory;
|
|||
public class LogUpdateProcessorFactory extends UpdateRequestProcessorFactory implements UpdateRequestProcessorFactory.RunAlways {
|
||||
|
||||
int maxNumToLog = 10;
|
||||
int slowUpdateThresholdMillis = 1000;
|
||||
@Override
|
||||
public void init( final NamedList args ) {
|
||||
if( args != null ) {
|
||||
SolrParams params = SolrParams.toSolrParams( args );
|
||||
maxNumToLog = params.getInt( "maxNumToLog", maxNumToLog );
|
||||
slowUpdateThresholdMillis = params.getInt("slowUpdateThresholdMillis", slowUpdateThresholdMillis);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -78,6 +80,7 @@ class LogUpdateProcessor extends UpdateRequestProcessor {
|
|||
private List<String> deletes;
|
||||
|
||||
private final int maxNumToLog;
|
||||
private final int slowUpdateThresholdMillis;
|
||||
|
||||
private final boolean logDebug = log.isDebugEnabled();//cache to avoid volatile-read
|
||||
|
||||
|
@ -88,6 +91,7 @@ class LogUpdateProcessor extends UpdateRequestProcessor {
|
|||
maxNumToLog = factory.maxNumToLog; // TODO: make configurable
|
||||
// TODO: make log level configurable as well, or is that overkill?
|
||||
// (ryan) maybe? I added it mostly to show that it *can* be configurable
|
||||
slowUpdateThresholdMillis = factory.slowUpdateThresholdMillis;
|
||||
|
||||
this.toLog = new SimpleOrderedMap<>();
|
||||
}
|
||||
|
@ -181,23 +185,32 @@ class LogUpdateProcessor extends UpdateRequestProcessor {
|
|||
// LOG A SUMMARY WHEN ALL DONE (INFO LEVEL)
|
||||
|
||||
if (log.isInfoEnabled()) {
|
||||
StringBuilder sb = new StringBuilder(rsp.getToLogAsString(req.getCore().getLogId()));
|
||||
|
||||
rsp.getToLog().clear(); // make it so SolrCore.exec won't log this again
|
||||
|
||||
// if id lists were truncated, show how many more there were
|
||||
if (adds != null && numAdds > maxNumToLog) {
|
||||
adds.add("... (" + numAdds + " adds)");
|
||||
}
|
||||
if (deletes != null && numDeletes > maxNumToLog) {
|
||||
deletes.add("... (" + numDeletes + " deletes)");
|
||||
}
|
||||
log.info(getLogStringAndClearRspToLog());
|
||||
} else if (log.isWarnEnabled()) {
|
||||
long elapsed = rsp.getEndTime() - req.getStartTime();
|
||||
|
||||
sb.append(toLog).append(" 0 ").append(elapsed);
|
||||
log.info(sb.toString());
|
||||
if (elapsed >= slowUpdateThresholdMillis) {
|
||||
log.warn(getLogStringAndClearRspToLog());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String getLogStringAndClearRspToLog() {
|
||||
StringBuilder sb = new StringBuilder(rsp.getToLogAsString(req.getCore().getLogId()));
|
||||
|
||||
rsp.getToLog().clear(); // make it so SolrCore.exec won't log this again
|
||||
|
||||
// if id lists were truncated, show how many more there were
|
||||
if (adds != null && numAdds > maxNumToLog) {
|
||||
adds.add("... (" + numAdds + " adds)");
|
||||
}
|
||||
if (deletes != null && numDeletes > maxNumToLog) {
|
||||
deletes.add("... (" + numDeletes + " deletes)");
|
||||
}
|
||||
long elapsed = rsp.getEndTime() - req.getStartTime();
|
||||
|
||||
sb.append(toLog).append(" 0 ").append(elapsed);
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -35,6 +35,7 @@
|
|||
</processor>
|
||||
<processor class="solr.LogUpdateProcessorFactory" >
|
||||
<int name="maxNumToLog">100</int>
|
||||
<int name="slowUpdateThresholdMillis">2000</int>
|
||||
</processor>
|
||||
<processor class="solr.CustomUpdateRequestProcessorFactory" >
|
||||
<lst name="name">
|
||||
|
|
|
@ -183,6 +183,8 @@
|
|||
</listener>
|
||||
-->
|
||||
|
||||
<slowQueryThresholdMillis>2000</slowQueryThresholdMillis>
|
||||
|
||||
</query>
|
||||
|
||||
<queryResponseWriter name="xml" default="true"
|
||||
|
|
|
@ -238,6 +238,14 @@ public class SolrCoreTest extends SolrTestCaseJ4 {
|
|||
assertNotNull("bean not registered", bean);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testConfiguration() throws Exception {
|
||||
assertEquals("wrong config for slowQueryThresholdMillis", 2000, solrConfig.slowQueryThresholdMillis);
|
||||
assertEquals("wrong config for maxBooleanClauses", 1024, solrConfig.booleanQueryMaxClauseCount);
|
||||
assertEquals("wrong config for enableLazyFieldLoading", true, solrConfig.enableLazyFieldLoading);
|
||||
assertEquals("wrong config for queryResultWindowSize", 10, solrConfig.queryResultWindowSize);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -53,10 +53,12 @@ public class UpdateRequestProcessorFactoryTest extends AbstractSolrTestCase {
|
|||
assertEquals("wrong factory at front of chain",
|
||||
LogUpdateProcessorFactory.class, first.getClass());
|
||||
LogUpdateProcessorFactory log = (LogUpdateProcessorFactory)first;
|
||||
assertEquals("wrong config for LogUpdateProcessorFactory",
|
||||
assertEquals("wrong config for LogUpdateProcessorFactory.maxNumToLog",
|
||||
100, log.maxNumToLog );
|
||||
|
||||
|
||||
assertEquals("wrong config for LogUpdateProcessorFactory.slowUpdateThresholdMillis",
|
||||
2000, log.slowUpdateThresholdMillis);
|
||||
|
||||
|
||||
UpdateRequestProcessorChain custom = core.getUpdateProcessingChain( null );
|
||||
CustomUpdateRequestProcessorFactory link = (CustomUpdateRequestProcessorFactory) custom.getFactories()[0];
|
||||
|
||||
|
|
|
@ -481,6 +481,21 @@
|
|||
-->
|
||||
<maxBooleanClauses>1024</maxBooleanClauses>
|
||||
|
||||
|
||||
<!-- Slow Query Threshold (in millis)
|
||||
|
||||
At high request rates, logging all requests can become a bottleneck
|
||||
and therefore INFO logging is often turned off. However, it is still
|
||||
useful to be able to set a latency threshold above which a request
|
||||
is considered "slow" and log that request at WARN level so we can
|
||||
easily identify slow queries.
|
||||
|
||||
This setting only applies if the logger for the
|
||||
org.apache.solr.core.SolrCore is set to WARN or greater.
|
||||
|
||||
-->
|
||||
<slowQueryThresholdMillis>1000</slowQueryThresholdMillis>
|
||||
|
||||
|
||||
<!-- Solr Internal Query Caches
|
||||
|
||||
|
|
Loading…
Reference in New Issue