mirror of https://github.com/apache/lucene.git
SOLR-310: bound pending deletes
git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@572831 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
3954767c5e
commit
b9da3b2ace
|
@ -130,7 +130,11 @@ New Features
|
|||
Changes in runtime behavior
|
||||
|
||||
Optimizations
|
||||
1. SOLR-276: improve JSON writer speed. (yonik)
|
||||
1. SOLR-276: improve JSON writer speed. (yonik)
|
||||
|
||||
2. SOLR-310: bound and reduce memory usage by providing <maxBufferedDeletes> parameter,
|
||||
which flushes deleted without forcing the user to use <commit/> for this purpose.
|
||||
(klaas)
|
||||
|
||||
Bug Fixes
|
||||
1. Make TextField respect sortMissingFirst and sortMissingLast fields.
|
||||
|
|
|
@ -82,10 +82,19 @@
|
|||
org.apache.solr.(search|update|request|core|analysis)
|
||||
-->
|
||||
|
||||
<!-- autocommit pending docs if certain criteria are met
|
||||
<!-- Limit the number of deletions Solr will buffer during doc updating.
|
||||
|
||||
Setting this lower can help bound memory use during indexing.
|
||||
-->
|
||||
<maxPendingDeletes>100000</maxPendingDeletes>
|
||||
|
||||
<!-- Perform a <commit/> automatically under certain conditions:
|
||||
|
||||
maxDocs - number of updates since last commit is greater than this
|
||||
maxTime - oldest uncommited update (in ms) is this long ago
|
||||
<autoCommit>
|
||||
<maxDocs>10000</maxDocs>
|
||||
<maxTime>1000</maxTime>
|
||||
<maxTime>1000</maxTime>
|
||||
</autoCommit>
|
||||
-->
|
||||
|
||||
|
|
|
@ -143,6 +143,7 @@ public class DirectUpdateHandler2 extends UpdateHandler {
|
|||
// The key is the id, the value (Integer) is the number
|
||||
// of docs to save (delete all except the last "n" added)
|
||||
protected final Map<String,Integer> pset;
|
||||
protected int maxPendingDeletes = SolrConfig.config.getInt("updateHandler/maxPendingDeletes", -1);
|
||||
|
||||
// commonly used constants for the count in the pset
|
||||
protected final static Integer ZERO = 0;
|
||||
|
@ -274,6 +275,17 @@ public class DirectUpdateHandler2 extends UpdateHandler {
|
|||
numDocsPending.incrementAndGet();
|
||||
}
|
||||
}
|
||||
if (maxPendingDeletes > 0 && pset.size() > maxPendingDeletes) {
|
||||
iwCommit.lock();
|
||||
try {
|
||||
// note: this may be entered multiple times since the synchro is
|
||||
// inside the if(), but doDeletions() is a cheap no-op if it has
|
||||
// already executed
|
||||
doDeletions();
|
||||
} finally {
|
||||
iwCommit.unlock();
|
||||
}
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
|
|
@ -114,7 +114,7 @@ public class AutoCommitTest extends AbstractSolrTestCase {
|
|||
req.setContentStreams( toContentStreams(
|
||||
adoc("id", "B15", "subject", "info" ), null ) );
|
||||
handler.handleRequest( req, rsp );
|
||||
|
||||
|
||||
assertQ("should find one", req("id:B14") ,"//result[@numFound=1]" );
|
||||
assertEquals( 2, tracker.autoCommitCount );
|
||||
assertQ("should find none", req("id:B15") ,"//result[@numFound=0]" );
|
||||
|
@ -176,9 +176,40 @@ public class AutoCommitTest extends AbstractSolrTestCase {
|
|||
Thread.sleep( 1000 );
|
||||
req.setContentStreams( toContentStreams(
|
||||
adoc("id", "531", "field_t", "what's inside?", "subject", "info"), null ) );
|
||||
handler.handleRequest( req, rsp );
|
||||
|
||||
assertQ("now it should", req("id:500") ,"//result[@numFound=1]" );
|
||||
assertQ("but not this", req("id:531") ,"//result[@numFound=0]" );
|
||||
assertEquals( 3, tracker.autoCommitCount );
|
||||
}
|
||||
|
||||
public void testMaxPending() throws Exception {
|
||||
|
||||
DirectUpdateHandler2 updater = (DirectUpdateHandler2)SolrCore.getSolrCore().getUpdateHandler();
|
||||
updater.maxPendingDeletes = 14;
|
||||
|
||||
XmlUpdateRequestHandler handler = new XmlUpdateRequestHandler();
|
||||
handler.init( null );
|
||||
|
||||
SolrCore core = SolrCore.getSolrCore();
|
||||
MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
|
||||
|
||||
// Add a single document
|
||||
SolrQueryResponse rsp = new SolrQueryResponse();
|
||||
SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
|
||||
for( int i=0; i<14; i++ ) {
|
||||
req.setContentStreams( toContentStreams(
|
||||
adoc("id", "A"+i, "subject", "info" ), null ) );
|
||||
handler.handleRequest( req, rsp );
|
||||
}
|
||||
assertEquals(updater.numDocsPending.get(), 14);
|
||||
|
||||
req.setContentStreams( toContentStreams(
|
||||
adoc("id", "A14", "subject", "info" ), null ) );
|
||||
handler.handleRequest( req, rsp );
|
||||
|
||||
assertEquals(updater.numDocsPending.get(), 0);
|
||||
assertEquals(updater.commitCommands.get(), 0);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue