mirror of https://github.com/apache/lucene.git
SOLR-310: bound pending deletes
git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@572831 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
3954767c5e
commit
b9da3b2ace
|
@ -130,7 +130,11 @@ New Features
|
||||||
Changes in runtime behavior
|
Changes in runtime behavior
|
||||||
|
|
||||||
Optimizations
|
Optimizations
|
||||||
1. SOLR-276: improve JSON writer speed. (yonik)
|
1. SOLR-276: improve JSON writer speed. (yonik)
|
||||||
|
|
||||||
|
2. SOLR-310: bound and reduce memory usage by providing <maxBufferedDeletes> parameter,
|
||||||
|
which flushes deleted without forcing the user to use <commit/> for this purpose.
|
||||||
|
(klaas)
|
||||||
|
|
||||||
Bug Fixes
|
Bug Fixes
|
||||||
1. Make TextField respect sortMissingFirst and sortMissingLast fields.
|
1. Make TextField respect sortMissingFirst and sortMissingLast fields.
|
||||||
|
|
|
@ -82,10 +82,19 @@
|
||||||
org.apache.solr.(search|update|request|core|analysis)
|
org.apache.solr.(search|update|request|core|analysis)
|
||||||
-->
|
-->
|
||||||
|
|
||||||
<!-- autocommit pending docs if certain criteria are met
|
<!-- Limit the number of deletions Solr will buffer during doc updating.
|
||||||
|
|
||||||
|
Setting this lower can help bound memory use during indexing.
|
||||||
|
-->
|
||||||
|
<maxPendingDeletes>100000</maxPendingDeletes>
|
||||||
|
|
||||||
|
<!-- Perform a <commit/> automatically under certain conditions:
|
||||||
|
|
||||||
|
maxDocs - number of updates since last commit is greater than this
|
||||||
|
maxTime - oldest uncommited update (in ms) is this long ago
|
||||||
<autoCommit>
|
<autoCommit>
|
||||||
<maxDocs>10000</maxDocs>
|
<maxDocs>10000</maxDocs>
|
||||||
<maxTime>1000</maxTime>
|
<maxTime>1000</maxTime>
|
||||||
</autoCommit>
|
</autoCommit>
|
||||||
-->
|
-->
|
||||||
|
|
||||||
|
|
|
@ -143,6 +143,7 @@ public class DirectUpdateHandler2 extends UpdateHandler {
|
||||||
// The key is the id, the value (Integer) is the number
|
// The key is the id, the value (Integer) is the number
|
||||||
// of docs to save (delete all except the last "n" added)
|
// of docs to save (delete all except the last "n" added)
|
||||||
protected final Map<String,Integer> pset;
|
protected final Map<String,Integer> pset;
|
||||||
|
protected int maxPendingDeletes = SolrConfig.config.getInt("updateHandler/maxPendingDeletes", -1);
|
||||||
|
|
||||||
// commonly used constants for the count in the pset
|
// commonly used constants for the count in the pset
|
||||||
protected final static Integer ZERO = 0;
|
protected final static Integer ZERO = 0;
|
||||||
|
@ -274,6 +275,17 @@ public class DirectUpdateHandler2 extends UpdateHandler {
|
||||||
numDocsPending.incrementAndGet();
|
numDocsPending.incrementAndGet();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (maxPendingDeletes > 0 && pset.size() > maxPendingDeletes) {
|
||||||
|
iwCommit.lock();
|
||||||
|
try {
|
||||||
|
// note: this may be entered multiple times since the synchro is
|
||||||
|
// inside the if(), but doDeletions() is a cheap no-op if it has
|
||||||
|
// already executed
|
||||||
|
doDeletions();
|
||||||
|
} finally {
|
||||||
|
iwCommit.unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
return rc;
|
return rc;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -114,7 +114,7 @@ public class AutoCommitTest extends AbstractSolrTestCase {
|
||||||
req.setContentStreams( toContentStreams(
|
req.setContentStreams( toContentStreams(
|
||||||
adoc("id", "B15", "subject", "info" ), null ) );
|
adoc("id", "B15", "subject", "info" ), null ) );
|
||||||
handler.handleRequest( req, rsp );
|
handler.handleRequest( req, rsp );
|
||||||
|
|
||||||
assertQ("should find one", req("id:B14") ,"//result[@numFound=1]" );
|
assertQ("should find one", req("id:B14") ,"//result[@numFound=1]" );
|
||||||
assertEquals( 2, tracker.autoCommitCount );
|
assertEquals( 2, tracker.autoCommitCount );
|
||||||
assertQ("should find none", req("id:B15") ,"//result[@numFound=0]" );
|
assertQ("should find none", req("id:B15") ,"//result[@numFound=0]" );
|
||||||
|
@ -176,9 +176,40 @@ public class AutoCommitTest extends AbstractSolrTestCase {
|
||||||
Thread.sleep( 1000 );
|
Thread.sleep( 1000 );
|
||||||
req.setContentStreams( toContentStreams(
|
req.setContentStreams( toContentStreams(
|
||||||
adoc("id", "531", "field_t", "what's inside?", "subject", "info"), null ) );
|
adoc("id", "531", "field_t", "what's inside?", "subject", "info"), null ) );
|
||||||
|
handler.handleRequest( req, rsp );
|
||||||
|
|
||||||
assertQ("now it should", req("id:500") ,"//result[@numFound=1]" );
|
assertQ("now it should", req("id:500") ,"//result[@numFound=1]" );
|
||||||
assertQ("but not this", req("id:531") ,"//result[@numFound=0]" );
|
assertQ("but not this", req("id:531") ,"//result[@numFound=0]" );
|
||||||
assertEquals( 3, tracker.autoCommitCount );
|
assertEquals( 3, tracker.autoCommitCount );
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testMaxPending() throws Exception {
|
||||||
|
|
||||||
|
DirectUpdateHandler2 updater = (DirectUpdateHandler2)SolrCore.getSolrCore().getUpdateHandler();
|
||||||
|
updater.maxPendingDeletes = 14;
|
||||||
|
|
||||||
|
XmlUpdateRequestHandler handler = new XmlUpdateRequestHandler();
|
||||||
|
handler.init( null );
|
||||||
|
|
||||||
|
SolrCore core = SolrCore.getSolrCore();
|
||||||
|
MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
|
||||||
|
|
||||||
|
// Add a single document
|
||||||
|
SolrQueryResponse rsp = new SolrQueryResponse();
|
||||||
|
SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
|
||||||
|
for( int i=0; i<14; i++ ) {
|
||||||
|
req.setContentStreams( toContentStreams(
|
||||||
|
adoc("id", "A"+i, "subject", "info" ), null ) );
|
||||||
|
handler.handleRequest( req, rsp );
|
||||||
|
}
|
||||||
|
assertEquals(updater.numDocsPending.get(), 14);
|
||||||
|
|
||||||
|
req.setContentStreams( toContentStreams(
|
||||||
|
adoc("id", "A14", "subject", "info" ), null ) );
|
||||||
|
handler.handleRequest( req, rsp );
|
||||||
|
|
||||||
|
assertEquals(updater.numDocsPending.get(), 0);
|
||||||
|
assertEquals(updater.commitCommands.get(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue