SOLR-310: bound pending deletes

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@572831 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -130,7 +130,11 @@ New Features
 Changes in runtime behavior
 
 Optimizations
- 1.  SOLR-276: improve JSON writer speed. (yonik)
+ 1. SOLR-276: improve JSON writer speed. (yonik)
+ 
+ 2. SOLR-310: bound and reduce memory usage by providing <maxBufferedDeletes> parameter,
+    which flushes deleted without forcing the user to use <commit/> for this purpose.
+    (klaas) 
 
 Bug Fixes
  1. Make TextField respect sortMissingFirst and sortMissingLast fields.

example/solr/conf/solrconfig.xml

@@ -82,7 +82,16 @@
          org.apache.solr.(search|update|request|core|analysis)
      -->
 
-    <!-- autocommit pending docs if certain criteria are met 
+    <!-- Limit the number of deletions Solr will buffer during doc updating.
+        
+        Setting this lower can help bound memory use during indexing.
+    -->
+    <maxPendingDeletes>100000</maxPendingDeletes>
+
+    <!-- Perform a <commit/> automatically under certain conditions:
+
+         maxDocs - number of updates since last commit is greater than this
+         maxTime - oldest uncommited update (in ms) is this long ago
     <autoCommit> 
       <maxDocs>10000</maxDocs>
       <maxTime>1000</maxTime> 

src/java/org/apache/solr/update/DirectUpdateHandler2.java

@@ -143,6 +143,7 @@ public class DirectUpdateHandler2 extends UpdateHandler {
   // The key is the id, the value (Integer) is the number
   // of docs to save (delete all except the last "n" added)
   protected final Map<String,Integer> pset;
+  protected int maxPendingDeletes = SolrConfig.config.getInt("updateHandler/maxPendingDeletes", -1);
 
   // commonly used constants for the count in the pset
   protected final static Integer ZERO = 0;
@@ -274,6 +275,17 @@ public class DirectUpdateHandler2 extends UpdateHandler {
         numDocsPending.incrementAndGet();
       }
     }
+    if (maxPendingDeletes > 0 && pset.size() > maxPendingDeletes) {
+      iwCommit.lock();
+      try {
+        // note: this may be entered multiple times since the synchro is 
+        // inside the if(), but doDeletions() is a cheap no-op if it has
+        // already executed
+        doDeletions();
+      } finally {
+        iwCommit.unlock();
+      }
+    }    
     return rc;
   }
 

src/test/org/apache/solr/update/AutoCommitTest.java

@@ -176,9 +176,40 @@ public class AutoCommitTest extends AbstractSolrTestCase {
     Thread.sleep( 1000 );
     req.setContentStreams( toContentStreams(
       adoc("id", "531", "field_t", "what's inside?", "subject", "info"), null ) );
+    handler.handleRequest( req, rsp );
 
     assertQ("now it should", req("id:500") ,"//result[@numFound=1]" );
     assertQ("but not this", req("id:531") ,"//result[@numFound=0]" );
     assertEquals( 3, tracker.autoCommitCount );
   }
+
+  public void testMaxPending() throws Exception {
+    
+    DirectUpdateHandler2 updater = (DirectUpdateHandler2)SolrCore.getSolrCore().getUpdateHandler();
+    updater.maxPendingDeletes = 14;
+    
+    XmlUpdateRequestHandler handler = new XmlUpdateRequestHandler();
+    handler.init( null );
+    
+    SolrCore core = SolrCore.getSolrCore();
+    MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
+    
+    // Add a single document
+    SolrQueryResponse rsp = new SolrQueryResponse();
+    SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
+    for( int i=0; i<14; i++ ) {
+      req.setContentStreams( toContentStreams(
+        adoc("id", "A"+i, "subject", "info" ), null ) );
+      handler.handleRequest( req, rsp );
+    }
+    assertEquals(updater.numDocsPending.get(), 14);
+
+    req.setContentStreams( toContentStreams(
+        adoc("id", "A14", "subject", "info" ), null ) );
+    handler.handleRequest( req, rsp );
+
+    assertEquals(updater.numDocsPending.get(), 0);
+    assertEquals(updater.commitCommands.get(), 0);
+  }
+
 }