Merged /lucene/dev/trunk:r1441423-1441567

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1441570 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -68,6 +68,9 @@ New Features
 
 * SOLR-2827: RegexpBoost Update Processor (janhoy)
 
+* SOLR-4370: Allow configuring commitWithin to do hard commits. 
+  (Mark Miller, Senthuran Sivananthan)
+
 Bug Fixes
 ----------------------
 

solr/core/src/java/org/apache/solr/core/SolrConfig.java

@@ -238,7 +238,8 @@ public class SolrConfig extends Config {
             getBool("updateHandler/autoCommit/openSearcher",true),
             getInt("updateHandler/commitIntervalLowerBound",-1),
             getInt("updateHandler/autoSoftCommit/maxDocs",-1),
-            getInt("updateHandler/autoSoftCommit/maxTime",-1));
+            getInt("updateHandler/autoSoftCommit/maxTime",-1),
+            getBool("updateHandler/commitWithin/softCommit",true));
   }
 
   private void loadPluginInfo(Class clazz, String tag, boolean requireName, boolean requireClass) {
@@ -402,6 +403,7 @@ public class SolrConfig extends Config {
     public final int autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound,
         autoSoftCommmitMaxDocs,autoSoftCommmitMaxTime;
     public final boolean openSearcher;  // is opening a new searcher part of hard autocommit?
+    public final boolean commitWithinSoftCommit;
 
     /**
      * @param autoCommmitMaxDocs set -1 as default
@@ -409,7 +411,7 @@ public class SolrConfig extends Config {
      * @param commitIntervalLowerBound set -1 as default
      */
     public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, boolean openSearcher, int commitIntervalLowerBound,
-        int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime) {
+        int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean commitWithinSoftCommit) {
       this.className = className;
       this.autoCommmitMaxDocs = autoCommmitMaxDocs;
       this.autoCommmitMaxTime = autoCommmitMaxTime;
@@ -418,6 +420,8 @@ public class SolrConfig extends Config {
       
       this.autoSoftCommmitMaxDocs = autoSoftCommmitMaxDocs;
       this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime;
+      
+      this.commitWithinSoftCommit = commitWithinSoftCommit;
     } 
   }
 

solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java

@@ -91,6 +91,8 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
   // tracks when auto-commit should occur
   protected final CommitTracker commitTracker;
   protected final CommitTracker softCommitTracker;
+  
+  protected boolean commitWithinSoftCommit;
 
   public DirectUpdateHandler2(SolrCore core) {
     super(core);
@@ -106,6 +108,8 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
     int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs", -1);
     int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime", -1);
     softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true);
+    
+    commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
   }
   
   public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) {
@@ -126,6 +130,8 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
     if (this.ulog != null) {
       this.ulog.init(this, core);
     }
+    
+    commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
   }
 
   private void deleteAll() throws IOException {
@@ -229,8 +235,13 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
         }
         
         if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
-          commitTracker.addedDocument(-1);
+          if (commitWithinSoftCommit) {
-          softCommitTracker.addedDocument(cmd.commitWithin);
+            commitTracker.addedDocument(-1);
+            softCommitTracker.addedDocument(cmd.commitWithin);
+          } else {
+            softCommitTracker.addedDocument(-1);
+            commitTracker.addedDocument(cmd.commitWithin);
+          }
         }
         
         rc = 1;
@@ -252,7 +263,11 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
   
   private void updateDeleteTrackers(DeleteUpdateCommand cmd) {
     if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
-      softCommitTracker.deletedDocument(cmd.commitWithin);
+      if (commitWithinSoftCommit) {
+        softCommitTracker.deletedDocument(cmd.commitWithin);
+      } else {
+        commitTracker.deletedDocument(cmd.commitWithin);
+      }
       
       if (commitTracker.getTimeUpperBound() > 0) {
         commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());

solr/core/src/test-files/solr/collection1/conf/solrconfig.xml

@@ -97,6 +97,10 @@
     <updateLog enable="${enable.update.log:false}">
   	  <str name="dir">${solr.ulog.dir:}</str>
     </updateLog> 
+    
+    <commitWithin>
+      <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+    </commitWithin>
 
   </updateHandler>
 

solr/core/src/test/org/apache/solr/update/AutoCommitTest.java

@@ -286,7 +286,7 @@ public class AutoCommitTest extends AbstractSolrTestCase {
     
     MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
     
-    // Add a single document with commitWithin == 1 second
+    // Add a single document with commitWithin == 2 second
     SolrQueryResponse rsp = new SolrQueryResponse();
     SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
     req.setContentStreams( toContentStreams(

solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java

@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update;
+
+import java.util.HashMap;
+
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.UpdateRequestHandler;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.util.AbstractSolrTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+@Slow
+public class HardAutoCommitTest extends AbstractSolrTestCase {
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    System.setProperty("solr.commitwithin.softcommit", "false");
+    initCore("solrconfig.xml", "schema.xml");
+  }
+  
+  @AfterClass
+  public static void afterClass() {
+    System.clearProperty("solr.commitwithin.softcommit");
+  }
+
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    clearIndex();
+    // reload the core to clear stats
+    h.getCoreContainer().reload(h.getCore().getName());
+  }
+
+
+  public void testCommitWithin() throws Exception {
+    SolrCore core = h.getCore();
+    
+    NewSearcherListener trigger = new NewSearcherListener();    
+    core.registerNewSearcherListener(trigger);
+    DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler();
+    CommitTracker tracker = updater.commitTracker;
+    tracker.setTimeUpperBound(0);
+    tracker.setDocsUpperBound(-1);
+    
+    UpdateRequestHandler handler = new UpdateRequestHandler();
+    handler.init( null );
+    
+    MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
+    
+    // Add a single document with commitWithin == 2 second
+    SolrQueryResponse rsp = new SolrQueryResponse();
+    SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
+    req.setContentStreams( AutoCommitTest.toContentStreams(
+      adoc(2000, "id", "529", "field_t", "what's inside?", "subject", "info"), null ) );
+    trigger.reset();
+    handler.handleRequest( req, rsp );
+
+    // Check it isn't in the index
+    assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" );
+    
+    // Wait longer than the commitWithin time
+    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
+
+    // Add one document without commitWithin
+    req.setContentStreams( AutoCommitTest.toContentStreams(
+        adoc("id", "530", "field_t", "what's inside?", "subject", "info"), null ) );
+      trigger.reset();
+      handler.handleRequest( req, rsp );
+      
+    // Check it isn't in the index
+    assertQ("shouldn't find any", req("id:530") ,"//result[@numFound=0]" );
+    
+    // Delete one document with commitWithin
+    req.setContentStreams( AutoCommitTest.toContentStreams(
+      delI("529", "commitWithin", "1000"), null ) );
+    trigger.reset();
+    handler.handleRequest( req, rsp );
+      
+    // Now make sure we can find it
+    assertQ("should find one", req("id:529") ,"//result[@numFound=1]" );
+    
+    // Wait for the commit to happen
+    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
+    
+    // Now we shouldn't find it
+    assertQ("should find none", req("id:529") ,"//result[@numFound=0]" );
+    // ... but we should find the new one
+    assertQ("should find one", req("id:530") ,"//result[@numFound=1]" );
+    
+    trigger.reset();
+    
+    // now make the call 10 times really fast and make sure it 
+    // only commits once
+    req.setContentStreams( AutoCommitTest.toContentStreams(
+        adoc(2000, "id", "500" ), null ) );
+    for( int i=0;i<10; i++ ) {
+      handler.handleRequest( req, rsp );
+    }
+    assertQ("should not be there yet", req("id:500") ,"//result[@numFound=0]" );
+    
+    // the same for the delete
+    req.setContentStreams( AutoCommitTest.toContentStreams(
+        delI("530", "commitWithin", "1000"), null ) );
+    for( int i=0;i<10; i++ ) {
+      handler.handleRequest( req, rsp );
+    }
+    assertQ("should be there", req("id:530") ,"//result[@numFound=1]" );
+    
+    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
+    assertQ("should be there", req("id:500") ,"//result[@numFound=1]" );
+    assertQ("should not be there", req("id:530") ,"//result[@numFound=0]" );
+    
+    assertEquals(3, tracker.getCommitCount());
+  }
+
+}