From e3e644b28d360d3e623117283110a4c3bc7cc163 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Sat, 13 Aug 2011 21:06:01 +0000 Subject: [PATCH] SOLR-2565: Stop trying to avoid soft/hard commit together for now - it's more complicated than this - also allow name for CommitTracker. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1157425 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/solr/update/CommitTracker.java | 9 ++- .../solr/update/DirectUpdateHandler2.java | 19 ++--- .../apache/solr/update/AutoCommitTest.java | 81 ++++++++++++++++++- 3 files changed, 92 insertions(+), 17 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/CommitTracker.java b/solr/core/src/java/org/apache/solr/update/CommitTracker.java index 4360adcd7ef..8001136c335 100644 --- a/solr/core/src/java/org/apache/solr/update/CommitTracker.java +++ b/solr/core/src/java/org/apache/solr/update/CommitTracker.java @@ -60,9 +60,12 @@ final class CommitTracker implements Runnable { private boolean softCommit; private boolean waitSearcher; + + private String name; - public CommitTracker(SolrCore core, int docsUpperBound, int timeUpperBound, boolean waitSearcher, boolean softCommit) { + public CommitTracker(String name, SolrCore core, int docsUpperBound, int timeUpperBound, boolean waitSearcher, boolean softCommit) { this.core = core; + this.name = name; docsSinceCommit = 0; pending = null; @@ -72,7 +75,7 @@ final class CommitTracker implements Runnable { this.softCommit = softCommit; this.waitSearcher = waitSearcher; - SolrCore.log.info("AutoCommit: " + this); + SolrCore.log.info(name + " AutoCommit: " + this); } public void close() { @@ -121,7 +124,7 @@ final class CommitTracker implements Runnable { _scheduleCommitWithin(ctime); triggered = true; } - + return triggered; } diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java index 24f87afe348..c49b000c752 100644 --- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -83,11 +83,11 @@ public class DirectUpdateHandler2 extends UpdateHandler { .getUpdateHandlerInfo(); int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs; // getInt("updateHandler/autoCommit/maxDocs", -1); int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime; // getInt("updateHandler/autoCommit/maxTime", -1); - commitTracker = new CommitTracker(core, docsUpperBound, timeUpperBound, true, false); + commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, true, false); int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs", -1); int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime", -1); - softCommitTracker = new CommitTracker(core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true); + softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true); } public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) throws IOException { @@ -104,11 +104,11 @@ public class DirectUpdateHandler2 extends UpdateHandler { .getUpdateHandlerInfo(); int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs; // getInt("updateHandler/autoCommit/maxDocs", -1); int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime; // getInt("updateHandler/autoCommit/maxTime", -1); - commitTracker = new CommitTracker(core, docsUpperBound, timeUpperBound, true, false); + commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, true, false); int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs", -1); int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime", -1); - softCommitTracker = new CommitTracker(core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true); + softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true); } @@ -137,15 +137,8 @@ public class DirectUpdateHandler2 extends UpdateHandler { try { - boolean triggered = commitTracker.addedDocument( cmd.commitWithin ); - - if (!triggered) { - // if we hard commit, don't soft commit - softCommitTracker.addedDocument( cmd.commitWithin ); - } else { - // still inc softCommit - softCommitTracker.docsSinceCommit++; - } + commitTracker.addedDocument( cmd.commitWithin ); + softCommitTracker.addedDocument( cmd.commitWithin ); if (cmd.overwrite) { Term updateTerm; diff --git a/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java b/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java index c4b65287780..eac38d4ae89 100644 --- a/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java +++ b/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java @@ -282,7 +282,7 @@ public class AutoCommitTest extends AbstractSolrTestCase { assertTrue(trigger.waitForNewSearcher(10000)); assertQ("should find 10", req("*:*") ,"//result[@numFound=10]" ); - assertEquals( 1, softTracker.getCommitCount()); + assertEquals( 2, softTracker.getCommitCount()); assertEquals( 1, tracker.getCommitCount()); } @@ -362,4 +362,83 @@ public class AutoCommitTest extends AbstractSolrTestCase { assertQ("now it should", req("id:500") ,"//result[@numFound=1]" ); assertQ("but not this", req("id:531") ,"//result[@numFound=0]" ); } + + public void testSoftAndHardCommitMaxTime() throws Exception { + SolrCore core = h.getCore(); + NewSearcherListener trigger = new NewSearcherListener(); + core.registerNewSearcherListener(trigger); + DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler(); + CommitTracker hardTracker = updater.commitTracker; + CommitTracker softTracker = updater.softCommitTracker; + + // too low of a number can cause a slow host to commit before the test code checks that it + // isn't there... causing a failure at "shouldn't find any" + softTracker.timeUpperBound = 1000; + softTracker.docsUpperBound = -1; + hardTracker.timeUpperBound = 3000; + hardTracker.docsUpperBound = -1; + // updater.commitCallbacks.add(trigger); + + XmlUpdateRequestHandler handler = new XmlUpdateRequestHandler(); + handler.init( null ); + + MapSolrParams params = new MapSolrParams( new HashMap() ); + + // Add a single document + SolrQueryResponse rsp = new SolrQueryResponse(); + SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {}; + req.setContentStreams( toContentStreams( + adoc("id", "529", "field_t", "what's inside?", "subject", "info"), null ) ); + trigger.reset(); + handler.handleRequest( req, rsp ); + + // Check it it is in the index + assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" ); + + // Wait longer than the autocommit time + assertTrue(trigger.waitForNewSearcher(30000)); + trigger.reset(); + req.setContentStreams( toContentStreams( + adoc("id", "530", "field_t", "what's inside?", "subject", "info"), null ) ); + handler.handleRequest( req, rsp ); + + // Now make sure we can find it + assertQ("should find one", req("id:529") ,"//result[@numFound=1]" ); + // But not this one + assertQ("should find none", req("id:530") ,"//result[@numFound=0]" ); + + // Delete the document + assertU( delI("529") ); + assertQ("deleted, but should still be there", req("id:529") ,"//result[@numFound=1]" ); + // Wait longer than the autocommit time + assertTrue(trigger.waitForNewSearcher(15000)); + trigger.reset(); + req.setContentStreams( toContentStreams( + adoc("id", "550", "field_t", "what's inside?", "subject", "info"), null ) ); + handler.handleRequest( req, rsp ); + assertEquals( 2, softTracker.getCommitCount() ); + assertQ("deleted and time has passed", req("id:529") ,"//result[@numFound=0]" ); + + // now make the call 5 times really fast and make sure it + // only commits once + req.setContentStreams( toContentStreams( + adoc("id", "500" ), null ) ); + for( int i=0;i<5; i++ ) { + handler.handleRequest( req, rsp ); + } + assertQ("should not be there yet", req("id:500") ,"//result[@numFound=0]" ); + + // Wait longer than the autocommit time + assertTrue(trigger.waitForNewSearcher(15000)); + trigger.reset(); + + req.setContentStreams( toContentStreams( + adoc("id", "531", "field_t", "what's inside?", "subject", "info"), null ) ); + handler.handleRequest( req, rsp ); + assertEquals( 2, softTracker.getCommitCount() ); + assertEquals( 1, hardTracker.getCommitCount() ); + + assertQ("now it should", req("id:500") ,"//result[@numFound=1]" ); + assertQ("but not this", req("id:531") ,"//result[@numFound=0]" ); + } }