mirror of https://github.com/apache/lucene.git
SOLR-2565: reenable AutoCommitTest but move tests with soft commit to new test class that tests with a better method - likely some hardening tweaks still needed, but a the start of a large improvement.
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1196589 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller
parent
145d7503a3
commit
8125422e60
|
|
@ -31,10 +31,8 @@ import org.apache.solr.handler.XmlUpdateRequestHandler;
|
||||||
import org.apache.solr.request.SolrQueryRequestBase;
|
import org.apache.solr.request.SolrQueryRequestBase;
|
||||||
import org.apache.solr.response.SolrQueryResponse;
|
import org.apache.solr.response.SolrQueryResponse;
|
||||||
import org.apache.solr.search.SolrIndexSearcher;
|
import org.apache.solr.search.SolrIndexSearcher;
|
||||||
import org.apache.solr.update.NewSearcherListener.TriggerOn;
|
|
||||||
import org.apache.solr.util.AbstractSolrTestCase;
|
import org.apache.solr.util.AbstractSolrTestCase;
|
||||||
import org.apache.solr.util.RefCounted;
|
import org.apache.solr.util.RefCounted;
|
||||||
import org.junit.Ignore;
|
|
||||||
|
|
||||||
class NewSearcherListener implements SolrEventListener {
|
class NewSearcherListener implements SolrEventListener {
|
||||||
|
|
||||||
|
|
@ -106,7 +104,6 @@ class NewSearcherListener implements SolrEventListener {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Ignore("fix me")
|
|
||||||
public class AutoCommitTest extends AbstractSolrTestCase {
|
public class AutoCommitTest extends AbstractSolrTestCase {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
@ -262,269 +259,4 @@ public class AutoCommitTest extends AbstractSolrTestCase {
|
||||||
assertQ("now it should", req("id:500") ,"//result[@numFound=1]" );
|
assertQ("now it should", req("id:500") ,"//result[@numFound=1]" );
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testSoftCommitMaxDocs() throws Exception {
|
|
||||||
|
|
||||||
SolrCore core = h.getCore();
|
|
||||||
NewSearcherListener trigger = new NewSearcherListener(TriggerOn.Hard);
|
|
||||||
|
|
||||||
core.registerNewSearcherListener(trigger);
|
|
||||||
|
|
||||||
DirectUpdateHandler2 updateHandler = (DirectUpdateHandler2)core.getUpdateHandler();
|
|
||||||
updateHandler.registerCommitCallback(trigger);
|
|
||||||
|
|
||||||
CommitTracker tracker = updateHandler.commitTracker;
|
|
||||||
tracker.setTimeUpperBound(-1);
|
|
||||||
tracker.setDocsUpperBound(8);
|
|
||||||
|
|
||||||
|
|
||||||
NewSearcherListener softTrigger = new NewSearcherListener(TriggerOn.Soft);
|
|
||||||
updateHandler.registerSoftCommitCallback(softTrigger);
|
|
||||||
core.registerNewSearcherListener(softTrigger);
|
|
||||||
|
|
||||||
CommitTracker softTracker = updateHandler.softCommitTracker;
|
|
||||||
softTracker.setTimeUpperBound(-1);
|
|
||||||
softTracker.setDocsUpperBound(4);
|
|
||||||
|
|
||||||
|
|
||||||
XmlUpdateRequestHandler handler = new XmlUpdateRequestHandler();
|
|
||||||
handler.init( null );
|
|
||||||
|
|
||||||
MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
|
|
||||||
|
|
||||||
// Add documents
|
|
||||||
SolrQueryResponse rsp = new SolrQueryResponse();
|
|
||||||
SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
|
|
||||||
for( int i=0; i<4; i++ ) {
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", Integer.toString(i), "subject", "info" ), null ) );
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
}
|
|
||||||
// It should not be there right away
|
|
||||||
assertQ("shouldn't find any", req("id:1") ,"//result[@numFound=0]" );
|
|
||||||
assertEquals( 0, tracker.getCommitCount());
|
|
||||||
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "4", "subject", "info" ), null ) );
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
|
|
||||||
assertTrue(softTrigger.waitForNewSearcher(10000));
|
|
||||||
softTrigger.reset();
|
|
||||||
|
|
||||||
assertQ("should find 5", req("*:*") ,"//result[@numFound=5]" );
|
|
||||||
assertEquals( 1, softTracker.getCommitCount());
|
|
||||||
assertEquals( 0, tracker.getCommitCount());
|
|
||||||
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "5", "subject", "info" ), null ) );
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
|
|
||||||
// Now make sure we can find it
|
|
||||||
assertQ("should find one", req("id:4") ,"//result[@numFound=1]" );
|
|
||||||
assertEquals( 1, softTracker.getCommitCount());
|
|
||||||
// But not the one added afterward
|
|
||||||
assertQ("should not find one", req("id:5") ,"//result[@numFound=0]" );
|
|
||||||
assertEquals( 1, softTracker.getCommitCount());
|
|
||||||
|
|
||||||
for( int i=6; i<10; i++ ) {
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", Integer.toString(i), "subject", "info" ), null ) );
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
}
|
|
||||||
req.close();
|
|
||||||
|
|
||||||
assertTrue(softTrigger.waitForNewSearcher(30000));
|
|
||||||
softTrigger.reset();
|
|
||||||
|
|
||||||
assertTrue(trigger.waitForNewSearcher(10000));
|
|
||||||
assertQ("should find 10", req("*:*") ,"//result[@numFound=10]" );
|
|
||||||
assertEquals( 2, softTracker.getCommitCount());
|
|
||||||
assertEquals( 1, tracker.getCommitCount());
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testSoftCommitMaxTime() throws Exception {
|
|
||||||
SolrCore core = h.getCore();
|
|
||||||
NewSearcherListener trigger = new NewSearcherListener();
|
|
||||||
core.registerNewSearcherListener(trigger);
|
|
||||||
DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler();
|
|
||||||
CommitTracker tracker = updater.commitTracker;
|
|
||||||
CommitTracker softTracker = updater.softCommitTracker;
|
|
||||||
|
|
||||||
// too low of a number can cause a slow host to commit before the test code checks that it
|
|
||||||
// isn't there... causing a failure at "shouldn't find any"
|
|
||||||
softTracker.setTimeUpperBound(2000);
|
|
||||||
softTracker.setDocsUpperBound(-1);
|
|
||||||
// updater.commitCallbacks.add(trigger);
|
|
||||||
|
|
||||||
XmlUpdateRequestHandler handler = new XmlUpdateRequestHandler();
|
|
||||||
handler.init( null );
|
|
||||||
|
|
||||||
MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
|
|
||||||
|
|
||||||
// Add a single document
|
|
||||||
SolrQueryResponse rsp = new SolrQueryResponse();
|
|
||||||
SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "529", "field_t", "what's inside?", "subject", "info"), null ) );
|
|
||||||
trigger.reset();
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
|
|
||||||
// Check it it is in the index
|
|
||||||
assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" );
|
|
||||||
assertEquals(0, softTracker.getCommitCount());
|
|
||||||
|
|
||||||
// Wait longer than the autocommit time
|
|
||||||
assertTrue(trigger.waitForNewSearcher(30000));
|
|
||||||
trigger.reset();
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "530", "field_t", "what's inside?", "subject", "info"), null ) );
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
|
|
||||||
// Now make sure we can find it
|
|
||||||
assertQ("should find one", req("id:529") ,"//result[@numFound=1]" );
|
|
||||||
// But not this one
|
|
||||||
assertQ("should find none", req("id:530") ,"//result[@numFound=0]" );
|
|
||||||
verbose("###about to delete 529");
|
|
||||||
// Delete the document
|
|
||||||
assertU(delI("529"));
|
|
||||||
assertQ("deleted, but should still be there", req("id:529") ,"//result[@numFound=1]" );
|
|
||||||
// Wait longer than the autocommit time
|
|
||||||
verbose("###starting to wait for new searcher. softTracker.getCommitCount()==",softTracker.getCommitCount());
|
|
||||||
assertTrue(trigger.waitForNewSearcher(15000));
|
|
||||||
trigger.reset();
|
|
||||||
verbose("###done waiting for new searcher. softTracker.getCommitCount()==",softTracker.getCommitCount());
|
|
||||||
|
|
||||||
// what's the point of this update?
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "550", "field_t", "what's inside?", "subject", "info"), null ) );
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
|
|
||||||
|
|
||||||
assertEquals( 2, softTracker.getCommitCount() );
|
|
||||||
assertQ("deleted and time has passed", req("id:529") ,"//result[@numFound=0]" );
|
|
||||||
|
|
||||||
// now make the call 5 times really fast and make sure it
|
|
||||||
// only commits once
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "500" ), null ) );
|
|
||||||
for( int i=0;i<5; i++ ) {
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
}
|
|
||||||
assertQ("should not be there yet", req("id:500") ,"//result[@numFound=0]" );
|
|
||||||
|
|
||||||
// Wait longer than the autocommit time
|
|
||||||
assertTrue(trigger.waitForNewSearcher(15000));
|
|
||||||
trigger.reset();
|
|
||||||
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "531", "field_t", "what's inside?", "subject", "info"), null ) );
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
assertEquals( 3, softTracker.getCommitCount() );
|
|
||||||
assertEquals( 0, tracker.getCommitCount() );
|
|
||||||
|
|
||||||
assertQ("now it should", req("id:500") ,"//result[@numFound=1]" );
|
|
||||||
assertQ("but not this", req("id:531") ,"//result[@numFound=0]" );
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testSoftAndHardCommitMaxTime() throws Exception {
|
|
||||||
SolrCore core = h.getCore();
|
|
||||||
NewSearcherListener softTrigger = new NewSearcherListener(TriggerOn.Soft);
|
|
||||||
NewSearcherListener hardTrigger = new NewSearcherListener(TriggerOn.Hard);
|
|
||||||
core.registerNewSearcherListener(softTrigger);
|
|
||||||
core.registerNewSearcherListener(hardTrigger);
|
|
||||||
DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler();
|
|
||||||
|
|
||||||
updater.registerSoftCommitCallback(softTrigger);
|
|
||||||
updater.registerCommitCallback(hardTrigger);
|
|
||||||
|
|
||||||
CommitTracker hardTracker = updater.commitTracker;
|
|
||||||
CommitTracker softTracker = updater.softCommitTracker;
|
|
||||||
|
|
||||||
// too low of a number can cause a slow host to commit before the test code checks that it
|
|
||||||
// isn't there... causing a failure at "shouldn't find any"
|
|
||||||
softTracker.setTimeUpperBound(500);
|
|
||||||
softTracker.setDocsUpperBound(-1);
|
|
||||||
hardTracker.setTimeUpperBound(1200);
|
|
||||||
hardTracker.setDocsUpperBound(-1);
|
|
||||||
// updater.commitCallbacks.add(trigger);
|
|
||||||
|
|
||||||
XmlUpdateRequestHandler handler = new XmlUpdateRequestHandler();
|
|
||||||
handler.init( null );
|
|
||||||
|
|
||||||
MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
|
|
||||||
|
|
||||||
// Add a single document
|
|
||||||
SolrQueryResponse rsp = new SolrQueryResponse();
|
|
||||||
SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "529", "field_t", "what's inside?", "subject", "info"), null ) );
|
|
||||||
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
|
|
||||||
// Check if it is in the index
|
|
||||||
assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" );
|
|
||||||
|
|
||||||
// Wait longer than the autocommit time
|
|
||||||
assertTrue(softTrigger.waitForNewSearcher(30000));
|
|
||||||
softTrigger.reset();
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "530", "field_t", "what's inside?", "subject", "info"), null ) );
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
|
|
||||||
// Now make sure we can find it
|
|
||||||
assertQ("should find one", req("id:529") ,"//result[@numFound=1]" );
|
|
||||||
// But not this one
|
|
||||||
assertQ("should find none", req("id:530") ,"//result[@numFound=0]" );
|
|
||||||
|
|
||||||
// Delete the document
|
|
||||||
assertU( delI("529") );
|
|
||||||
assertQ("deleted, but should still be there", req("id:529") ,"//result[@numFound=1]" );
|
|
||||||
|
|
||||||
// Wait longer than the autocommit time
|
|
||||||
assertTrue(softTrigger.waitForNewSearcher(30000));
|
|
||||||
softTrigger.reset();
|
|
||||||
|
|
||||||
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "550", "field_t", "what's inside?", "subject", "info"), null ) );
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
int totalCommits = softTracker.getCommitCount() + hardTracker.getCommitCount();
|
|
||||||
assertTrue("expected:>=2 but got " + totalCommits, totalCommits >= 2);
|
|
||||||
assertQ("deleted and time has passed", req("id:529") ,"//result[@numFound=0]" );
|
|
||||||
|
|
||||||
// now make the call 2 times really fast and make sure id:500
|
|
||||||
// is not visible right away
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "500" ), null ) );
|
|
||||||
for( int i=0;i<2; i++ ) {
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
}
|
|
||||||
assertQ("should not be there yet", req("id:500") ,"//result[@numFound=0]" );
|
|
||||||
|
|
||||||
// Wait longer than the autocommit time
|
|
||||||
assertTrue(softTrigger.waitForNewSearcher(30000));
|
|
||||||
softTrigger.reset();
|
|
||||||
|
|
||||||
req.setContentStreams( toContentStreams(
|
|
||||||
adoc("id", "531", "field_t", "what's inside?", "subject", "info"), null ) );
|
|
||||||
handler.handleRequest( req, rsp );
|
|
||||||
|
|
||||||
// depending on timing, you might see 2 or 3 soft commits
|
|
||||||
int softCommitCnt = softTracker.getCommitCount();
|
|
||||||
assertTrue("commit cnt:" + softCommitCnt, softCommitCnt == 2
|
|
||||||
|| softCommitCnt == 3);
|
|
||||||
|
|
||||||
// depending on timing, you might see 1 or 2 hard commits
|
|
||||||
assertTrue(hardTrigger.waitForNewSearcher(30000));
|
|
||||||
hardTrigger.reset();
|
|
||||||
|
|
||||||
int hardCommitCnt = hardTracker.getCommitCount();
|
|
||||||
assertTrue("commit cnt:" + hardCommitCnt, hardCommitCnt == 1
|
|
||||||
|| hardCommitCnt == 2);
|
|
||||||
|
|
||||||
assertTrue(softTrigger.waitForNewSearcher(30000));
|
|
||||||
softTrigger.reset();
|
|
||||||
|
|
||||||
assertQ("now it should", req("id:500") ,"//result[@numFound=1]" );
|
|
||||||
assertQ("but not this", req("id:531") ,"//result[@numFound=1]" );
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,372 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.solr.update;
|
||||||
|
|
||||||
|
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||||
|
import static java.util.concurrent.TimeUnit.SECONDS;
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
|
||||||
|
import java.util.concurrent.BlockingQueue;
|
||||||
|
import java.util.concurrent.LinkedBlockingQueue;
|
||||||
|
|
||||||
|
import org.apache.solr.common.util.NamedList;
|
||||||
|
import org.apache.solr.core.SolrCore;
|
||||||
|
import org.apache.solr.core.SolrEventListener;
|
||||||
|
import org.apache.solr.search.SolrIndexSearcher;
|
||||||
|
import org.apache.solr.util.AbstractSolrTestCase;
|
||||||
|
import org.junit.Before;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test auto commit functionality in a way that doesn't suck.
|
||||||
|
* <p>
|
||||||
|
* AutoCommitTest is an abomination that is way to brittle in how it
|
||||||
|
* tries to check that commits happened, and when they happened.
|
||||||
|
* The goal of this test class is to (ultimately) completely replace all
|
||||||
|
* of the functionality of that test class using:
|
||||||
|
* </p>
|
||||||
|
* <ul>
|
||||||
|
* <li>A more robust monitor of commit/newSearcher events that records
|
||||||
|
* the times of those events in a queue that can be polled.
|
||||||
|
* Multiple events in rapid succession are not lost.
|
||||||
|
* </li>
|
||||||
|
* <li>Timing checks that are forgiving of slow machines and use
|
||||||
|
* knowledge of how slow A->B was to affect the expectation of
|
||||||
|
* how slow B->C will be
|
||||||
|
* </li>
|
||||||
|
* </ul>
|
||||||
|
*/
|
||||||
|
public class SoftAutoCommitTest extends AbstractSolrTestCase {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getSchemaFile() { return "schema.xml"; }
|
||||||
|
@Override
|
||||||
|
public String getSolrConfigFile() { return "solrconfig.xml"; }
|
||||||
|
|
||||||
|
private MockEventListener monitor;
|
||||||
|
private DirectUpdateHandler2 updater;
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void createMonitor() throws Exception {
|
||||||
|
SolrCore core = h.getCore();
|
||||||
|
|
||||||
|
updater = (DirectUpdateHandler2) core.getUpdateHandler();
|
||||||
|
monitor = new MockEventListener();
|
||||||
|
|
||||||
|
core.registerNewSearcherListener(monitor);
|
||||||
|
updater.registerSoftCommitCallback(monitor);
|
||||||
|
updater.registerCommitCallback(monitor);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testSoftAndHardCommitMaxTimeMixedAdds() throws Exception {
|
||||||
|
|
||||||
|
final int softCommitWaitMillis = 500;
|
||||||
|
final int hardCommitWaitMillis = 1200;
|
||||||
|
|
||||||
|
CommitTracker hardTracker = updater.commitTracker;
|
||||||
|
CommitTracker softTracker = updater.softCommitTracker;
|
||||||
|
|
||||||
|
softTracker.setTimeUpperBound(softCommitWaitMillis);
|
||||||
|
softTracker.setDocsUpperBound(-1);
|
||||||
|
hardTracker.setTimeUpperBound(hardCommitWaitMillis);
|
||||||
|
hardTracker.setDocsUpperBound(-1);
|
||||||
|
|
||||||
|
// Add a single document
|
||||||
|
long add529 = System.currentTimeMillis();
|
||||||
|
assertU(adoc("id", "529", "subject", "the doc we care about in this test"));
|
||||||
|
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
// Wait for the soft commit with some fudge
|
||||||
|
Long soft529 = monitor.soft.poll(softCommitWaitMillis * 2, MILLISECONDS);
|
||||||
|
assertNotNull("soft529 wasn't fast enough", soft529);
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
// check for the searcher, should have happend right after soft commit
|
||||||
|
Long searcher529 = monitor.searcher.poll(softCommitWaitMillis, MILLISECONDS);
|
||||||
|
assertNotNull("searcher529 wasn't fast enough", searcher529);
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
// toss in another doc, shouldn't affect first hard commit time we poll
|
||||||
|
assertU(adoc("id", "530", "subject", "just for noise/activity"));
|
||||||
|
|
||||||
|
// wait for the hard commit, shouldn't need any fudge given
|
||||||
|
// other actions already taken
|
||||||
|
Long hard529 = monitor.hard.poll(hardCommitWaitMillis * 2, MILLISECONDS);
|
||||||
|
assertNotNull("hard529 wasn't fast enough", hard529);
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
assertTrue("soft529 occured too fast: " +
|
||||||
|
add529 + " + " + softCommitWaitMillis + " !<= " + soft529,
|
||||||
|
add529 + softCommitWaitMillis <= soft529);
|
||||||
|
assertTrue("hard529 occured too fast: " +
|
||||||
|
add529 + " + " + hardCommitWaitMillis + " !<= " + hard529,
|
||||||
|
add529 + hardCommitWaitMillis <= hard529);
|
||||||
|
|
||||||
|
// however slow the machine was to do the soft commit compared to expected,
|
||||||
|
// assume newSearcher had some magnitude of that much overhead as well
|
||||||
|
long slowTestFudge = Math.max(100, 6 * (soft529 - add529 - softCommitWaitMillis));
|
||||||
|
assertTrue("searcher529 wasn't soon enough after soft529: " +
|
||||||
|
searcher529 + " !< " + soft529 + " + " + slowTestFudge + " (fudge)",
|
||||||
|
searcher529 < soft529 + slowTestFudge );
|
||||||
|
|
||||||
|
assertTrue("hard529 was before searcher529: " +
|
||||||
|
searcher529 + " !<= " + hard529,
|
||||||
|
searcher529 <= hard529);
|
||||||
|
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
// there may have been (or will be) a second hard commit for 530
|
||||||
|
Long hard530 = monitor.hard.poll(hardCommitWaitMillis, MILLISECONDS);
|
||||||
|
assertEquals("Tracker reports too many hard commits",
|
||||||
|
(null == hard530 ? 1 : 2),
|
||||||
|
hardTracker.getCommitCount());
|
||||||
|
|
||||||
|
// there may have been a second soft commit for 530,
|
||||||
|
// but if so it must have already happend
|
||||||
|
Long soft530 = monitor.soft.poll(0, MILLISECONDS);
|
||||||
|
if (null != soft530) {
|
||||||
|
assertEquals("Tracker reports too many soft commits",
|
||||||
|
2, softTracker.getCommitCount());
|
||||||
|
if (null != hard530) {
|
||||||
|
assertTrue("soft530 after hard530: " +
|
||||||
|
soft530 + " !<= " + hard530,
|
||||||
|
soft530 <= hard530);
|
||||||
|
} else {
|
||||||
|
assertTrue("soft530 after hard529 but no hard530: " +
|
||||||
|
soft530 + " !<= " + hard529,
|
||||||
|
soft530 <= hard529);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
assertEquals("Tracker reports too many soft commits",
|
||||||
|
1, softTracker.getCommitCount());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (null != soft530 || null != hard530) {
|
||||||
|
assertNotNull("at least one extra commit for 530, but no searcher",
|
||||||
|
monitor.searcher.poll(0, MILLISECONDS));
|
||||||
|
}
|
||||||
|
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
// wait a bit, w/o other action we definitley shouldn't see any
|
||||||
|
// new hard/soft commits
|
||||||
|
assertNull("Got a hard commit we weren't expecting",
|
||||||
|
monitor.hard.poll(2, SECONDS));
|
||||||
|
assertNull("Got a soft commit we weren't expecting",
|
||||||
|
monitor.soft.poll(0, MILLISECONDS));
|
||||||
|
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testSoftAndHardCommitMaxTimeDelete() throws Exception {
|
||||||
|
|
||||||
|
final int softCommitWaitMillis = 500;
|
||||||
|
final int hardCommitWaitMillis = 1200;
|
||||||
|
|
||||||
|
CommitTracker hardTracker = updater.commitTracker;
|
||||||
|
CommitTracker softTracker = updater.softCommitTracker;
|
||||||
|
|
||||||
|
softTracker.setTimeUpperBound(softCommitWaitMillis);
|
||||||
|
softTracker.setDocsUpperBound(-1);
|
||||||
|
hardTracker.setTimeUpperBound(hardCommitWaitMillis);
|
||||||
|
hardTracker.setDocsUpperBound(-1);
|
||||||
|
|
||||||
|
// add a doc and force a commit
|
||||||
|
assertU(adoc("id", "529", "subject", "the doc we care about in this test"));
|
||||||
|
assertU(commit());
|
||||||
|
long postAdd529 = System.currentTimeMillis();
|
||||||
|
|
||||||
|
// wait for first hard/soft commit
|
||||||
|
Long soft529 = monitor.soft.poll(softCommitWaitMillis * 2, MILLISECONDS);
|
||||||
|
assertNotNull("soft529 wasn't fast enough", soft529);
|
||||||
|
Long manCommit = monitor.hard.poll(0, MILLISECONDS);
|
||||||
|
|
||||||
|
assertNotNull("manCommit wasn't fast enough", manCommit);
|
||||||
|
assertTrue("forced manCommit didn't happen when it should have: " +
|
||||||
|
manCommit + " !< " + postAdd529,
|
||||||
|
manCommit < postAdd529);
|
||||||
|
|
||||||
|
Long hard529 = monitor.hard.poll(hardCommitWaitMillis, MILLISECONDS);
|
||||||
|
assertNotNull("hard529 wasn't fast enough", hard529);
|
||||||
|
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
monitor.clear();
|
||||||
|
|
||||||
|
// Delete the document
|
||||||
|
long del529 = System.currentTimeMillis();
|
||||||
|
assertU( delI("529") );
|
||||||
|
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
// Wait for the soft commit with some fudge
|
||||||
|
soft529 = monitor.soft.poll(softCommitWaitMillis * 2, MILLISECONDS);
|
||||||
|
assertNotNull("soft529 wasn't fast enough", soft529);
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
// check for the searcher, should have happened right after soft commit
|
||||||
|
Long searcher529 = monitor.searcher.poll(softCommitWaitMillis, MILLISECONDS);
|
||||||
|
assertNotNull("searcher529 wasn't fast enough", searcher529);
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
// toss in another doc, shouldn't affect first hard commit time we poll
|
||||||
|
assertU(adoc("id", "550", "subject", "just for noise/activity"));
|
||||||
|
|
||||||
|
// wait for the hard commit, shouldn't need any fudge given
|
||||||
|
// other actions already taken
|
||||||
|
hard529 = monitor.hard.poll(hardCommitWaitMillis * 2, MILLISECONDS);
|
||||||
|
assertNotNull("hard529 wasn't fast enough", hard529);
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
assertTrue("soft529 occured too fast: " +
|
||||||
|
del529 + " + " + softCommitWaitMillis + " !<= " + soft529,
|
||||||
|
del529 + softCommitWaitMillis <= soft529);
|
||||||
|
assertTrue("hard529 occured too fast: " +
|
||||||
|
del529 + " + " + hardCommitWaitMillis + " !<= " + hard529,
|
||||||
|
del529 + hardCommitWaitMillis <= hard529);
|
||||||
|
|
||||||
|
// however slow the machine was to do the soft commit compared to expected,
|
||||||
|
// assume newSearcher had some magnitude of that much overhead as well
|
||||||
|
long slowTestFudge = Math.max(100, 3 * (soft529 - del529 - softCommitWaitMillis));
|
||||||
|
assertTrue("searcher529 wasn't soon enough after soft529: " +
|
||||||
|
searcher529 + " !< " + soft529 + " + " + slowTestFudge + " (fudge)",
|
||||||
|
searcher529 < soft529 + slowTestFudge );
|
||||||
|
|
||||||
|
assertTrue("hard529 was before searcher529: " +
|
||||||
|
searcher529 + " !<= " + hard529,
|
||||||
|
searcher529 <= hard529);
|
||||||
|
|
||||||
|
// clear commmits
|
||||||
|
monitor.hard.clear();
|
||||||
|
monitor.soft.clear();
|
||||||
|
|
||||||
|
// wait a bit, w/o other action we definitely shouldn't see any
|
||||||
|
// new hard/soft commits
|
||||||
|
assertNull("Got a hard commit we weren't expecting",
|
||||||
|
monitor.hard.poll(2, SECONDS));
|
||||||
|
assertNull("Got a soft commit we weren't expecting",
|
||||||
|
monitor.soft.poll(0, MILLISECONDS));
|
||||||
|
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testSoftAndHardCommitMaxTimeRapidAdds() throws Exception {
|
||||||
|
|
||||||
|
final int softCommitWaitMillis = 500;
|
||||||
|
final int hardCommitWaitMillis = 1200;
|
||||||
|
|
||||||
|
CommitTracker hardTracker = updater.commitTracker;
|
||||||
|
CommitTracker softTracker = updater.softCommitTracker;
|
||||||
|
|
||||||
|
softTracker.setTimeUpperBound(softCommitWaitMillis);
|
||||||
|
softTracker.setDocsUpperBound(-1);
|
||||||
|
hardTracker.setTimeUpperBound(hardCommitWaitMillis);
|
||||||
|
hardTracker.setDocsUpperBound(-1);
|
||||||
|
|
||||||
|
// try to add 5 docs really fast
|
||||||
|
long fast5start = System.currentTimeMillis();
|
||||||
|
for( int i=0;i<5; i++ ) {
|
||||||
|
assertU(adoc("id", ""+500 + i, "subject", "five fast docs"));
|
||||||
|
}
|
||||||
|
long fast5end = System.currentTimeMillis() - 100; // minus a tad of slop
|
||||||
|
long fast5time = 1 + fast5end - fast5start;
|
||||||
|
|
||||||
|
// total time for all 5 adds determines the number of soft to expect
|
||||||
|
long expectedSoft = (long)Math.ceil(fast5time / softCommitWaitMillis);
|
||||||
|
long expectedHard = (long)Math.ceil(fast5time / hardCommitWaitMillis);
|
||||||
|
|
||||||
|
// note: counting from 1 for multiplication
|
||||||
|
for (int i = 1; i <= expectedSoft; i++) {
|
||||||
|
// Wait for the soft commit with some fudge
|
||||||
|
Long soft = monitor.soft.poll(softCommitWaitMillis * 2, MILLISECONDS);
|
||||||
|
assertNotNull(i + ": soft wasn't fast enough", soft);
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
// have to assume none of the docs were added until
|
||||||
|
// very end of the add window
|
||||||
|
assertTrue(i + ": soft occured too fast: " +
|
||||||
|
fast5end + " + (" + softCommitWaitMillis + " * " + i +
|
||||||
|
") !<= " + soft,
|
||||||
|
fast5end + (softCommitWaitMillis * i) <= soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
// note: counting from 1 for multiplication
|
||||||
|
for (int i = 1; i <= expectedHard; i++) {
|
||||||
|
// wait for the hard commit, shouldn't need any fudge given
|
||||||
|
// other actions already taken
|
||||||
|
Long hard = monitor.hard.poll(hardCommitWaitMillis, MILLISECONDS);
|
||||||
|
assertNotNull(i + ": hard wasn't fast enough", hard);
|
||||||
|
monitor.assertSaneOffers();
|
||||||
|
|
||||||
|
// have to assume none of the docs were added until
|
||||||
|
// very end of the add window
|
||||||
|
assertTrue(i + ": soft occured too fast: " +
|
||||||
|
fast5end + " + (" + hardCommitWaitMillis + " * " + i +
|
||||||
|
") !<= " + hard,
|
||||||
|
fast5end + (hardCommitWaitMillis * i) <= hard);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class MockEventListener implements SolrEventListener {
|
||||||
|
|
||||||
|
// use capacity bound Queues just so we're sure we don't OOM
|
||||||
|
public final BlockingQueue<Long> soft = new LinkedBlockingQueue<Long>(1000);
|
||||||
|
public final BlockingQueue<Long> hard = new LinkedBlockingQueue<Long>(1000);
|
||||||
|
public final BlockingQueue<Long> searcher = new LinkedBlockingQueue<Long>(1000);
|
||||||
|
|
||||||
|
// if non enpty, then at least one offer failed (queues full)
|
||||||
|
private StringBuffer fail = new StringBuffer();
|
||||||
|
|
||||||
|
public MockEventListener() { /* NOOP */ }
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(NamedList args) {}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void newSearcher(SolrIndexSearcher newSearcher,
|
||||||
|
SolrIndexSearcher currentSearcher) {
|
||||||
|
Long now = System.currentTimeMillis();
|
||||||
|
if (!searcher.offer(now)) fail.append(", newSearcher @ " + now);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void postCommit() {
|
||||||
|
Long now = System.currentTimeMillis();
|
||||||
|
if (!hard.offer(now)) fail.append(", hardCommit @ " + now);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void postSoftCommit() {
|
||||||
|
Long now = System.currentTimeMillis();
|
||||||
|
if (!soft.offer(now)) fail.append(", softCommit @ " + now);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void clear() {
|
||||||
|
soft.clear();
|
||||||
|
hard.clear();
|
||||||
|
searcher.clear();
|
||||||
|
fail.setLength(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void assertSaneOffers() {
|
||||||
|
assertEquals("Failure of MockEventListener" + fail.toString(),
|
||||||
|
0, fail.length());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Loading…
Reference in New Issue