Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-11055: Add 'commitWithin' testing (of both soft/hard commits) to SoftAutoCommitTest

This commit is contained in:
Chris Hostetter 2017-10-10 09:08:25 -07:00
parent 371f731520
commit b21721f152
2 changed files with 78 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
View File

@ -71,8 +71,6 @@ import org.slf4j.LoggerFactory;
/**
* <code>DirectUpdateHandler2</code> implements an UpdateHandler where documents are added
* directly to the main Lucene index as opposed to adding to a separate smaller index.
* <p>
* TODO: add soft commitWithin support
*/
public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState.IndexWriterCloser, SolrMetricProducer {
protected final SolrCoreState solrCoreState;
@ -99,6 +97,13 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
protected final CommitTracker softCommitTracker;
protected boolean commitWithinSoftCommit;
/**
* package access for testing
* @lucene.internal
*/
void setCommitWithinSoftCommit(boolean value) {
this.commitWithinSoftCommit = value;
}
protected boolean indexWriterCloseWaitsForMerges;

103
solr/core/src/test/org/apache/solr/update/SoftAutoCommitTest.java
View File

@ -76,6 +76,7 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
SolrCore core = h.getCore();
updater = (DirectUpdateHandler2) core.getUpdateHandler();
updater.setCommitWithinSoftCommit(true); // foce to default, let tests change as needed
monitor = new MockEventListener();
core.registerNewSearcherListener(monitor);
@ -163,11 +164,24 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
}
public void testSoftAndHardCommitMaxTimeMixedAdds() throws Exception {
doTestSoftAndHardCommitMaxTimeMixedAdds(CommitWithinType.NONE);
}
public void testSoftCommitWithinAndHardCommitMaxTimeMixedAdds() throws Exception {
doTestSoftAndHardCommitMaxTimeMixedAdds(CommitWithinType.SOFT);
}
public void testHardCommitWithinAndSoftCommitMaxTimeMixedAdds() throws Exception {
doTestSoftAndHardCommitMaxTimeMixedAdds(CommitWithinType.HARD);
}
private void doTestSoftAndHardCommitMaxTimeMixedAdds(final CommitWithinType commitWithinType)
throws Exception {
final int softCommitWaitMillis = 500;
final int hardCommitWaitMillis = 1200;
final int commitWithin = commitWithinType.useValue(softCommitWaitMillis, hardCommitWaitMillis);
CommitTracker hardTracker = updater.commitTracker;
CommitTracker softTracker = updater.softCommitTracker;
updater.setCommitWithinSoftCommit(commitWithinType.equals(CommitWithinType.SOFT));
// wait out any leaked commits
monitor.soft.poll(softCommitWaitMillis * 2, MILLISECONDS);
@ -176,16 +190,16 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
int startingHardCommits = hardTracker.getCommitCount();
int startingSoftCommits = softTracker.getCommitCount();
softTracker.setTimeUpperBound(softCommitWaitMillis);
softTracker.setTimeUpperBound(commitWithinType.equals(CommitWithinType.SOFT) ? -1 : softCommitWaitMillis);
softTracker.setDocsUpperBound(-1);
hardTracker.setTimeUpperBound(hardCommitWaitMillis);
hardTracker.setTimeUpperBound(commitWithinType.equals(CommitWithinType.HARD) ? -1 : hardCommitWaitMillis);
hardTracker.setDocsUpperBound(-1);
// simplify whats going on by only having soft auto commits trigger new searchers
hardTracker.setOpenSearcher(false);
// Add a single document
long add529 = System.nanoTime();
assertU(adoc("id", "529", "subject", "the doc we care about in this test"));
assertU(adoc(commitWithin, "id", "529", "subject", "the doc we care about in this test"));
monitor.assertSaneOffers();
@ -205,7 +219,7 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
monitor.assertSaneOffers();
// toss in another doc, shouldn't affect first hard commit time we poll
assertU(adoc("id", "530", "subject", "just for noise/activity"));
assertU(adoc(commitWithin, "id", "530", "subject", "just for noise/activity"));
monitor.assertSaneOffers();
@ -280,55 +294,48 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
}
public void testSoftAndHardCommitMaxTimeDelete() throws Exception {
doTestSoftAndHardCommitMaxTimeDelete(CommitWithinType.NONE);
}
public void testSoftCommitWithinAndHardCommitMaxTimeDelete() throws Exception {
doTestSoftAndHardCommitMaxTimeDelete(CommitWithinType.SOFT);
}
public void testHardCommitWithinAndSoftCommitMaxTimeDelete() throws Exception {
doTestSoftAndHardCommitMaxTimeDelete(CommitWithinType.HARD);
}
private void doTestSoftAndHardCommitMaxTimeDelete(final CommitWithinType commitWithinType)
throws Exception {
final int softCommitWaitMillis = 500;
final int hardCommitWaitMillis = 1200;
final int commitWithin = commitWithinType.useValue(softCommitWaitMillis, hardCommitWaitMillis);
CommitTracker hardTracker = updater.commitTracker;
CommitTracker softTracker = updater.softCommitTracker;
updater.setCommitWithinSoftCommit(commitWithinType.equals(CommitWithinType.SOFT));
int startingHardCommits = hardTracker.getCommitCount();
int startingSoftCommits = softTracker.getCommitCount();
softTracker.setTimeUpperBound(softCommitWaitMillis);
softTracker.setTimeUpperBound(commitWithinType.equals(CommitWithinType.SOFT) ? -1 : softCommitWaitMillis);
softTracker.setDocsUpperBound(-1);
hardTracker.setTimeUpperBound(hardCommitWaitMillis);
hardTracker.setTimeUpperBound(commitWithinType.equals(CommitWithinType.HARD) ? -1 : hardCommitWaitMillis);
hardTracker.setDocsUpperBound(-1);
// we don't want to overlap soft and hard opening searchers - this now blocks commits and we
// are looking for prompt timings
hardTracker.setOpenSearcher(false);
// add a doc and force a commit
assertU(adoc("id", "529", "subject", "the doc we care about in this test"));
assertU(adoc(commitWithin, "id", "529", "subject", "the doc we care about in this test"));
assertU(commit());
Long soft529;
Long hard529;
/*** an explicit commit can (and should) clear pending auto-commits
long postAdd529 = System.currentTimeMillis();
// wait for first hard/soft commit
Long soft529 = monitor.soft.poll(softCommitWaitMillis * 3, MILLISECONDS);
assertNotNull("soft529 wasn't fast enough", soft529);
Long manCommit = monitor.hard.poll(0, MILLISECONDS);
assertNotNull("manCommit wasn't fast enough", manCommit);
assertTrue("forced manCommit didn't happen when it should have: " +
manCommit + " !<= " + postAdd529,
manCommit <= postAdd529);
Long hard529 = monitor.hard.poll(hardCommitWaitMillis * 2, MILLISECONDS);
assertNotNull("hard529 wasn't fast enough", hard529);
monitor.assertSaneOffers();
***/
monitor.clear();
// Delete the document
long del529 = System.nanoTime();
assertU( delI("529") );
assertU( delI("529", "commitWithin", ""+commitWithin));
monitor.assertSaneOffers();
@ -343,7 +350,7 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
monitor.assertSaneOffers();
// toss in another doc, shouldn't affect first hard commit time we poll
assertU(adoc("id", "550", "subject", "just for noise/activity"));
assertU(adoc(commitWithin, "id", "550", "subject", "just for noise/activity"));
// wait for the hard commit
hard529 = monitor.hard.poll(hardCommitWaitMillis * 5, MILLISECONDS);
@ -395,16 +402,28 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
}
public void testSoftAndHardCommitMaxTimeRapidAdds() throws Exception {
doTestSoftAndHardCommitMaxTimeRapidAdds(CommitWithinType.NONE);
}
public void testSoftCommitWithinAndHardCommitMaxTimeRapidAdds() throws Exception {
doTestSoftAndHardCommitMaxTimeRapidAdds(CommitWithinType.SOFT);
}
public void testHardCommitWithinAndSoftCommitMaxTimeRapidAdds() throws Exception {
doTestSoftAndHardCommitMaxTimeRapidAdds(CommitWithinType.HARD);
}
public void doTestSoftAndHardCommitMaxTimeRapidAdds(final CommitWithinType commitWithinType)
throws Exception {
final int softCommitWaitMillis = 500;
final int hardCommitWaitMillis = 1200;
final int commitWithin = commitWithinType.useValue(softCommitWaitMillis, hardCommitWaitMillis);
CommitTracker hardTracker = updater.commitTracker;
CommitTracker softTracker = updater.softCommitTracker;
updater.setCommitWithinSoftCommit(commitWithinType.equals(CommitWithinType.SOFT));
softTracker.setTimeUpperBound(softCommitWaitMillis);
softTracker.setTimeUpperBound(commitWithinType.equals(CommitWithinType.SOFT) ? -1 : softCommitWaitMillis);
softTracker.setDocsUpperBound(-1);
hardTracker.setTimeUpperBound(hardCommitWaitMillis);
hardTracker.setTimeUpperBound(commitWithinType.equals(CommitWithinType.HARD) ? -1 : hardCommitWaitMillis);
hardTracker.setDocsUpperBound(-1);
// we don't want to overlap soft and hard opening searchers - this now blocks commits and we
// are looking for prompt timings
@ -414,7 +433,7 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
final long preFirstNanos = System.nanoTime();
for( int i=0;i<5; i++ ) {
assertU(adoc("id", ""+500 + i, "subject", "five fast docs"));
assertU(adoc(commitWithin, "id", ""+500 + i, "subject", "five fast docs"));
}
final long postLastNanos = System.nanoTime();
@ -490,6 +509,26 @@ public class SoftAutoCommitTest extends AbstractSolrTestCase {
commitNanos = queue.poll(commitWaitMillis * 3, MILLISECONDS);
}
}
/** enum for indicating if a test should use commitWithin, and if so what type: hard or soft */
private static enum CommitWithinType {
NONE {
@Override public int useValue(final int softCommitWaitMillis, final int hardCommitWaitMillis) {
return -1;
}
},
SOFT {
@Override public int useValue(final int softCommitWaitMillis, final int hardCommitWaitMillis) {
return softCommitWaitMillis;
}
},
HARD {
@Override public int useValue(final int softCommitWaitMillis, final int hardCommitWaitMillis) {
return hardCommitWaitMillis;
}
};
public abstract int useValue(final int softCommitWaitMillis, final int hardCommitWaitMillis);
}
}
class MockEventListener implements SolrEventListener {