Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merged /lucene/dev/trunk:r1441423-1441567 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1441570 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2013-02-01 18:25:23 +00:00
parent 00f66bf975 ef4fd560c5
commit 9e4e017601
6 changed files with 168 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -68,6 +68,9 @@ New Features
* SOLR-2827: RegexpBoost Update Processor (janhoy)
* SOLR-4370: Allow configuring commitWithin to do hard commits.
(Mark Miller, Senthuran Sivananthan)
Bug Fixes
----------------------

8
solr/core/src/java/org/apache/solr/core/SolrConfig.java
View File

@ -238,7 +238,8 @@ public class SolrConfig extends Config {
getBool("updateHandler/autoCommit/openSearcher",true),
getInt("updateHandler/commitIntervalLowerBound",-1),
getInt("updateHandler/autoSoftCommit/maxDocs",-1),
getInt("updateHandler/autoSoftCommit/maxTime",-1));
getInt("updateHandler/autoSoftCommit/maxTime",-1),
getBool("updateHandler/commitWithin/softCommit",true));
}
private void loadPluginInfo(Class clazz, String tag, boolean requireName, boolean requireClass) {
@ -402,6 +403,7 @@ public class SolrConfig extends Config {
public final int autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound,
autoSoftCommmitMaxDocs,autoSoftCommmitMaxTime;
public final boolean openSearcher; // is opening a new searcher part of hard autocommit?
public final boolean commitWithinSoftCommit;
/**
* @param autoCommmitMaxDocs set -1 as default
@ -409,7 +411,7 @@ public class SolrConfig extends Config {
* @param commitIntervalLowerBound set -1 as default
*/
public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, boolean openSearcher, int commitIntervalLowerBound,
int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime) {
int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean commitWithinSoftCommit) {
this.className = className;
this.autoCommmitMaxDocs = autoCommmitMaxDocs;
this.autoCommmitMaxTime = autoCommmitMaxTime;
@ -418,6 +420,8 @@ public class SolrConfig extends Config {
this.autoSoftCommmitMaxDocs = autoSoftCommmitMaxDocs;
this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime;
this.commitWithinSoftCommit = commitWithinSoftCommit;
}
}

15
solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
View File

@ -92,6 +92,8 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
protected final CommitTracker commitTracker;
protected final CommitTracker softCommitTracker;
protected boolean commitWithinSoftCommit;
public DirectUpdateHandler2(SolrCore core) {
super(core);
@ -106,6 +108,8 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs", -1);
int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime", -1);
softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true);
commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
}
public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) {
@ -126,6 +130,8 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
if (this.ulog != null) {
this.ulog.init(this, core);
}
commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
}
private void deleteAll() throws IOException {
@ -229,8 +235,13 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
}
if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
if (commitWithinSoftCommit) {
commitTracker.addedDocument(-1);
softCommitTracker.addedDocument(cmd.commitWithin);
} else {
softCommitTracker.addedDocument(-1);
commitTracker.addedDocument(cmd.commitWithin);
}
}
rc = 1;
@ -252,7 +263,11 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
private void updateDeleteTrackers(DeleteUpdateCommand cmd) {
if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
if (commitWithinSoftCommit) {
softCommitTracker.deletedDocument(cmd.commitWithin);
} else {
commitTracker.deletedDocument(cmd.commitWithin);
}
if (commitTracker.getTimeUpperBound() > 0) {
commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());

4
solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
View File

@ -98,6 +98,10 @@
<str name="dir">${solr.ulog.dir:}</str>
</updateLog>
<commitWithin>
<softCommit>${solr.commitwithin.softcommit:true}</softCommit>
</commitWithin>
</updateHandler>

2
solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
View File

@ -286,7 +286,7 @@ public class AutoCommitTest extends AbstractSolrTestCase {
MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
// Add a single document with commitWithin == 1 second
// Add a single document with commitWithin == 2 second
SolrQueryResponse rsp = new SolrQueryResponse();
SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
req.setContentStreams( toContentStreams(

136
solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java Normal file
View File

@ -0,0 +1,136 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.update;
import java.util.HashMap;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.UpdateRequestHandler;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@Slow
public class HardAutoCommitTest extends AbstractSolrTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
System.setProperty("solr.commitwithin.softcommit", "false");
initCore("solrconfig.xml", "schema.xml");
}
@AfterClass
public static void afterClass() {
System.clearProperty("solr.commitwithin.softcommit");
}
@Override
public void setUp() throws Exception {
super.setUp();
clearIndex();
// reload the core to clear stats
h.getCoreContainer().reload(h.getCore().getName());
}
public void testCommitWithin() throws Exception {
SolrCore core = h.getCore();
NewSearcherListener trigger = new NewSearcherListener();
core.registerNewSearcherListener(trigger);
DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler();
CommitTracker tracker = updater.commitTracker;
tracker.setTimeUpperBound(0);
tracker.setDocsUpperBound(-1);
UpdateRequestHandler handler = new UpdateRequestHandler();
handler.init( null );
MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
// Add a single document with commitWithin == 2 second
SolrQueryResponse rsp = new SolrQueryResponse();
SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
req.setContentStreams( AutoCommitTest.toContentStreams(
adoc(2000, "id", "529", "field_t", "what's inside?", "subject", "info"), null ) );
trigger.reset();
handler.handleRequest( req, rsp );
// Check it isn't in the index
assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" );
// Wait longer than the commitWithin time
assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
// Add one document without commitWithin
req.setContentStreams( AutoCommitTest.toContentStreams(
adoc("id", "530", "field_t", "what's inside?", "subject", "info"), null ) );
trigger.reset();
handler.handleRequest( req, rsp );
// Check it isn't in the index
assertQ("shouldn't find any", req("id:530") ,"//result[@numFound=0]" );
// Delete one document with commitWithin
req.setContentStreams( AutoCommitTest.toContentStreams(
delI("529", "commitWithin", "1000"), null ) );
trigger.reset();
handler.handleRequest( req, rsp );
// Now make sure we can find it
assertQ("should find one", req("id:529") ,"//result[@numFound=1]" );
// Wait for the commit to happen
assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
// Now we shouldn't find it
assertQ("should find none", req("id:529") ,"//result[@numFound=0]" );
// ... but we should find the new one
assertQ("should find one", req("id:530") ,"//result[@numFound=1]" );
trigger.reset();
// now make the call 10 times really fast and make sure it
// only commits once
req.setContentStreams( AutoCommitTest.toContentStreams(
adoc(2000, "id", "500" ), null ) );
for( int i=0;i<10; i++ ) {
handler.handleRequest( req, rsp );
}
assertQ("should not be there yet", req("id:500") ,"//result[@numFound=0]" );
// the same for the delete
req.setContentStreams( AutoCommitTest.toContentStreams(
delI("530", "commitWithin", "1000"), null ) );
for( int i=0;i<10; i++ ) {
handler.handleRequest( req, rsp );
}
assertQ("should be there", req("id:530") ,"//result[@numFound=1]" );
assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
assertQ("should be there", req("id:500") ,"//result[@numFound=1]" );
assertQ("should not be there", req("id:530") ,"//result[@numFound=0]" );
assertEquals(3, tracker.getCommitCount());
}
}