SOLR-3062: implement openSearcher=false, make commitWithin soft, refactor commit param parsing

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1239155 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2012-02-01 14:51:07 +00:00
parent 3d55e144ac
commit 8e357cac55
13 changed files with 139 additions and 132 deletions

View File

@ -63,6 +63,7 @@ Upgrading from Solr 3.6-dev
renamed from "update.processor" to "update.chain". The old parameter was
deprecated but still working since Solr3.2, but is now removed entirely.
Detailed Change List
----------------------
@ -195,6 +196,10 @@ New Features
* SOLR-2906: Added LFU cache options to Solr. (Shawn Heisey via Erick Erickson)
* SOLR-3069: Ability to add openSearcher=false to not open a searcher when doing
a hard commit. commitWithin now only invokes a softCommit. (yonik)
Optimizations
----------------------

View File

@ -204,6 +204,7 @@ public class SolrConfig extends Config {
return new UpdateHandlerInfo(get("updateHandler/@class",null),
getInt("updateHandler/autoCommit/maxDocs",-1),
getInt("updateHandler/autoCommit/maxTime",-1),
getBool("updateHandler/autoCommit/openSearcher",true),
getInt("updateHandler/commitIntervalLowerBound",-1),
getInt("updateHandler/autoSoftCommit/maxDocs",-1),
getInt("updateHandler/autoSoftCommit/maxTime",-1));
@ -365,6 +366,7 @@ public class SolrConfig extends Config {
public final String className;
public final int autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound,
autoSoftCommmitMaxDocs,autoSoftCommmitMaxTime;
public final boolean openSearcher; // is opening a new searcher part of hard autocommit?
/**
* @param className
@ -372,11 +374,12 @@ public class SolrConfig extends Config {
* @param autoCommmitMaxTime set -1 as default
* @param commitIntervalLowerBound set -1 as default
*/
public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, int commitIntervalLowerBound,
public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, boolean openSearcher, int commitIntervalLowerBound,
int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime) {
this.className = className;
this.autoCommmitMaxDocs = autoCommmitMaxDocs;
this.autoCommmitMaxTime = autoCommmitMaxTime;
this.openSearcher = openSearcher;
this.commitIntervalLowerBound = commitIntervalLowerBound;
this.autoSoftCommmitMaxDocs = autoSoftCommmitMaxDocs;

View File

@ -19,14 +19,18 @@ package org.apache.solr.handler;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Iterator;
import java.util.Map;
import java.util.Stack;
import org.apache.commons.io.IOUtils;
import org.apache.noggit.JSONParser;
import org.apache.noggit.JSONUtil;
import org.apache.noggit.ObjectBuilder;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.params.*;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@ -203,34 +207,30 @@ class JsonLoader extends ContentStreamLoader {
void parseCommitOptions(CommitUpdateCommand cmd ) throws IOException
{
assertNextEvent( JSONParser.OBJECT_START );
final Map<String,Object> map = (Map)ObjectBuilder.getVal(parser);
while( true ) {
int ev = parser.nextEvent();
if( ev == JSONParser.STRING ) {
String key = parser.getString();
if( parser.wasKey() ) {
if( XmlUpdateRequestHandler.WAIT_SEARCHER.equals( key ) ) {
cmd.waitSearcher = parser.getBoolean();
}
else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown key: "+key+" ["+parser.getPosition()+"]" );
}
}
else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"invalid string: " + key
+" at ["+parser.getPosition()+"]" );
}
// SolrParams currently expects string values...
SolrParams p = new SolrParams() {
@Override
public String get(String param) {
Object o = map.get(param);
return o == null ? null : o.toString();
}
else if( ev == JSONParser.OBJECT_END ) {
return;
@Override
public String[] getParams(String param) {
return new String[]{get(param)};
}
else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Got: "+JSONParser.getEventString( ev )
+" at ["+parser.getPosition()+"]" );
@Override
public Iterator<String> getParameterNamesIterator() {
return map.keySet().iterator();
}
}
};
RequestHandlerUtils.validateCommitParams(p);
p = SolrParams.wrapDefaults(p, req.getParams()); // default to the normal request params for commit options
RequestHandlerUtils.updateCommit(cmd, p);
}
AddUpdateCommand parseAdd() throws IOException

View File

@ -18,8 +18,9 @@
package org.apache.solr.handler;
import java.io.IOException;
import java.util.HashMap;
import java.util.*;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
@ -44,69 +45,27 @@ public class RequestHandlerUtils
{
rsp.add( "WARNING", "This response format is experimental. It is likely to change in the future." );
}
/**
* Check the request parameters and decide if it should commit or optimize.
* If it does, it will check parameters for "waitFlush" and "waitSearcher"
*
* @deprecated Use {@link #handleCommit(SolrQueryRequest,UpdateRequestProcessor,SolrParams,boolean)}
*
* @since solr 1.2
*/
@Deprecated
public static boolean handleCommit( SolrQueryRequest req, SolrQueryResponse rsp, boolean force ) throws IOException
{
SolrParams params = req.getParams();
if( params == null ) {
params = new MapSolrParams( new HashMap<String, String>() );
}
boolean optimize = params.getBool( UpdateParams.OPTIMIZE, false );
boolean commit = params.getBool( UpdateParams.COMMIT, false );
if( optimize || commit || force ) {
CommitUpdateCommand cmd = new CommitUpdateCommand(req, optimize );
cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, cmd.waitSearcher );
cmd.softCommit = params.getBool( UpdateParams.SOFT_COMMIT, cmd.softCommit );
cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, cmd.expungeDeletes);
cmd.maxOptimizeSegments = params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
req.getCore().getUpdateHandler().commit( cmd );
// Lets wait till after solr1.2 to define consistent output format
//if( optimize ) {
// rsp.add( "optimize", true );
//}
//else {
// rsp.add( "commit", true );
//}
return true;
}
return false;
}
/**
* Check the request parameters and decide if it should commit or optimize.
* If it does, it will check parameters for "waitFlush" and "waitSearcher"
* If it does, it will check other related parameters such as "waitFlush" and "waitSearcher"
*/
public static boolean handleCommit(SolrQueryRequest req, UpdateRequestProcessor processor, SolrParams params, boolean force ) throws IOException
{
if( params == null ) {
if( params == null) {
params = new MapSolrParams( new HashMap<String, String>() );
}
boolean optimize = params.getBool( UpdateParams.OPTIMIZE, false );
boolean commit = params.getBool( UpdateParams.COMMIT, false );
boolean softCommit = params.getBool( UpdateParams.SOFT_COMMIT, false );
boolean prepareCommit = params.getBool( UpdateParams.PREPARE_COMMIT, false );
if( optimize || commit || prepareCommit || force ) {
if( optimize || commit || softCommit || prepareCommit || force ) {
CommitUpdateCommand cmd = new CommitUpdateCommand(req, optimize );
cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, cmd.waitSearcher );
cmd.softCommit = params.getBool( UpdateParams.SOFT_COMMIT, cmd.softCommit );
cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, cmd.expungeDeletes);
cmd.maxOptimizeSegments = params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
cmd.prepareCommit = prepareCommit;
updateCommit(cmd, params);
processor.processCommit( cmd );
return true;
}
@ -115,6 +74,35 @@ public class RequestHandlerUtils
return false;
}
private static Set<String> commitParams = new HashSet<String>(Arrays.asList(new String[]{UpdateParams.OPEN_SEARCHER, UpdateParams.WAIT_SEARCHER, UpdateParams.SOFT_COMMIT, UpdateParams.EXPUNGE_DELETES, UpdateParams.MAX_OPTIMIZE_SEGMENTS, UpdateParams.PREPARE_COMMIT}));
public static void validateCommitParams(SolrParams params) {
Iterator<String> i = params.getParameterNamesIterator();
while (i.hasNext()) {
String key = i.next();
if (!commitParams.contains(key)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown commit parameter '" + key + "'");
}
}
}
/**
* Modify UpdateCommand based on request parameters
*/
public static void updateCommit(CommitUpdateCommand cmd, SolrParams params) throws IOException
{
if( params == null ) return;
cmd.openSearcher = params.getBool( UpdateParams.OPEN_SEARCHER, cmd.openSearcher );
cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, cmd.waitSearcher );
cmd.softCommit = params.getBool( UpdateParams.SOFT_COMMIT, cmd.softCommit );
cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, cmd.expungeDeletes );
cmd.maxOptimizeSegments = params.getInt( UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments );
cmd.prepareCommit = params.getBool( UpdateParams.PREPARE_COMMIT, cmd.prepareCommit );
}
/**
* @since Solr 1.4
*/

View File

@ -16,6 +16,7 @@ package org.apache.solr.handler;
* limitations under the License.
*/
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.CommitUpdateCommand;
@ -139,23 +140,18 @@ class XMLLoader extends ContentStreamLoader {
XmlUpdateRequestHandler.log.trace("parsing " + currTag);
CommitUpdateCommand cmd = new CommitUpdateCommand(req, XmlUpdateRequestHandler.OPTIMIZE.equals(currTag));
ModifiableSolrParams mp = new ModifiableSolrParams();
for (int i = 0; i < parser.getAttributeCount(); i++) {
String attrName = parser.getAttributeLocalName(i);
String attrVal = parser.getAttributeValue(i);
if (XmlUpdateRequestHandler.WAIT_SEARCHER.equals(attrName)) {
cmd.waitSearcher = StrUtils.parseBoolean(attrVal);
} else if (XmlUpdateRequestHandler.SOFT_COMMIT.equals(attrName)) {
cmd.softCommit = StrUtils.parseBoolean(attrVal);
} else if (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) {
cmd.maxOptimizeSegments = Integer.parseInt(attrVal);
} else if (UpdateParams.EXPUNGE_DELETES.equals(attrName)) {
cmd.expungeDeletes = StrUtils.parseBoolean(attrVal);
} else {
XmlUpdateRequestHandler.log.warn("unexpected attribute commit/@" + attrName);
}
mp.set(attrName, attrVal);
}
RequestHandlerUtils.validateCommitParams(mp);
SolrParams p = SolrParams.wrapDefaults(mp, req.getParams()); // default to the normal request params for commit options
RequestHandlerUtils.updateCommit(cmd, p);
processor.processCommit(cmd);
} // end commit
else if (XmlUpdateRequestHandler.ROLLBACK.equals(currTag)) {

View File

@ -61,11 +61,12 @@ final class CommitTracker implements Runnable {
private final SolrCore core;
private final boolean softCommit;
private final boolean waitSearcher;
private final boolean openSearcher;
private final boolean waitSearcher = true;
private String name;
public CommitTracker(String name, SolrCore core, int docsUpperBound, int timeUpperBound, boolean waitSearcher, boolean softCommit) {
public CommitTracker(String name, SolrCore core, int docsUpperBound, int timeUpperBound, boolean openSearcher, boolean softCommit) {
this.core = core;
this.name = name;
pending = null;
@ -74,7 +75,7 @@ final class CommitTracker implements Runnable {
this.timeUpperBound = timeUpperBound;
this.softCommit = softCommit;
this.waitSearcher = waitSearcher;
this.openSearcher = openSearcher;
SolrCore.log.info(name + " AutoCommit: " + this);
}
@ -183,6 +184,7 @@ final class CommitTracker implements Runnable {
new ModifiableSolrParams());
try {
CommitUpdateCommand command = new CommitUpdateCommand(req, false);
command.openSearcher = openSearcher;
command.waitSearcher = waitSearcher;
command.softCommit = softCommit;
// no need for command.maxOptimizeSegments = 1; since it is not optimizing

View File

@ -24,6 +24,7 @@ import org.apache.solr.request.SolrQueryRequest;
*/
public class CommitUpdateCommand extends UpdateCommand {
public boolean optimize;
public boolean openSearcher=true; // open a new searcher as part of a hard commit
public boolean waitSearcher=true;
public boolean expungeDeletes = false;
public boolean softCommit = false;
@ -49,6 +50,7 @@ public class CommitUpdateCommand extends UpdateCommand {
@Override
public String toString() {
return super.toString() + ",optimize="+optimize
+",openSearcher="+openSearcher
+",waitSearcher="+waitSearcher
+",expungeDeletes="+expungeDeletes
+",softCommit="+softCommit

View File

@ -93,7 +93,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
.getUpdateHandlerInfo();
int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs; // getInt("updateHandler/autoCommit/maxDocs", -1);
int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime; // getInt("updateHandler/autoCommit/maxTime", -1);
commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, true, false);
commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, updateHandlerInfo.openSearcher, false);
int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs", -1);
int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime", -1);
@ -114,11 +114,11 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
.getUpdateHandlerInfo();
int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs; // getInt("updateHandler/autoCommit/maxDocs", -1);
int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime; // getInt("updateHandler/autoCommit/maxTime", -1);
commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, true, false);
commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, updateHandlerInfo.openSearcher, false);
int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs", -1);
int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime", -1);
softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true);
softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, updateHandlerInfo.openSearcher, true);
this.ulog = updateHandler.getUpdateLog();
if (this.ulog != null) {
@ -186,8 +186,8 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
if (ulog != null) ulog.add(cmd);
if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
commitTracker.addedDocument( cmd.commitWithin );
softCommitTracker.addedDocument( -1 ); // TODO: support commitWithin with soft update
commitTracker.addedDocument( -1 );
softCommitTracker.addedDocument( cmd.commitWithin );
}
rc = 1;
@ -203,6 +203,20 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
return rc;
}
private void updateDeleteTrackers(DeleteUpdateCommand cmd) {
if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
softCommitTracker.deletedDocument( cmd.commitWithin );
if (commitTracker.getTimeUpperBound() > 0) {
commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());
}
if (softCommitTracker.getTimeUpperBound() > 0) {
softCommitTracker.scheduleCommitWithin(softCommitTracker.getTimeUpperBound());
}
}
}
// we don't return the number of docs deleted because it's not always possible to quickly know that info.
@Override
@ -214,21 +228,12 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
Term deleteTerm = new Term(idField.getName(), cmd.getIndexedId());
// SolrCore.verbose("deleteDocuments",deleteTerm,writer);
commitTracker.deletedDocument( cmd.commitWithin );
writer.deleteDocuments(deleteTerm);
// SolrCore.verbose("deleteDocuments",deleteTerm,"DONE");
if (ulog != null) ulog.delete(cmd);
if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
if (commitTracker.getTimeUpperBound() > 0) {
commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());
}
if (softCommitTracker.getTimeUpperBound() > 0) {
softCommitTracker.scheduleCommitWithin(softCommitTracker.getTimeUpperBound());
}
}
updateDeleteTrackers(cmd);
}
// we don't return the number of docs deleted because it's not always possible to quickly know that info.
@ -267,8 +272,6 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
boolean delAll = MatchAllDocsQuery.class == q.getClass();
commitTracker.deletedDocument(cmd.commitWithin);
//
// synchronized to prevent deleteByQuery from running during the "open new searcher"
// part of a commit. DBQ needs to signal that a fresh reader will be needed for
@ -286,15 +289,9 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
}
madeIt = true;
if (commitTracker.getTimeUpperBound() > 0) {
commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());
}
if (softCommitTracker.getTimeUpperBound()> 0) {
softCommitTracker.scheduleCommitWithin(softCommitTracker.getTimeUpperBound());
}
updateDeleteTrackers(cmd);
} finally {
if (!madeIt) {
numErrors.incrementAndGet();
@ -416,7 +413,12 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
} else {
synchronized (this) {
if (ulog != null) ulog.preSoftCommit(cmd);
core.getSearcher(true, false, waitSearcher);
if (cmd.openSearcher) {
core.getSearcher(true, false, waitSearcher);
} else {
// force open a new realtime searcher so realtime-get and versioning code can see the latest
core.openNewSearcher(true,true);
}
if (ulog != null) ulog.postSoftCommit(cmd);
}
if (ulog != null) ulog.postCommit(cmd); // postCommit currently means new searcher has

View File

@ -65,7 +65,7 @@ public class JsonLoaderTest extends SolrTestCaseJ4 {
"},\n" +
"\n" +
"'commit': {},\n" +
"'optimize': { 'waitSearcher':false },\n" +
"'optimize': { 'waitSearcher':false, 'openSearcher':false },\n" +
"\n" +
"'delete': { 'id':'ID' },\n" +
"'delete': { 'id':'ID', 'commitWithin':'500' },\n" +
@ -108,11 +108,13 @@ public class JsonLoaderTest extends SolrTestCaseJ4 {
CommitUpdateCommand commit = p.commitCommands.get( 0 );
assertFalse( commit.optimize );
assertTrue( commit.waitSearcher );
assertTrue( commit.openSearcher );
commit = p.commitCommands.get( 1 );
assertTrue( commit.optimize );
assertFalse( commit.waitSearcher );
assertFalse( commit.openSearcher );
// DELETE COMMANDS
assertEquals( 4, p.deleteCommands.size() );

View File

@ -144,7 +144,7 @@ public class AutoCommitTest extends AbstractSolrTestCase {
NewSearcherListener trigger = new NewSearcherListener();
DirectUpdateHandler2 updateHandler = (DirectUpdateHandler2)core.getUpdateHandler();
CommitTracker tracker = updateHandler.commitTracker;
CommitTracker tracker = updateHandler.softCommitTracker;
tracker.setTimeUpperBound(-1);
tracker.setDocsUpperBound(14);
core.registerNewSearcherListener(trigger);
@ -191,7 +191,7 @@ public class AutoCommitTest extends AbstractSolrTestCase {
NewSearcherListener trigger = new NewSearcherListener();
core.registerNewSearcherListener(trigger);
DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler();
CommitTracker tracker = updater.commitTracker;
CommitTracker tracker = updater.softCommitTracker;
// too low of a number can cause a slow host to commit before the test code checks that it
// isn't there... causing a failure at "shouldn't find any"
tracker.setTimeUpperBound(1000);
@ -264,7 +264,7 @@ public class AutoCommitTest extends AbstractSolrTestCase {
NewSearcherListener trigger = new NewSearcherListener();
core.registerNewSearcherListener(trigger);
DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler();
CommitTracker tracker = updater.commitTracker;
CommitTracker tracker = updater.softCommitTracker;
tracker.setTimeUpperBound(0);
tracker.setDocsUpperBound(-1);

View File

@ -24,5 +24,7 @@ for f in $FILES; do
done
#send the commit command to make sure all the changes are flushed and visible
curl $URL --data-binary '<commit/>' -H 'Content-type:application/xml'
#curl $URL --data-binary '<commit softCommit=true/>' -H 'Content-type:application/xml'
curl "$URL?softCommit=true"
echo

View File

@ -291,12 +291,14 @@
maxTime - Maximum amount of time in ms that is allowed to pass
since a document was added before automaticly
triggering a new commit.
openSearcher - if false, the commit causes recent index changes
to be flushed to stable storage, but does not cause a new
searcher to be opened to make those changes visible.
-->
<!--
<autoCommit>
<maxTime>60000</maxTime>
</autoCommit>
-->
<autoCommit>
<maxTime>15000</maxTime>
<openSearcher>false</openSearcher>
</autoCommit>
<!-- softAutoCommit is like autoCommit except it causes a
'soft' commit which only ensures that changes are visible

View File

@ -25,10 +25,13 @@ package org.apache.solr.common.params;
*/
public interface UpdateParams
{
/** wait for the search to warm up */
/** Open up a new searcher as part of a commit */
public static String OPEN_SEARCHER = "openSearcher";
/** wait for the searcher to be registered/visible */
public static String WAIT_SEARCHER = "waitSearcher";
public static String SOFT_COMMIT = "softCommit";
/** overwrite indexing fields */