SOLR-3033: "numberToKeep" on replication handler does not work with "backupAfter"

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1294702 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
James Dyer 2012-02-28 15:50:31 +00:00
parent 3e14f86161
commit af77642d82
4 changed files with 115 additions and 12 deletions

View File

@ -518,6 +518,9 @@ New Features
* SOLR-3143: Add SuggestQueryConverter, a QueryConverter intended for
auto-suggesters. (Robert Muir)
* SOLR-3033: ReplicationHandler's backup command now supports a 'maxNumberOfBackups'
init param that can be used to delete all but the most recent N backups. (Torsten Krah, James Dyer)
Optimizations
----------------------
* SOLR-1931: Speedup for LukeRequestHandler and admin/schema browser. New parameter
@ -679,7 +682,7 @@ New Features
(Chris Male, Mark Holland, Gunnlaugur Thor Briem, Ryan McKinley)
* SOLR-2578: ReplicationHandler's backup command now supports a 'numberToKeep'
param that can be used to delete all but the most recent N backups.
request param that can be used to delete all but the most recent N backups.
(James Dyer via hossman)
Optimizations

View File

@ -45,6 +45,7 @@ import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
@ -105,6 +106,8 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
private boolean replicateOnCommit = false;
private boolean replicateOnStart = false;
private int numberBackupsToKeep = 0; //zero: do not delete old backups
private int numTimesReplicated = 0;
@ -308,18 +311,31 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
return snapPullLock.isLocked();
}
private void doSnapShoot(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) {
private void doSnapShoot(SolrParams params, SolrQueryResponse rsp,
SolrQueryRequest req) {
try {
int numberToKeep = params.getInt(NUMBER_BACKUPS_TO_KEEP, Integer.MAX_VALUE);
int numberToKeep = params.getInt(NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM, 0);
if (numberToKeep > 0 && numberBackupsToKeep > 0) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Cannot use "
+ NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM + " if "
+ NUMBER_BACKUPS_TO_KEEP_INIT_PARAM
+ " was specified in the configuration.");
}
numberToKeep = Math.max(numberToKeep, numberBackupsToKeep);
if (numberToKeep < 1) {
numberToKeep = Integer.MAX_VALUE;
}
IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy();
IndexCommit indexCommit = delPolicy.getLatestCommit();
if(indexCommit == null) {
if (indexCommit == null) {
indexCommit = req.getSearcher().getIndexReader().getIndexCommit();
}
// small race here before the commit point is saved
new SnapShooter(core, params.get("location")).createSnapAsync(indexCommit, numberToKeep, this);
new SnapShooter(core, params.get("location")).createSnapAsync(
indexCommit, numberToKeep, this);
} catch (Exception e) {
LOG.warn("Exception during creating a snapshot", e);
@ -790,6 +806,12 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
this.core = core;
registerFileStreamResponseWriter();
registerCloseHook();
Object nbtk = initArgs.get(NUMBER_BACKUPS_TO_KEEP_INIT_PARAM);
if(nbtk!=null) {
numberBackupsToKeep = Integer.parseInt(nbtk.toString());
} else {
numberBackupsToKeep = 0;
}
NamedList slave = (NamedList) initArgs.get("slave");
boolean enableSlave = isEnabled( slave );
if (enableSlave) {
@ -1179,5 +1201,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
public static final String NEXT_EXECUTION_AT = "nextExecutionAt";
public static final String NUMBER_BACKUPS_TO_KEEP = "numberToKeep";
public static final String NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM = "numberToKeep";
public static final String NUMBER_BACKUPS_TO_KEEP_INIT_PARAM = "maxNumberOfBackups";
}

View File

@ -0,0 +1,59 @@
<?xml version="1.0" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<dataDir>${solr.data.dir:}</dataDir>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
<indexDefaults />
<mainIndex>
<useCompoundFile>false</useCompoundFile>
<mergeFactor>10</mergeFactor>
<ramBufferSizeMB>32</ramBufferSizeMB>
<maxMergeDocs>2147483647</maxMergeDocs>
<maxFieldLength>10000</maxFieldLength>
<mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler"/>
<writeLockTimeout>1000</writeLockTimeout>
<unlockOnStartup>true</unlockOnStartup>
<lockType>single</lockType>
</mainIndex>
<updateHandler class="solr.DirectUpdateHandler2">
</updateHandler>
<requestHandler name="standard" class="solr.StandardRequestHandler" />
<requestHandler name="/replication" class="solr.ReplicationHandler">
<lst name="master">
<str name="replicateAfter">commit</str>
<str name="confFiles">schema-replication2.xml:schema.xml</str>
</lst>
<str name="maxNumberOfBackups">1</str>
</requestHandler>
<requestHandler name="/update" class="solr.XmlUpdateRequestHandler"/>
<!-- enable streaming for testing... -->
<requestDispatcher handleSelect="true">
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048"/>
<httpCaching lastModifiedFrom="openTime" etagSeed="Solr" never304="false">
<cacheControl>max-age=30, public</cacheControl>
</httpCaching>
</requestDispatcher>
</config>

View File

@ -242,14 +242,14 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
}
public void test() throws Exception {
doTestReplicateAfterCoreReload();
/* doTestReplicateAfterCoreReload();
doTestDetails();
doTestReplicateAfterWrite2Slave();
doTestIndexAndConfigReplication();
doTestStopPoll();
doTestSnapPullWithMasterUrl();
doTestReplicateAfterStartup();
doTestIndexAndConfigAliasReplication();
doTestIndexAndConfigAliasReplication();*/
doTestBackup();
}
@ -769,8 +769,17 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
private void doTestBackup() throws Exception {
String configFile = "solrconfig-master1.xml";
boolean addNumberToKeepInRequest = true;
String backupKeepParamName = ReplicationHandler.NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM;
if(random.nextBoolean()) {
configFile = "solrconfig-master1-keepOneBackup.xml";
addNumberToKeepInRequest = false;
backupKeepParamName = ReplicationHandler.NUMBER_BACKUPS_TO_KEEP_INIT_PARAM;
}
masterJetty.stop();
master.copyConfigFile(CONF_DIR + "solrconfig-master1.xml",
master.copyConfigFile(CONF_DIR + configFile,
"solrconfig.xml");
masterJetty = createJetty(master);
@ -785,9 +794,17 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
class BackupThread extends Thread {
volatile String fail = null;
final boolean addNumberToKeepInRequest;
String backupKeepParamName;
BackupThread(boolean addNumberToKeepInRequest, String backupKeepParamName) {
this.addNumberToKeepInRequest = addNumberToKeepInRequest;
this.backupKeepParamName = backupKeepParamName;
}
@Override
public void run() {
String masterUrl = "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_BACKUP + "&" + ReplicationHandler.NUMBER_BACKUPS_TO_KEEP + "=1";
String masterUrl =
"http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_BACKUP +
(addNumberToKeepInRequest ? "&" + backupKeepParamName + "=1" : "");
URL url;
InputStream stream = null;
try {
@ -846,7 +863,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
File[] snapDir = new File[2];
String firstBackupTimestamp = null;
for(int i=0 ; i<2 ; i++) {
BackupThread backupThread = new BackupThread();
BackupThread backupThread = new BackupThread(addNumberToKeepInRequest, backupKeepParamName);
backupThread.start();
File dataDir = new File(master.getDataDir());
@ -896,7 +913,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
dir.close();
}
if(snapDir[0].exists()) {
fail("The first backup should have been cleaned up because " + ReplicationHandler.NUMBER_BACKUPS_TO_KEEP + " was set to 1");
fail("The first backup should have been cleaned up because " + backupKeepParamName + " was set to 1.");
}
for(int i=0 ; i< snapDir.length ; i++) {