Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-2578: ReplicationHandler's backup command now supports a 'numberToKeep' param that can be used to delete all but the most recent N backups. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1202969 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris M. Hostetter 2011-11-17 01:33:21 +00:00
parent 873f199924
commit 2a94133a2c
4 changed files with 151 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
solr/CHANGES.txt
View File

@ -396,6 +396,10 @@ New Features
* SOLR-1023: StatsComponent now supports date fields and string fields.
(Chris Male, Mark Holland, Gunnlaugur Thor Briem, Ryan McKinley)
* SOLR-2578: ReplicationHandler's backup command now supports a 'numberToKeep'
param that can be used to delete all but the most recent N backups.
(James Dyer via hossman)
Optimizations
----------------------

5
solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
View File

@ -296,6 +296,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
private void doSnapShoot(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) {
try {
int numberToKeep = params.getInt(NUMBER_BACKUPS_TO_KEEP, Integer.MAX_VALUE);
IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy();
IndexCommit indexCommit = delPolicy.getLatestCommit();
@ -304,7 +305,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
}
// small race here before the commit point is saved
new SnapShooter(core, params.get("location")).createSnapAsync(indexCommit, this);
new SnapShooter(core, params.get("location")).createSnapAsync(indexCommit, numberToKeep, this);
} catch (Exception e) {
LOG.warn("Exception during creating a snapshot", e);
@ -1146,4 +1147,6 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
public static final String OK_STATUS = "OK";
public static final String NEXT_EXECUTION_AT = "nextExecutionAt";
public static final String NUMBER_BACKUPS_TO_KEEP = "numberToKeep";
}

56
solr/core/src/java/org/apache/solr/handler/SnapShooter.java
View File

@ -22,9 +22,14 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.index.IndexCommit;
@ -61,17 +66,21 @@ public class SnapShooter {
}
void createSnapAsync(final IndexCommit indexCommit, final ReplicationHandler replicationHandler) {
createSnapAsync(indexCommit, Integer.MAX_VALUE, replicationHandler);
}
void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, final ReplicationHandler replicationHandler) {
replicationHandler.core.getDeletionPolicy().saveCommitPoint(indexCommit.getVersion());
new Thread() {
@Override
public void run() {
createSnapshot(indexCommit, replicationHandler);
createSnapshot(indexCommit, numberToKeep, replicationHandler);
}
}.start();
}
void createSnapshot(final IndexCommit indexCommit, ReplicationHandler replicationHandler) {
void createSnapshot(final IndexCommit indexCommit, int numberToKeep, ReplicationHandler replicationHandler) {
NamedList<Object> details = new NamedList<Object>();
details.add("startTime", new Date().toString());
@ -79,6 +88,9 @@ public class SnapShooter {
String directoryName = null;
Lock lock = null;
try {
if(numberToKeep<Integer.MAX_VALUE) {
deleteOldBackups(numberToKeep);
}
SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT, Locale.US);
directoryName = "snapshot." + fmt.format(new Date());
lock = lockFactory.makeLock(directoryName + ".lock");
@ -111,6 +123,46 @@ public class SnapShooter {
}
}
}
private void deleteOldBackups(int numberToKeep) {
File[] files = new File(snapDir).listFiles();
List<OldBackupDirectory> dirs = new ArrayList<OldBackupDirectory>();
for(File f : files) {
OldBackupDirectory obd = new OldBackupDirectory(f);
if(obd.dir != null) {
dirs.add(obd);
}
}
Collections.sort(dirs);
int i=1;
for(OldBackupDirectory dir : dirs) {
if( i > numberToKeep-1 ) {
SnapPuller.delTree(dir.dir);
}
}
}
private class OldBackupDirectory implements Comparable<OldBackupDirectory>{
File dir;
Date timestamp;
final Pattern dirNamePattern = Pattern.compile("^snapshot[.](.*)$");
OldBackupDirectory(File dir) {
if(dir.isDirectory()) {
Matcher m = dirNamePattern.matcher(dir.getName());
if(m.find()) {
try {
this.dir = dir;
this.timestamp = new SimpleDateFormat(DATE_FMT).parse(m.group(1));
} catch(Exception e) {
this.dir = null;
this.timestamp = null;
}
}
}
}
public int compareTo(OldBackupDirectory that) {
return that.timestamp.compareTo(this.timestamp);
}
}
public static final String SNAP_DIR = "snapDir";
public static final String DATE_FMT = "yyyyMMddHHmmss";

52
solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
View File

@ -25,6 +25,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.index.IndexReader;
@ -753,7 +755,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
volatile String fail = null;
@Override
public void run() {
String masterUrl = "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_BACKUP;
String masterUrl = "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_BACKUP + "&" + ReplicationHandler.NUMBER_BACKUPS_TO_KEEP + "=1";
URL url;
InputStream stream = null;
try {
@ -768,14 +770,18 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
};
};
BackupThread backupThread = new BackupThread();
backupThread.start();
File dataDir = new File(master.getDataDir());
class CheckStatus extends Thread {
volatile String fail = null;
volatile String response = null;
volatile boolean success = false;
volatile String backupTimestamp = null;
final String lastBackupTimestamp;
final Pattern p = Pattern.compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
CheckStatus(String lastBackupTimestamp) {
this.lastBackupTimestamp = lastBackupTimestamp;
}
@Override
public void run() {
String masterUrl = "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_DETAILS;
@ -786,8 +792,15 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
stream = url.openStream();
response = IOUtils.toString(stream, "UTF-8");
if(response.contains("<str name=\"status\">success</str>")) {
Matcher m = p.matcher(response);
if(!m.find()) {
fail("could not find the completed timestamp in response.");
}
backupTimestamp = m.group(1);
if(!backupTimestamp.equals(lastBackupTimestamp)) {
success = true;
}
}
stream.close();
} catch (Exception e) {
fail = e.getMessage();
@ -797,14 +810,27 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
};
};
File[] snapDir = new File[2];
String firstBackupTimestamp = null;
for(int i=0 ; i<2 ; i++) {
BackupThread backupThread = new BackupThread();
backupThread.start();
File dataDir = new File(master.getDataDir());
int waitCnt = 0;
CheckStatus checkStatus = new CheckStatus();
CheckStatus checkStatus = new CheckStatus(firstBackupTimestamp);
while(true) {
checkStatus.run();
if(checkStatus.fail != null) {
fail(checkStatus.fail);
}
if(checkStatus.success) {
if(i==0) {
firstBackupTimestamp = checkStatus.backupTimestamp;
Thread.sleep(1000); //ensure the next backup will have a different timestamp.
}
break;
}
Thread.sleep(200);
@ -828,17 +854,23 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
}
});
assertEquals(1, files.length);
File snapDir = files[0];
Directory dir = new SimpleFSDirectory(snapDir.getAbsoluteFile());
snapDir[i] = files[0];
Directory dir = new SimpleFSDirectory(snapDir[i].getAbsoluteFile());
IndexReader reader = IndexReader.open(dir);
IndexSearcher searcher = new IndexSearcher(reader);
TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
assertEquals(nDocs, hits.totalHits);
searcher.close();
reader.close();
searcher.close();
dir.close();
AbstractSolrTestCase.recurseDelete(snapDir); // clean up the snap dir
}
if(snapDir[0].exists()) {
fail("The first backup should have been cleaned up because " + ReplicationHandler.NUMBER_BACKUPS_TO_KEEP + " was set to 1");
}
for(int i=0 ; i< snapDir.length ; i++) {
AbstractSolrTestCase.recurseDelete(snapDir[i]); // clean up the snap dir
}
}
/* character copy of file using UTF-8 */