Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-2578: ReplicationHandler's backup command now supports a 'numberToKeep' param that can be used to delete all but the most recent N backups. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1202969 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris M. Hostetter 2011-11-17 01:33:21 +00:00
parent 873f199924
commit 2a94133a2c
4 changed files with 151 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
solr/CHANGES.txt
View File

@ -396,6 +396,10 @@ New Features
* SOLR-1023: StatsComponent now supports date fields and string fields.
(Chris Male, Mark Holland, Gunnlaugur Thor Briem, Ryan McKinley)
* SOLR-2578: ReplicationHandler's backup command now supports a 'numberToKeep'
param that can be used to delete all but the most recent N backups.
(James Dyer via hossman)
Optimizations
----------------------

23
solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
View File

@ -126,7 +126,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
}
// This command does not give the current index version of the master
// It gives the current 'replicateable' index version
if (command.equals(CMD_INDEX_VERSION)) {
if (command.equals(CMD_INDEX_VERSION)) {
IndexCommit commitPoint = indexCommitPoint; // make a copy so it won't change
if (commitPoint != null && replicationEnabled.get()) {
//
@ -202,10 +202,10 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
} else if (CMD_ENABLE_REPL.equalsIgnoreCase(command)) {
replicationEnabled.set(true);
rsp.add(STATUS, OK_STATUS);
} else if (CMD_DISABLE_REPL.equalsIgnoreCase(command)) {
replicationEnabled.set(false);
rsp.add(STATUS, OK_STATUS);
}
} else if (CMD_DISABLE_REPL.equalsIgnoreCase(command)) {
replicationEnabled.set(false);
rsp.add(STATUS, OK_STATUS);
}
}
private List<NamedList<Object>> getCommits() {
@ -296,16 +296,17 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
private void doSnapShoot(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) {
try {
int numberToKeep = params.getInt(NUMBER_BACKUPS_TO_KEEP, Integer.MAX_VALUE);
IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy();
IndexCommit indexCommit = delPolicy.getLatestCommit();
if(indexCommit == null) {
indexCommit = req.getSearcher().getIndexReader().getIndexCommit();
}
// small race here before the commit point is saved
new SnapShooter(core, params.get("location")).createSnapAsync(indexCommit, this);
new SnapShooter(core, params.get("location")).createSnapAsync(indexCommit, numberToKeep, this);
} catch (Exception e) {
LOG.warn("Exception during creating a snapshot", e);
rsp.add("exception", e);
@ -354,7 +355,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
rsp.add("status", "unable to get file names for given indexversion");
rsp.add("exception", e);
LOG.warn("Unable to get file names for indexCommit version: "
+ version, e);
+ version, e);
}
rsp.add(CMD_GET_FILE_LIST, result);
if (confFileNameAlias.size() < 1)
@ -1146,4 +1147,6 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
public static final String OK_STATUS = "OK";
public static final String NEXT_EXECUTION_AT = "nextExecutionAt";
public static final String NUMBER_BACKUPS_TO_KEEP = "numberToKeep";
}

66
solr/core/src/java/org/apache/solr/handler/SnapShooter.java
View File

@ -22,9 +22,14 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.index.IndexCommit;
@ -59,19 +64,23 @@ public class SnapShooter {
}
lockFactory = new SimpleFSLockFactory(snapDir);
}
void createSnapAsync(final IndexCommit indexCommit, final ReplicationHandler replicationHandler) {
createSnapAsync(indexCommit, Integer.MAX_VALUE, replicationHandler);
}
void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, final ReplicationHandler replicationHandler) {
replicationHandler.core.getDeletionPolicy().saveCommitPoint(indexCommit.getVersion());
new Thread() {
@Override
public void run() {
createSnapshot(indexCommit, replicationHandler);
createSnapshot(indexCommit, numberToKeep, replicationHandler);
}
}.start();
}
void createSnapshot(final IndexCommit indexCommit, ReplicationHandler replicationHandler) {
void createSnapshot(final IndexCommit indexCommit, int numberToKeep, ReplicationHandler replicationHandler) {
NamedList<Object> details = new NamedList<Object>();
details.add("startTime", new Date().toString());
@ -79,6 +88,9 @@ public class SnapShooter {
String directoryName = null;
Lock lock = null;
try {
if(numberToKeep<Integer.MAX_VALUE) {
deleteOldBackups(numberToKeep);
}
SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT, Locale.US);
directoryName = "snapshot." + fmt.format(new Date());
lock = lockFactory.makeLock(directoryName + ".lock");
@ -100,8 +112,8 @@ public class SnapShooter {
LOG.error("Exception while creating snapshot", e);
details.add("snapShootException", e.getMessage());
} finally {
replicationHandler.core.getDeletionPolicy().releaseCommitPoint(indexCommit.getVersion());
replicationHandler.snapShootDetails = details;
replicationHandler.core.getDeletionPolicy().releaseCommitPoint(indexCommit.getVersion());
replicationHandler.snapShootDetails = details;
if (lock != null) {
try {
lock.release();
@ -111,6 +123,46 @@ public class SnapShooter {
}
}
}
private void deleteOldBackups(int numberToKeep) {
File[] files = new File(snapDir).listFiles();
List<OldBackupDirectory> dirs = new ArrayList<OldBackupDirectory>();
for(File f : files) {
OldBackupDirectory obd = new OldBackupDirectory(f);
if(obd.dir != null) {
dirs.add(obd);
}
}
Collections.sort(dirs);
int i=1;
for(OldBackupDirectory dir : dirs) {
if( i > numberToKeep-1 ) {
SnapPuller.delTree(dir.dir);
}
}
}
private class OldBackupDirectory implements Comparable<OldBackupDirectory>{
File dir;
Date timestamp;
final Pattern dirNamePattern = Pattern.compile("^snapshot[.](.*)$");
OldBackupDirectory(File dir) {
if(dir.isDirectory()) {
Matcher m = dirNamePattern.matcher(dir.getName());
if(m.find()) {
try {
this.dir = dir;
this.timestamp = new SimpleDateFormat(DATE_FMT).parse(m.group(1));
} catch(Exception e) {
this.dir = null;
this.timestamp = null;
}
}
}
}
public int compareTo(OldBackupDirectory that) {
return that.timestamp.compareTo(this.timestamp);
}
}
public static final String SNAP_DIR = "snapDir";
public static final String DATE_FMT = "yyyyMMddHHmmss";
@ -135,7 +187,7 @@ public class SnapShooter {
}
public void copyFile(File source, File destination, boolean preserveFileDate)
throws IOException {
throws IOException {
// check source exists
if (!source.exists()) {
String message = "File " + source + " does not exist";
@ -185,7 +237,7 @@ public class SnapShooter {
if (source.length() != destination.length()) {
String message = "Failed to copy full contents from " + source + " to "
+ destination;
+ destination;
throw new IOException(message);
}

118
solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
View File

@ -25,7 +25,9 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Writer;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
@ -753,7 +755,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
volatile String fail = null;
@Override
public void run() {
String masterUrl = "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_BACKUP;
String masterUrl = "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_BACKUP + "&" + ReplicationHandler.NUMBER_BACKUPS_TO_KEEP + "=1";
URL url;
InputStream stream = null;
try {
@ -768,14 +770,18 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
};
};
BackupThread backupThread = new BackupThread();
backupThread.start();
File dataDir = new File(master.getDataDir());
class CheckStatus extends Thread {
volatile String fail = null;
volatile String response = null;
volatile boolean success = false;
volatile String backupTimestamp = null;
final String lastBackupTimestamp;
final Pattern p = Pattern.compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
CheckStatus(String lastBackupTimestamp) {
this.lastBackupTimestamp = lastBackupTimestamp;
}
@Override
public void run() {
String masterUrl = "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_DETAILS;
@ -786,7 +792,14 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
stream = url.openStream();
response = IOUtils.toString(stream, "UTF-8");
if(response.contains("<str name=\"status\">success</str>")) {
success = true;
Matcher m = p.matcher(response);
if(!m.find()) {
fail("could not find the completed timestamp in response.");
}
backupTimestamp = m.group(1);
if(!backupTimestamp.equals(lastBackupTimestamp)) {
success = true;
}
}
stream.close();
} catch (Exception e) {
@ -797,48 +810,67 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
};
};
int waitCnt = 0;
CheckStatus checkStatus = new CheckStatus();
while(true) {
checkStatus.run();
if(checkStatus.fail != null) {
fail(checkStatus.fail);
File[] snapDir = new File[2];
String firstBackupTimestamp = null;
for(int i=0 ; i<2 ; i++) {
BackupThread backupThread = new BackupThread();
backupThread.start();
File dataDir = new File(master.getDataDir());
int waitCnt = 0;
CheckStatus checkStatus = new CheckStatus(firstBackupTimestamp);
while(true) {
checkStatus.run();
if(checkStatus.fail != null) {
fail(checkStatus.fail);
}
if(checkStatus.success) {
if(i==0) {
firstBackupTimestamp = checkStatus.backupTimestamp;
Thread.sleep(1000); //ensure the next backup will have a different timestamp.
}
break;
}
Thread.sleep(200);
if(waitCnt == 10) {
fail("Backup success not detected:" + checkStatus.response);
}
waitCnt++;
}
if(checkStatus.success) {
break;
if(backupThread.fail != null) {
fail(backupThread.fail);
}
Thread.sleep(200);
if(waitCnt == 10) {
fail("Backup success not detected:" + checkStatus.response);
}
waitCnt++;
File[] files = dataDir.listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
if(name.startsWith("snapshot")) {
return true;
}
return false;
}
});
assertEquals(1, files.length);
snapDir[i] = files[0];
Directory dir = new SimpleFSDirectory(snapDir[i].getAbsoluteFile());
IndexReader reader = IndexReader.open(dir);
IndexSearcher searcher = new IndexSearcher(reader);
TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
assertEquals(nDocs, hits.totalHits);
reader.close();
searcher.close();
dir.close();
}
if(snapDir[0].exists()) {
fail("The first backup should have been cleaned up because " + ReplicationHandler.NUMBER_BACKUPS_TO_KEEP + " was set to 1");
}
if(backupThread.fail != null) {
fail(backupThread.fail);
for(int i=0 ; i< snapDir.length ; i++) {
AbstractSolrTestCase.recurseDelete(snapDir[i]); // clean up the snap dir
}
File[] files = dataDir.listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
if(name.startsWith("snapshot")) {
return true;
}
return false;
}
});
assertEquals(1, files.length);
File snapDir = files[0];
Directory dir = new SimpleFSDirectory(snapDir.getAbsoluteFile());
IndexReader reader = IndexReader.open(dir);
IndexSearcher searcher = new IndexSearcher(reader);
TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
assertEquals(nDocs, hits.totalHits);
searcher.close();
reader.close();
dir.close();
AbstractSolrTestCase.recurseDelete(snapDir); // clean up the snap dir
}
/* character copy of file using UTF-8 */