diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 695126d5fd2..0cceda931e6 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -396,6 +396,10 @@ New Features * SOLR-1023: StatsComponent now supports date fields and string fields. (Chris Male, Mark Holland, Gunnlaugur Thor Briem, Ryan McKinley) +* SOLR-2578: ReplicationHandler's backup command now supports a 'numberToKeep' + param that can be used to delete all but the most recent N backups. + (James Dyer via hossman) + Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index 90a00be51e3..1ee4a11d65e 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -126,7 +126,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw } // This command does not give the current index version of the master // It gives the current 'replicateable' index version - if (command.equals(CMD_INDEX_VERSION)) { + if (command.equals(CMD_INDEX_VERSION)) { IndexCommit commitPoint = indexCommitPoint; // make a copy so it won't change if (commitPoint != null && replicationEnabled.get()) { // @@ -202,10 +202,10 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw } else if (CMD_ENABLE_REPL.equalsIgnoreCase(command)) { replicationEnabled.set(true); rsp.add(STATUS, OK_STATUS); - } else if (CMD_DISABLE_REPL.equalsIgnoreCase(command)) { - replicationEnabled.set(false); - rsp.add(STATUS, OK_STATUS); - } + } else if (CMD_DISABLE_REPL.equalsIgnoreCase(command)) { + replicationEnabled.set(false); + rsp.add(STATUS, OK_STATUS); + } } private List> getCommits() { @@ -296,16 +296,17 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw private void doSnapShoot(SolrParams params, SolrQueryResponse rsp, SolrQueryRequest req) { try { + int numberToKeep = params.getInt(NUMBER_BACKUPS_TO_KEEP, Integer.MAX_VALUE); IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy(); IndexCommit indexCommit = delPolicy.getLatestCommit(); - + if(indexCommit == null) { indexCommit = req.getSearcher().getIndexReader().getIndexCommit(); } - + // small race here before the commit point is saved - new SnapShooter(core, params.get("location")).createSnapAsync(indexCommit, this); - + new SnapShooter(core, params.get("location")).createSnapAsync(indexCommit, numberToKeep, this); + } catch (Exception e) { LOG.warn("Exception during creating a snapshot", e); rsp.add("exception", e); @@ -354,7 +355,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw rsp.add("status", "unable to get file names for given indexversion"); rsp.add("exception", e); LOG.warn("Unable to get file names for indexCommit version: " - + version, e); + + version, e); } rsp.add(CMD_GET_FILE_LIST, result); if (confFileNameAlias.size() < 1) @@ -1146,4 +1147,6 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw public static final String OK_STATUS = "OK"; public static final String NEXT_EXECUTION_AT = "nextExecutionAt"; + + public static final String NUMBER_BACKUPS_TO_KEEP = "numberToKeep"; } diff --git a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java index d0afaa2ece7..84cf74ae067 100644 --- a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java +++ b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java @@ -22,9 +22,14 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Date; +import java.util.List; import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import org.apache.lucene.index.IndexCommit; @@ -59,19 +64,23 @@ public class SnapShooter { } lockFactory = new SimpleFSLockFactory(snapDir); } - + void createSnapAsync(final IndexCommit indexCommit, final ReplicationHandler replicationHandler) { + createSnapAsync(indexCommit, Integer.MAX_VALUE, replicationHandler); + } + + void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, final ReplicationHandler replicationHandler) { replicationHandler.core.getDeletionPolicy().saveCommitPoint(indexCommit.getVersion()); new Thread() { @Override public void run() { - createSnapshot(indexCommit, replicationHandler); + createSnapshot(indexCommit, numberToKeep, replicationHandler); } }.start(); } - void createSnapshot(final IndexCommit indexCommit, ReplicationHandler replicationHandler) { + void createSnapshot(final IndexCommit indexCommit, int numberToKeep, ReplicationHandler replicationHandler) { NamedList details = new NamedList(); details.add("startTime", new Date().toString()); @@ -79,6 +88,9 @@ public class SnapShooter { String directoryName = null; Lock lock = null; try { + if(numberToKeep dirs = new ArrayList(); + for(File f : files) { + OldBackupDirectory obd = new OldBackupDirectory(f); + if(obd.dir != null) { + dirs.add(obd); + } + } + Collections.sort(dirs); + int i=1; + for(OldBackupDirectory dir : dirs) { + if( i > numberToKeep-1 ) { + SnapPuller.delTree(dir.dir); + } + } + } + private class OldBackupDirectory implements Comparable{ + File dir; + Date timestamp; + final Pattern dirNamePattern = Pattern.compile("^snapshot[.](.*)$"); + + OldBackupDirectory(File dir) { + if(dir.isDirectory()) { + Matcher m = dirNamePattern.matcher(dir.getName()); + if(m.find()) { + try { + this.dir = dir; + this.timestamp = new SimpleDateFormat(DATE_FMT).parse(m.group(1)); + } catch(Exception e) { + this.dir = null; + this.timestamp = null; + } + } + } + } + public int compareTo(OldBackupDirectory that) { + return that.timestamp.compareTo(this.timestamp); + } + } public static final String SNAP_DIR = "snapDir"; public static final String DATE_FMT = "yyyyMMddHHmmss"; @@ -135,7 +187,7 @@ public class SnapShooter { } public void copyFile(File source, File destination, boolean preserveFileDate) - throws IOException { + throws IOException { // check source exists if (!source.exists()) { String message = "File " + source + " does not exist"; @@ -185,7 +237,7 @@ public class SnapShooter { if (source.length() != destination.length()) { String message = "Failed to copy full contents from " + source + " to " - + destination; + + destination; throw new IOException(message); } diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java index 77254981085..11343dcac76 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java @@ -25,7 +25,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.Writer; import java.net.URL; - +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import org.apache.commons.io.IOUtils; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; @@ -753,7 +755,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { volatile String fail = null; @Override public void run() { - String masterUrl = "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_BACKUP; + String masterUrl = "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_BACKUP + "&" + ReplicationHandler.NUMBER_BACKUPS_TO_KEEP + "=1"; URL url; InputStream stream = null; try { @@ -768,14 +770,18 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { }; }; - BackupThread backupThread = new BackupThread(); - backupThread.start(); - File dataDir = new File(master.getDataDir()); class CheckStatus extends Thread { volatile String fail = null; volatile String response = null; volatile boolean success = false; + volatile String backupTimestamp = null; + final String lastBackupTimestamp; + final Pattern p = Pattern.compile("(.*?)"); + + CheckStatus(String lastBackupTimestamp) { + this.lastBackupTimestamp = lastBackupTimestamp; + } @Override public void run() { String masterUrl = "http://localhost:" + masterJetty.getLocalPort() + "/solr/replication?command=" + ReplicationHandler.CMD_DETAILS; @@ -786,7 +792,14 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { stream = url.openStream(); response = IOUtils.toString(stream, "UTF-8"); if(response.contains("success")) { - success = true; + Matcher m = p.matcher(response); + if(!m.find()) { + fail("could not find the completed timestamp in response."); + } + backupTimestamp = m.group(1); + if(!backupTimestamp.equals(lastBackupTimestamp)) { + success = true; + } } stream.close(); } catch (Exception e) { @@ -797,48 +810,67 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { }; }; - int waitCnt = 0; - CheckStatus checkStatus = new CheckStatus(); - while(true) { - checkStatus.run(); - if(checkStatus.fail != null) { - fail(checkStatus.fail); + + File[] snapDir = new File[2]; + String firstBackupTimestamp = null; + for(int i=0 ; i<2 ; i++) { + BackupThread backupThread = new BackupThread(); + backupThread.start(); + + File dataDir = new File(master.getDataDir()); + + int waitCnt = 0; + CheckStatus checkStatus = new CheckStatus(firstBackupTimestamp); + while(true) { + checkStatus.run(); + if(checkStatus.fail != null) { + fail(checkStatus.fail); + } + if(checkStatus.success) { + if(i==0) { + firstBackupTimestamp = checkStatus.backupTimestamp; + Thread.sleep(1000); //ensure the next backup will have a different timestamp. + } + break; + } + Thread.sleep(200); + if(waitCnt == 10) { + fail("Backup success not detected:" + checkStatus.response); + } + waitCnt++; } - if(checkStatus.success) { - break; + + if(backupThread.fail != null) { + fail(backupThread.fail); } - Thread.sleep(200); - if(waitCnt == 10) { - fail("Backup success not detected:" + checkStatus.response); - } - waitCnt++; + + File[] files = dataDir.listFiles(new FilenameFilter() { + + public boolean accept(File dir, String name) { + if(name.startsWith("snapshot")) { + return true; + } + return false; + } + }); + assertEquals(1, files.length); + snapDir[i] = files[0]; + Directory dir = new SimpleFSDirectory(snapDir[i].getAbsoluteFile()); + IndexReader reader = IndexReader.open(dir); + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1); + assertEquals(nDocs, hits.totalHits); + reader.close(); + searcher.close(); + dir.close(); + } + if(snapDir[0].exists()) { + fail("The first backup should have been cleaned up because " + ReplicationHandler.NUMBER_BACKUPS_TO_KEEP + " was set to 1"); } - if(backupThread.fail != null) { - fail(backupThread.fail); + for(int i=0 ; i< snapDir.length ; i++) { + AbstractSolrTestCase.recurseDelete(snapDir[i]); // clean up the snap dir } - - File[] files = dataDir.listFiles(new FilenameFilter() { - - public boolean accept(File dir, String name) { - if(name.startsWith("snapshot")) { - return true; - } - return false; - } - }); - assertEquals(1, files.length); - File snapDir = files[0]; - Directory dir = new SimpleFSDirectory(snapDir.getAbsoluteFile()); - IndexReader reader = IndexReader.open(dir); - IndexSearcher searcher = new IndexSearcher(reader); - TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1); - - assertEquals(nDocs, hits.totalHits); - searcher.close(); - reader.close(); - dir.close(); - AbstractSolrTestCase.recurseDelete(snapDir); // clean up the snap dir } /* character copy of file using UTF-8 */