LUCENE-1311: add IndexReader.listCommits & IndexReader.open methods to open a specific commit

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@671218 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -76,6 +76,13 @@ API Changes
     returns when the reader is opened on the same commit.  (Jason
     Rutherglen via Mike McCandless)
 
+11. LUCENE-1311: Added IndexReader.listCommits(Directory) static
+    method to list all commits in a Directory, plus IndexReader.open
+    methods that accept an IndexCommit and open the index as of that
+    commit.  These methods are only useful if you implement a custom
+    DeletionPolicy that keeps more than the last commit around.
+    (Jason Rutherglen via Mike McCandless)
+
 Bug fixes
     
  1. LUCENE-1134: Fixed BooleanQuery.rewrite to only optimize a single 

src/java/org/apache/lucene/index/DirectoryIndexReader.java

@@ -18,6 +18,7 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.io.FileNotFoundException;
 
 import java.util.HashSet;
 import java.util.Collection;
@@ -77,8 +78,12 @@ abstract class DirectoryIndexReader extends IndexReader {
   }
   
   static DirectoryIndexReader open(final Directory directory, final boolean closeDirectory, final IndexDeletionPolicy deletionPolicy) throws CorruptIndexException, IOException {
+    return open(directory, closeDirectory, deletionPolicy, null);
+  }
 
-    return (DirectoryIndexReader) new SegmentInfos.FindSegmentsFile(directory) {
+  static DirectoryIndexReader open(final Directory directory, final boolean closeDirectory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit) throws CorruptIndexException, IOException {
+
+    SegmentInfos.FindSegmentsFile finder = new SegmentInfos.FindSegmentsFile(directory) {
 
       protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
 
@@ -95,7 +100,17 @@ abstract class DirectoryIndexReader extends IndexReader {
         reader.setDeletionPolicy(deletionPolicy);
         return reader;
       }
-    }.run();
+    };
+
+    if (commit == null)
+      return (DirectoryIndexReader) finder.run();
+    else {
+      if (directory != commit.getDirectory())
+        throw new IOException("the specified commit does not match the specified Directory");
+      // This can & will directly throw IOException if the
+      // specified commit point has been deleted:
+      return (DirectoryIndexReader) finder.doBody(commit.getSegmentsFileName());
+    }
   }
 
   public final synchronized IndexReader reopen() throws CorruptIndexException, IOException {
@@ -192,7 +207,7 @@ abstract class DirectoryIndexReader extends IndexReader {
    * @throws IOException if there is a low-level IO error
    */
   protected void doCommit() throws IOException {
-    if(hasChanges){
+    if (hasChanges) {
       if (segmentInfos != null) {
 
         // Default deleter (for backwards compatibility) is
@@ -387,4 +402,51 @@ abstract class DirectoryIndexReader extends IndexReader {
   public IndexCommit getIndexCommit() throws IOException {
     return new ReaderCommit(segmentInfos, directory);
   }
+
+  /** @see IndexReader#listCommits */
+  public static Collection listCommits(Directory dir) throws IOException {
+
+    final String[] files = dir.list();
+    if (files == null)
+      throw new IOException("cannot read directory " + dir + ": list() returned null");
+
+    Collection commits = new ArrayList();
+
+    SegmentInfos latest = new SegmentInfos();
+    latest.read(dir);
+    final long currentGen = latest.getGeneration();
+
+    commits.add(new ReaderCommit(latest, dir));
+    
+    for(int i=0;i<files.length;i++) {
+
+      final String fileName = files[i];
+
+      if (fileName.startsWith(IndexFileNames.SEGMENTS) &&
+          !fileName.equals(IndexFileNames.SEGMENTS_GEN) &&
+          SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) {
+
+        SegmentInfos sis = new SegmentInfos();
+        try {
+          // IOException allowed to throw there, in case
+          // segments_N is corrupt
+          sis.read(dir, fileName);
+        } catch (FileNotFoundException fnfe) {
+          // LUCENE-948: on NFS (and maybe others), if
+          // you have writers switching back and forth
+          // between machines, it's very likely that the
+          // dir listing will be stale and will claim a
+          // file segments_X exists when in fact it
+          // doesn't.  So, we catch this and handle it
+          // as if the file does not exist
+          sis = null;
+        }
+
+        if (sis != null)
+          commits.add(new ReaderCommit(sis, dir));
+      }
+    }
+
+    return commits;
+  }
 }

src/java/org/apache/lucene/index/IndexCommit.java

@@ -75,8 +75,7 @@ public abstract class IndexCommit implements IndexCommitPoint {
   }
 
   /**
-   * Two IndexCommits are equal if both their Directory and
-   * segmentsFileName are equal.
+   * Two IndexCommits are equal if both their Directory and versions are equal.
    */
   public boolean equals(Object other) {
     if (other instanceof IndexCommit) {

src/java/org/apache/lucene/index/IndexReader.java

@@ -170,7 +170,7 @@ public abstract class IndexReader {
    * @throws IOException if there is a low-level IO error
    * @param path the path to the index directory */
   public static IndexReader open(String path) throws CorruptIndexException, IOException {
-    return open(FSDirectory.getDirectory(path), true, null);
+    return open(FSDirectory.getDirectory(path), true, null, null);
   }
 
   /** Returns an IndexReader reading the index in an FSDirectory in the named
@@ -180,7 +180,7 @@ public abstract class IndexReader {
    * @throws IOException if there is a low-level IO error
    */
   public static IndexReader open(File path) throws CorruptIndexException, IOException {
-    return open(FSDirectory.getDirectory(path), true, null);
+    return open(FSDirectory.getDirectory(path), true, null, null);
   }
 
   /** Returns an IndexReader reading the index in the given Directory.
@@ -189,7 +189,17 @@ public abstract class IndexReader {
    * @throws IOException if there is a low-level IO error
    */
   public static IndexReader open(final Directory directory) throws CorruptIndexException, IOException {
-    return open(directory, false, null);
+    return open(directory, false, null, null);
+  }
+
+  /** Expert: returns an IndexReader reading the index in the given
+   * {@link IndexCommit}.
+   * @param commit the commit point to open
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
+  public static IndexReader open(final IndexCommit commit) throws CorruptIndexException, IOException {
+    return open(commit.getDirectory(), false, null, commit);
   }
 
   /** Expert: returns an IndexReader reading the index in the given
@@ -202,11 +212,26 @@ public abstract class IndexReader {
    * @throws IOException if there is a low-level IO error
    */
   public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy) throws CorruptIndexException, IOException {
-    return open(directory, false, deletionPolicy);
+    return open(directory, false, deletionPolicy, null);
   }
 
-  private static IndexReader open(final Directory directory, final boolean closeDirectory, final IndexDeletionPolicy deletionPolicy) throws CorruptIndexException, IOException {
-    return DirectoryIndexReader.open(directory, closeDirectory, deletionPolicy);
+  /** Expert: returns an IndexReader reading the index in the given
+   * Directory, using a specific commit and with a custom {@link IndexDeletionPolicy}.
+   * @param commit the specific {@link IndexCommit} to open;
+   * see {@link IndexReader#listCommits} to list all commits
+   * in a directory
+   * @param deletionPolicy a custom deletion policy (only used
+   *  if you use this reader to perform deletes or to set
+   *  norms); see {@link IndexWriter} for details.
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
+  public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy) throws CorruptIndexException, IOException {
+    return open(commit.getDirectory(), false, deletionPolicy, commit);
+  }
+
+  private static IndexReader open(final Directory directory, final boolean closeDirectory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit) throws CorruptIndexException, IOException {
+    return DirectoryIndexReader.open(directory, closeDirectory, deletionPolicy, commit);
   }
 
   /**
@@ -975,4 +1000,20 @@ public abstract class IndexReader {
       }
     }
   }
+
+  /** Returns all commit points that exist in the Directory.
+   *  Normally, because the default is {@link
+   *  KeepOnlyLastCommitDeletionPolicy}, there would be only
+   *  one commit point.  But if you're using a custom {@link
+   *  DeletionPolicy} then there could be many commits.
+   *  Once you have a given commit, you can open a reader on
+   *  it by calling {@link IndexReader#open(Directory,
+   *  IndexCommit)}.  There must be at least one commit in
+   *  the Directory, else this method throws {@link
+   *  java.io.IOException}.  Note that if a commit is in
+   *  progress while this method is running, that commit
+   *  may or may not be returned array.  */
+  public static Collection listCommits(Directory dir) throws IOException {
+    return DirectoryIndexReader.listCommits(dir);
+  }
 }

src/test/org/apache/lucene/index/TestDeletionPolicy.java

@@ -22,6 +22,7 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
+import java.util.Collection;
 
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.lucene.document.Document;
@@ -258,6 +259,7 @@ public class TestDeletionPolicy extends LuceneTestCase
       boolean autoCommit = pass < 2;
       boolean useCompoundFile = (pass % 2) > 0;
 
+      // Never deletes a commit
       KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy();
 
       Directory dir = new RAMDirectory();
@@ -267,6 +269,8 @@ public class TestDeletionPolicy extends LuceneTestCase
       writer.setUseCompoundFile(useCompoundFile);
       for(int i=0;i<107;i++) {
         addDoc(writer);
+        if (autoCommit && i%10 == 0)
+          writer.commit();
       }
       writer.close();
 
@@ -281,6 +285,24 @@ public class TestDeletionPolicy extends LuceneTestCase
         // be exactly 2 commits (one per close above):
         assertEquals(2, policy.numOnCommit);
 
+      // Test listCommits
+      Collection commits = IndexReader.listCommits(dir);
+      if (!autoCommit)
+        // 1 from opening writer + 2 from closing writer
+        assertEquals(3, commits.size());
+      else
+        // 1 from opening writer + 2 from closing writer +
+        // 11 from calling writer.commit() explicitly above
+        assertEquals(14, commits.size());
+
+      Iterator it = commits.iterator();
+      // Make sure we can open a reader on each commit:
+      while(it.hasNext()) {
+        IndexCommit commit = (IndexCommit) it.next();
+        IndexReader r = IndexReader.open(commit, null);
+        r.close();
+      }
+
       // Simplistic check: just verify all segments_N's still
       // exist, and, I can open a reader on each:
       dir.deleteFile(IndexFileNames.SEGMENTS_GEN);