LUCENE-1044: add doSync option to FSDirectory.getDirectory, defaulting to true, to sync() each file descriptor before close()

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@593799 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -168,8 +168,15 @@ New features
     obtained or released, throwing an exception if an illegal lock
     obtain occurred.  (Patrick Kimber via Mike McCandless)
 
-  6. LUCENE-1015: Added FieldCache extension (ExtendedFieldCache) to support doubles and longs.  
-  Added support into SortField for sorting on doubles and longs as well.  (Grant Ingersoll)
+ 6. LUCENE-1015: Added FieldCache extension (ExtendedFieldCache) to support doubles and longs.  
+    Added support into SortField for sorting on doubles and longs as well.  (Grant Ingersoll)
+
+ 7. LUCENE-1044: Added optional doSync boolean to
+    FSDirectory.getDirectory(...).  If true (the default) then we will
+    always sync() a file before closing it, which improves the
+    likelihood that the index will remain consistent when the OS or
+    machine crashes, or power to the machine is cut.  (Venkat Rangan
+    via Mike McCandless)
 
 Optimizations
 

contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java

@@ -126,7 +126,8 @@ public class PerfRunData {
         FileUtils.fullyDelete(indexDir);
       }
       indexDir.mkdirs();
-      directory = FSDirectory.getDirectory(indexDir);
+      final boolean doSync = config.get("fsdirectory.dosync", true);
+      directory = FSDirectory.getDirectory(indexDir, null, doSync);
     } else {
       directory = new RAMDirectory();
     }

contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html

@@ -478,6 +478,12 @@ Some of the currently defined properties are:
     This tells which directory to use for the performance test.
     </li>
 
+    <li>
+    <font color="#FF0066">fsdirectory.dosync</font> - true/false
+    If "true", the FSDirectory will call sync() before close() on each
+    file.
+    </li>
+
     <li>
     <b>Index work parameters</b>:
     Multi int/boolean values would be iterated with calls to NewRound.
@@ -543,6 +549,7 @@ Here is a list of currently defined properties:
     </li><li>merge.factor
     </li><li>max.buffered
     </li><li>directory
+    </li><li>fsdirectory.dosync
     </li><li>ram.flush.mb
     </li><li>autocommit
     </li></ul>
@@ -676,4 +683,4 @@ the latter, <code>elapsedSec</code> would bring more insight.
 </DIV>
 <DIV>&nbsp;</DIV>
 </BODY>
-</HTML>
+</HTML>

src/java/org/apache/lucene/store/FSDirectory.java

@@ -62,6 +62,14 @@ public class FSDirectory extends Directory {
 
   private static boolean disableLocks = false;
 
+  private static boolean DEFAULT_DO_SYNC = true;
+
+  // True if we should call sync() before closing a file.
+  // This improves chances that index will still be
+  // consistent if the machine or OS abruptly crashes.  See
+  // LUCENE-1044.
+  private boolean doSync = DEFAULT_DO_SYNC;
+
   // TODO: should this move up to the Directory base class?  Also: should we
   // make a per-instance (in addition to the static "default") version?
 
@@ -136,17 +144,34 @@ public class FSDirectory extends Directory {
    * @return the FSDirectory for the named file.  */
   public static FSDirectory getDirectory(String path)
       throws IOException {
-    return getDirectory(new File(path), null);
+    return getDirectory(new File(path), null, DEFAULT_DO_SYNC);
   }
 
   /** Returns the directory instance for the named location.
    * @param path the path to the directory.
    * @param lockFactory instance of {@link LockFactory} providing the
-   *        locking implementation.
+   *        locking implementation.  If null, the default
+   *        {@link SimpleFSLockFactory} is used.
    * @return the FSDirectory for the named file.  */
   public static FSDirectory getDirectory(String path, LockFactory lockFactory)
       throws IOException {
-    return getDirectory(new File(path), lockFactory);
+    return getDirectory(new File(path), lockFactory, DEFAULT_DO_SYNC);
+  }
+
+  /** Returns the directory instance for the named location.
+   * @param path the path to the directory.
+   * @param lockFactory instance of {@link LockFactory} providing the
+   *        locking implementation.  If null, the default
+   *        {@link SimpleFSLockFactory} is used.
+   * @param doSync if true (the default), sync() is called
+   *        on all file descriptors before close().  This
+   *        improves the likelihood that the index will
+   *        remain consistent when the OS or machine crashes
+   *        or the power cord is pulled.
+   * @return the FSDirectory for the named file.  */
+  public static FSDirectory getDirectory(String path, LockFactory lockFactory, boolean doSync)
+      throws IOException {
+    return getDirectory(new File(path), lockFactory, doSync);
   }
 
   /** Returns the directory instance for the named location.
@@ -154,15 +179,32 @@ public class FSDirectory extends Directory {
    * @return the FSDirectory for the named file.  */
   public static FSDirectory getDirectory(File file)
     throws IOException {
-    return getDirectory(file, null);
+    return getDirectory(file, null, DEFAULT_DO_SYNC);
   }
 
   /** Returns the directory instance for the named location.
    * @param file the path to the directory.
    * @param lockFactory instance of {@link LockFactory} providing the
-   *        locking implementation.
+   *        locking implementation.  If null, the default
+   *        {@link SimpleFSLockFactory} is used.
    * @return the FSDirectory for the named file.  */
   public static FSDirectory getDirectory(File file, LockFactory lockFactory)
+      throws IOException {
+    return getDirectory(file, lockFactory, DEFAULT_DO_SYNC);
+  }
+
+  /** Returns the directory instance for the named location.
+   * @param file the path to the directory.
+   * @param lockFactory instance of {@link LockFactory} providing the
+   *        locking implementation.  If null, the default
+   *        {@link SimpleFSLockFactory} is used.
+   * @param doSync if true (the default), sync() is called
+   *        on all file descriptors before close().  This
+   *        improves the likelihood that the index will
+   *        remain consistent when the OS or machine crashes
+   *        or the power cord is pulled.
+   * @return the FSDirectory for the named file.  */
+  public static FSDirectory getDirectory(File file, LockFactory lockFactory, boolean doSync)
     throws IOException
   {
     file = new File(file.getCanonicalPath());
@@ -183,7 +225,7 @@ public class FSDirectory extends Directory {
         } catch (Exception e) {
           throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e);
         }
-        dir.init(file, lockFactory);
+        dir.init(file, lockFactory, doSync);
         DIRECTORIES.put(file, dir);
       } else {
         // Catch the case where a Directory is pulled from the cache, but has a
@@ -254,7 +296,7 @@ public class FSDirectory extends Directory {
 
   protected FSDirectory() {};                     // permit subclassing
 
-  private void init(File path, LockFactory lockFactory) throws IOException {
+  private void init(File path, LockFactory lockFactory, boolean doSync) throws IOException {
 
     // Set up lockFactory with cascaded defaults: if an instance was passed in,
     // use that; else if locks are disabled, use NoLockFactory; else if the
@@ -262,6 +304,7 @@ public class FSDirectory extends Directory {
     // instantiate that; else, use SimpleFSLockFactory:
 
     directory = path;
+    this.doSync = doSync;
 
     boolean doClearLockID = false;
 
@@ -432,7 +475,7 @@ public class FSDirectory extends Directory {
     if (file.exists() && !file.delete())          // delete existing, if any
       throw new IOException("Cannot overwrite: " + file);
 
-    return new FSIndexOutput(file);
+    return new FSIndexOutput(file, doSync);
   }
 
   // Inherit javadoc
@@ -588,10 +631,13 @@ public class FSDirectory extends Directory {
     // remember if the file is open, so that we don't try to close it
     // more than once
     private boolean isOpen;
-  
-    public FSIndexOutput(File path) throws IOException {
+    private boolean doSync;
+
+    public FSIndexOutput(File path, boolean doSync) throws IOException {
       file = new RandomAccessFile(path, "rw");
+
       isOpen = true;
+      this.doSync = doSync;
     }
   
     /** output methods: */
@@ -601,9 +647,14 @@ public class FSDirectory extends Directory {
     public void close() throws IOException {
       // only close the file if it has not been closed yet
       if (isOpen) {
-        super.close();
-        file.close();
-        isOpen = false;
+        try {
+          super.close();
+          if (doSync)
+            file.getFD().sync();
+        } finally {
+          file.close();
+          isOpen = false;
+        }
       }
     }
   

src/test/org/apache/lucene/index/TestAtomicUpdate.java

@@ -177,7 +177,7 @@ public class TestAtomicUpdate extends LuceneTestCase {
     // Second in an FSDirectory:
     String tempDir = System.getProperty("java.io.tmpdir");
     File dirPath = new File(tempDir, "lucene.test.atomic");
-    directory = FSDirectory.getDirectory(dirPath);
+    directory = FSDirectory.getDirectory(dirPath, null, false);
     runTest(directory);
     directory.close();
     _TestUtil.rmDir(dirPath);

src/test/org/apache/lucene/index/TestBackwardsCompatibility.java

@@ -152,7 +152,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase
 
     dirName = fullDir(dirName);
 
-    Directory dir = FSDirectory.getDirectory(dirName);
+    Directory dir = FSDirectory.getDirectory(dirName, null, false);
     IndexSearcher searcher = new IndexSearcher(dir);
     
     Hits hits = searcher.search(new TermQuery(new Term("content", "aaa")));
@@ -172,7 +172,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase
 
     dirName = fullDir(dirName);
 
-    Directory dir = FSDirectory.getDirectory(dirName);
+    Directory dir = FSDirectory.getDirectory(dirName, null, false);
 
     // open writer
     IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
@@ -232,7 +232,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase
 
     dirName = fullDir(dirName);
 
-    Directory dir = FSDirectory.getDirectory(dirName);
+    Directory dir = FSDirectory.getDirectory(dirName, null, false);
 
     // make sure searching sees right # hits
     IndexSearcher searcher = new IndexSearcher(dir);
@@ -280,7 +280,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase
 
     dirName = fullDir(dirName);
 
-    Directory dir = FSDirectory.getDirectory(dirName);
+    Directory dir = FSDirectory.getDirectory(dirName, null, false);
     IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
     writer.setUseCompoundFile(doCFS);
     
@@ -311,7 +311,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase
       rmDir(outputDir);
 
       try {
-        Directory dir = FSDirectory.getDirectory(fullDir(outputDir));
+        Directory dir = FSDirectory.getDirectory(fullDir(outputDir), null, false);
 
         boolean autoCommit = 0 == pass;
  

src/test/org/apache/lucene/index/TestCompoundFile.java

@@ -62,7 +62,7 @@ public class TestCompoundFile extends LuceneTestCase
        super.setUp();
        File file = new File(System.getProperty("tempDir"), "testIndex");
        _TestUtil.rmDir(file);
-       dir = FSDirectory.getDirectory(file);
+       dir = FSDirectory.getDirectory(file, null, false);
     }
 
 

src/test/org/apache/lucene/index/TestIndexModifier.java

@@ -147,7 +147,7 @@ public class TestIndexModifier extends LuceneTestCase {
     if (tempDir == null)
       throw new IOException("java.io.tmpdir undefined, cannot run test");
     File indexDir = new File(tempDir, "lucenetestindex");
-    Directory rd = FSDirectory.getDirectory(indexDir);
+    Directory rd = FSDirectory.getDirectory(indexDir, null, false);
     IndexThread.id = 0;
     IndexThread.idStack.clear();
     IndexModifier index = new IndexModifier(rd, new StandardAnalyzer(), create);

src/test/org/apache/lucene/index/TestNorms.java

@@ -81,7 +81,7 @@ public class TestNorms extends LuceneTestCase {
     
     // test with a single index: index1
     File indexDir1 = new File(tempDir, "lucenetestindex1");
-    Directory dir1 = FSDirectory.getDirectory(indexDir1);
+    Directory dir1 = FSDirectory.getDirectory(indexDir1, null, false);
 
     norms = new ArrayList();
     modifiedNorms = new ArrayList();
@@ -99,14 +99,14 @@ public class TestNorms extends LuceneTestCase {
     numDocNorms = 0;
     
     File indexDir2 = new File(tempDir, "lucenetestindex2");
-    Directory dir2 = FSDirectory.getDirectory(indexDir2);
+    Directory dir2 = FSDirectory.getDirectory(indexDir2, null, false);
 
     createIndex(dir2);
     doTestNorms(dir2);
 
     // add index1 and index2 to a third index: index3
     File indexDir3 = new File(tempDir, "lucenetestindex3");
-    Directory dir3 = FSDirectory.getDirectory(indexDir3);
+    Directory dir3 = FSDirectory.getDirectory(indexDir3, null, false);
 
     createIndex(dir3);
     IndexWriter iw = new IndexWriter(dir3,anlzr,false);

src/test/org/apache/lucene/index/TestStressIndexing.java

@@ -178,7 +178,7 @@ public class TestStressIndexing extends LuceneTestCase {
     // FSDir
     String tempDir = System.getProperty("java.io.tmpdir");
     File dirPath = new File(tempDir, "lucene.test.stress");
-    directory = FSDirectory.getDirectory(dirPath);
+    directory = FSDirectory.getDirectory(dirPath, null, false);
     runStressTest(directory, true, null);
     directory.close();
 
@@ -188,7 +188,7 @@ public class TestStressIndexing extends LuceneTestCase {
     directory.close();
 
     // With ConcurrentMergeScheduler, in FSDir
-    directory = FSDirectory.getDirectory(dirPath);
+    directory = FSDirectory.getDirectory(dirPath, null, false);
     runStressTest(directory, true, new ConcurrentMergeScheduler());
     directory.close();
 
@@ -198,7 +198,7 @@ public class TestStressIndexing extends LuceneTestCase {
     directory.close();
 
     // With ConcurrentMergeScheduler and autoCommit=false, in FSDir
-    directory = FSDirectory.getDirectory(dirPath);
+    directory = FSDirectory.getDirectory(dirPath, null, false);
     runStressTest(directory, false, new ConcurrentMergeScheduler());
     directory.close();
 

src/test/org/apache/lucene/index/TestThreadedOptimize.java

@@ -149,7 +149,7 @@ public class TestThreadedOptimize extends LuceneTestCase {
       throw new IOException("tempDir undefined, cannot run test");
 
     String dirName = tempDir + "/luceneTestThreadedOptimize";
-    directory = FSDirectory.getDirectory(dirName);
+    directory = FSDirectory.getDirectory(dirName, null, false);
     runTest(directory, false, null);
     runTest(directory, true, null);
     runTest(directory, false, new ConcurrentMergeScheduler());

src/test/org/apache/lucene/store/TestLockFactory.java

@@ -320,7 +320,7 @@ public class TestLockFactory extends LuceneTestCase {
     }
 
     public void _testStressLocks(LockFactory lockFactory, String indexDirName) throws IOException {
-        FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory);
+        FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory, false);
 
         // First create a 1 doc index:
         IndexWriter w = new IndexWriter(fs1, new WhitespaceAnalyzer(), true);