diff --git a/CHANGES.txt b/CHANGES.txt index 5df7248dd24..dc6395eb911 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -168,8 +168,15 @@ New features obtained or released, throwing an exception if an illegal lock obtain occurred. (Patrick Kimber via Mike McCandless) - 6. LUCENE-1015: Added FieldCache extension (ExtendedFieldCache) to support doubles and longs. - Added support into SortField for sorting on doubles and longs as well. (Grant Ingersoll) + 6. LUCENE-1015: Added FieldCache extension (ExtendedFieldCache) to support doubles and longs. + Added support into SortField for sorting on doubles and longs as well. (Grant Ingersoll) + + 7. LUCENE-1044: Added optional doSync boolean to + FSDirectory.getDirectory(...). If true (the default) then we will + always sync() a file before closing it, which improves the + likelihood that the index will remain consistent when the OS or + machine crashes, or power to the machine is cut. (Venkat Rangan + via Mike McCandless) Optimizations diff --git a/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java b/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java index 56205306aa5..7eed12a0976 100644 --- a/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java +++ b/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java @@ -126,7 +126,8 @@ public class PerfRunData { FileUtils.fullyDelete(indexDir); } indexDir.mkdirs(); - directory = FSDirectory.getDirectory(indexDir); + final boolean doSync = config.get("fsdirectory.dosync", true); + directory = FSDirectory.getDirectory(indexDir, null, doSync); } else { directory = new RAMDirectory(); } diff --git a/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html b/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html index 520c0f3ad01..53bb9710338 100644 --- a/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html +++ b/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html @@ -478,6 +478,12 @@ Some of the currently defined properties are: This tells which directory to use for the performance test. +
  • + fsdirectory.dosync - true/false + If "true", the FSDirectory will call sync() before close() on each + file. +
    • +
  • Index work parameters: Multi int/boolean values would be iterated with calls to NewRound. @@ -543,6 +549,7 @@ Here is a list of currently defined properties:
  • merge.factor
  • max.buffered
  • directory +
  • fsdirectory.dosync
  • ram.flush.mb
  • autocommit
    • @@ -676,4 +683,4 @@ the latter, elapsedSec would bring more insight.
     
    - \ No newline at end of file + diff --git a/src/java/org/apache/lucene/store/FSDirectory.java b/src/java/org/apache/lucene/store/FSDirectory.java index 51ac2a54ed3..d4352dea52a 100644 --- a/src/java/org/apache/lucene/store/FSDirectory.java +++ b/src/java/org/apache/lucene/store/FSDirectory.java @@ -62,6 +62,14 @@ public class FSDirectory extends Directory { private static boolean disableLocks = false; + private static boolean DEFAULT_DO_SYNC = true; + + // True if we should call sync() before closing a file. + // This improves chances that index will still be + // consistent if the machine or OS abruptly crashes. See + // LUCENE-1044. + private boolean doSync = DEFAULT_DO_SYNC; + // TODO: should this move up to the Directory base class? Also: should we // make a per-instance (in addition to the static "default") version? @@ -136,17 +144,34 @@ public class FSDirectory extends Directory { * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(String path) throws IOException { - return getDirectory(new File(path), null); + return getDirectory(new File(path), null, DEFAULT_DO_SYNC); } /** Returns the directory instance for the named location. * @param path the path to the directory. * @param lockFactory instance of {@link LockFactory} providing the - * locking implementation. + * locking implementation. If null, the default + * {@link SimpleFSLockFactory} is used. * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(String path, LockFactory lockFactory) throws IOException { - return getDirectory(new File(path), lockFactory); + return getDirectory(new File(path), lockFactory, DEFAULT_DO_SYNC); + } + + /** Returns the directory instance for the named location. + * @param path the path to the directory. + * @param lockFactory instance of {@link LockFactory} providing the + * locking implementation. If null, the default + * {@link SimpleFSLockFactory} is used. + * @param doSync if true (the default), sync() is called + * on all file descriptors before close(). This + * improves the likelihood that the index will + * remain consistent when the OS or machine crashes + * or the power cord is pulled. + * @return the FSDirectory for the named file. */ + public static FSDirectory getDirectory(String path, LockFactory lockFactory, boolean doSync) + throws IOException { + return getDirectory(new File(path), lockFactory, doSync); } /** Returns the directory instance for the named location. @@ -154,15 +179,32 @@ public class FSDirectory extends Directory { * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(File file) throws IOException { - return getDirectory(file, null); + return getDirectory(file, null, DEFAULT_DO_SYNC); } /** Returns the directory instance for the named location. * @param file the path to the directory. * @param lockFactory instance of {@link LockFactory} providing the - * locking implementation. + * locking implementation. If null, the default + * {@link SimpleFSLockFactory} is used. * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(File file, LockFactory lockFactory) + throws IOException { + return getDirectory(file, lockFactory, DEFAULT_DO_SYNC); + } + + /** Returns the directory instance for the named location. + * @param file the path to the directory. + * @param lockFactory instance of {@link LockFactory} providing the + * locking implementation. If null, the default + * {@link SimpleFSLockFactory} is used. + * @param doSync if true (the default), sync() is called + * on all file descriptors before close(). This + * improves the likelihood that the index will + * remain consistent when the OS or machine crashes + * or the power cord is pulled. + * @return the FSDirectory for the named file. */ + public static FSDirectory getDirectory(File file, LockFactory lockFactory, boolean doSync) throws IOException { file = new File(file.getCanonicalPath()); @@ -183,7 +225,7 @@ public class FSDirectory extends Directory { } catch (Exception e) { throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e); } - dir.init(file, lockFactory); + dir.init(file, lockFactory, doSync); DIRECTORIES.put(file, dir); } else { // Catch the case where a Directory is pulled from the cache, but has a @@ -254,7 +296,7 @@ public class FSDirectory extends Directory { protected FSDirectory() {}; // permit subclassing - private void init(File path, LockFactory lockFactory) throws IOException { + private void init(File path, LockFactory lockFactory, boolean doSync) throws IOException { // Set up lockFactory with cascaded defaults: if an instance was passed in, // use that; else if locks are disabled, use NoLockFactory; else if the @@ -262,6 +304,7 @@ public class FSDirectory extends Directory { // instantiate that; else, use SimpleFSLockFactory: directory = path; + this.doSync = doSync; boolean doClearLockID = false; @@ -432,7 +475,7 @@ public class FSDirectory extends Directory { if (file.exists() && !file.delete()) // delete existing, if any throw new IOException("Cannot overwrite: " + file); - return new FSIndexOutput(file); + return new FSIndexOutput(file, doSync); } // Inherit javadoc @@ -588,10 +631,13 @@ public class FSDirectory extends Directory { // remember if the file is open, so that we don't try to close it // more than once private boolean isOpen; - - public FSIndexOutput(File path) throws IOException { + private boolean doSync; + + public FSIndexOutput(File path, boolean doSync) throws IOException { file = new RandomAccessFile(path, "rw"); + isOpen = true; + this.doSync = doSync; } /** output methods: */ @@ -601,9 +647,14 @@ public class FSDirectory extends Directory { public void close() throws IOException { // only close the file if it has not been closed yet if (isOpen) { - super.close(); - file.close(); - isOpen = false; + try { + super.close(); + if (doSync) + file.getFD().sync(); + } finally { + file.close(); + isOpen = false; + } } } diff --git a/src/test/org/apache/lucene/index/TestAtomicUpdate.java b/src/test/org/apache/lucene/index/TestAtomicUpdate.java index f6a698cb32a..17c7e813c09 100644 --- a/src/test/org/apache/lucene/index/TestAtomicUpdate.java +++ b/src/test/org/apache/lucene/index/TestAtomicUpdate.java @@ -177,7 +177,7 @@ public class TestAtomicUpdate extends LuceneTestCase { // Second in an FSDirectory: String tempDir = System.getProperty("java.io.tmpdir"); File dirPath = new File(tempDir, "lucene.test.atomic"); - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.getDirectory(dirPath, null, false); runTest(directory); directory.close(); _TestUtil.rmDir(dirPath); diff --git a/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index 0e65f2729bb..d85a352cc86 100644 --- a/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -152,7 +152,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.getDirectory(dirName, null, false); IndexSearcher searcher = new IndexSearcher(dir); Hits hits = searcher.search(new TermQuery(new Term("content", "aaa"))); @@ -172,7 +172,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.getDirectory(dirName, null, false); // open writer IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false); @@ -232,7 +232,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.getDirectory(dirName, null, false); // make sure searching sees right # hits IndexSearcher searcher = new IndexSearcher(dir); @@ -280,7 +280,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName); + Directory dir = FSDirectory.getDirectory(dirName, null, false); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true); writer.setUseCompoundFile(doCFS); @@ -311,7 +311,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase rmDir(outputDir); try { - Directory dir = FSDirectory.getDirectory(fullDir(outputDir)); + Directory dir = FSDirectory.getDirectory(fullDir(outputDir), null, false); boolean autoCommit = 0 == pass; diff --git a/src/test/org/apache/lucene/index/TestCompoundFile.java b/src/test/org/apache/lucene/index/TestCompoundFile.java index de0b617c39c..26f96575e28 100644 --- a/src/test/org/apache/lucene/index/TestCompoundFile.java +++ b/src/test/org/apache/lucene/index/TestCompoundFile.java @@ -62,7 +62,7 @@ public class TestCompoundFile extends LuceneTestCase super.setUp(); File file = new File(System.getProperty("tempDir"), "testIndex"); _TestUtil.rmDir(file); - dir = FSDirectory.getDirectory(file); + dir = FSDirectory.getDirectory(file, null, false); } diff --git a/src/test/org/apache/lucene/index/TestIndexModifier.java b/src/test/org/apache/lucene/index/TestIndexModifier.java index cd89051f8b2..392c472418d 100644 --- a/src/test/org/apache/lucene/index/TestIndexModifier.java +++ b/src/test/org/apache/lucene/index/TestIndexModifier.java @@ -147,7 +147,7 @@ public class TestIndexModifier extends LuceneTestCase { if (tempDir == null) throw new IOException("java.io.tmpdir undefined, cannot run test"); File indexDir = new File(tempDir, "lucenetestindex"); - Directory rd = FSDirectory.getDirectory(indexDir); + Directory rd = FSDirectory.getDirectory(indexDir, null, false); IndexThread.id = 0; IndexThread.idStack.clear(); IndexModifier index = new IndexModifier(rd, new StandardAnalyzer(), create); diff --git a/src/test/org/apache/lucene/index/TestNorms.java b/src/test/org/apache/lucene/index/TestNorms.java index 1b662690c3b..37d1d7f1431 100755 --- a/src/test/org/apache/lucene/index/TestNorms.java +++ b/src/test/org/apache/lucene/index/TestNorms.java @@ -81,7 +81,7 @@ public class TestNorms extends LuceneTestCase { // test with a single index: index1 File indexDir1 = new File(tempDir, "lucenetestindex1"); - Directory dir1 = FSDirectory.getDirectory(indexDir1); + Directory dir1 = FSDirectory.getDirectory(indexDir1, null, false); norms = new ArrayList(); modifiedNorms = new ArrayList(); @@ -99,14 +99,14 @@ public class TestNorms extends LuceneTestCase { numDocNorms = 0; File indexDir2 = new File(tempDir, "lucenetestindex2"); - Directory dir2 = FSDirectory.getDirectory(indexDir2); + Directory dir2 = FSDirectory.getDirectory(indexDir2, null, false); createIndex(dir2); doTestNorms(dir2); // add index1 and index2 to a third index: index3 File indexDir3 = new File(tempDir, "lucenetestindex3"); - Directory dir3 = FSDirectory.getDirectory(indexDir3); + Directory dir3 = FSDirectory.getDirectory(indexDir3, null, false); createIndex(dir3); IndexWriter iw = new IndexWriter(dir3,anlzr,false); diff --git a/src/test/org/apache/lucene/index/TestStressIndexing.java b/src/test/org/apache/lucene/index/TestStressIndexing.java index 0f9868ecdf6..cfa8f73e78b 100644 --- a/src/test/org/apache/lucene/index/TestStressIndexing.java +++ b/src/test/org/apache/lucene/index/TestStressIndexing.java @@ -178,7 +178,7 @@ public class TestStressIndexing extends LuceneTestCase { // FSDir String tempDir = System.getProperty("java.io.tmpdir"); File dirPath = new File(tempDir, "lucene.test.stress"); - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.getDirectory(dirPath, null, false); runStressTest(directory, true, null); directory.close(); @@ -188,7 +188,7 @@ public class TestStressIndexing extends LuceneTestCase { directory.close(); // With ConcurrentMergeScheduler, in FSDir - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.getDirectory(dirPath, null, false); runStressTest(directory, true, new ConcurrentMergeScheduler()); directory.close(); @@ -198,7 +198,7 @@ public class TestStressIndexing extends LuceneTestCase { directory.close(); // With ConcurrentMergeScheduler and autoCommit=false, in FSDir - directory = FSDirectory.getDirectory(dirPath); + directory = FSDirectory.getDirectory(dirPath, null, false); runStressTest(directory, false, new ConcurrentMergeScheduler()); directory.close(); diff --git a/src/test/org/apache/lucene/index/TestThreadedOptimize.java b/src/test/org/apache/lucene/index/TestThreadedOptimize.java index 5ee7a76fead..e5fee5c8965 100644 --- a/src/test/org/apache/lucene/index/TestThreadedOptimize.java +++ b/src/test/org/apache/lucene/index/TestThreadedOptimize.java @@ -149,7 +149,7 @@ public class TestThreadedOptimize extends LuceneTestCase { throw new IOException("tempDir undefined, cannot run test"); String dirName = tempDir + "/luceneTestThreadedOptimize"; - directory = FSDirectory.getDirectory(dirName); + directory = FSDirectory.getDirectory(dirName, null, false); runTest(directory, false, null); runTest(directory, true, null); runTest(directory, false, new ConcurrentMergeScheduler()); diff --git a/src/test/org/apache/lucene/store/TestLockFactory.java b/src/test/org/apache/lucene/store/TestLockFactory.java index c7e7349e538..66b0136b67e 100755 --- a/src/test/org/apache/lucene/store/TestLockFactory.java +++ b/src/test/org/apache/lucene/store/TestLockFactory.java @@ -320,7 +320,7 @@ public class TestLockFactory extends LuceneTestCase { } public void _testStressLocks(LockFactory lockFactory, String indexDirName) throws IOException { - FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory); + FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory, false); // First create a 1 doc index: IndexWriter w = new IndexWriter(fs1, new WhitespaceAnalyzer(), true);