From b0d2b1c90eac6fe8e1a12e7d85308b6baabc3519 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Fri, 7 Dec 2007 17:42:33 +0000 Subject: [PATCH] LUCENE-1044: revert the doSync option to FSDirectory git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@602165 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 15 +--- .../lucene/benchmark/byTask/PerfRunData.java | 3 +- .../lucene/benchmark/byTask/package.html | 9 +-- .../org/apache/lucene/store/FSDirectory.java | 77 +++---------------- .../apache/lucene/index/TestAtomicUpdate.java | 2 +- .../index/TestBackwardsCompatibility.java | 10 +-- .../apache/lucene/index/TestCompoundFile.java | 2 +- .../lucene/index/TestIndexModifier.java | 2 +- .../org/apache/lucene/index/TestNorms.java | 6 +- .../lucene/index/TestStressIndexing.java | 6 +- .../lucene/index/TestThreadedOptimize.java | 2 +- .../apache/lucene/store/TestLockFactory.java | 2 +- 12 files changed, 33 insertions(+), 103 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index a301e3f1dee..ab670f81e45 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -218,30 +218,23 @@ New features support doubles and longs. Added support into SortField for sorting on doubles and longs as well. (Grant Ingersoll) - 7. LUCENE-1044: Added optional doSync boolean to - FSDirectory.getDirectory(...). If true (the default) then we will - always sync() a file before closing it, which improves the - likelihood that the index will remain consistent when the OS or - machine crashes, or power to the machine is cut. (Venkat Rangan - via Mike McCandless) - - 8. LUCENE-1020: Created basic index checking & repair tool + 7. LUCENE-1020: Created basic index checking & repair tool (o.a.l.index.CheckIndex). When run without -fix it does a detailed test of all segments in the index and reports summary information and any errors it hit. With -fix it will remove segments that had errors. (Mike McCandless) - 9. LUCENE-743: Add IndexReader.reopen() method that re-opens an + 8. LUCENE-743: Add IndexReader.reopen() method that re-opens an existing IndexReader by only loading those portions of an index that have changed since the reader was (re)opened. reopen() can be significantly faster than open(), depending on the amount of index changes. SegmentReader, MultiSegmentReader, MultiReader, and ParallelReader implement reopen(). (Michael Busch) -10. LUCENE-1040: CharArraySet useful for efficiently checking + 9. LUCENE-1040: CharArraySet useful for efficiently checking set membership of text specified by char[]. (yonik) -11. LUCENE-1073: Created SnapshotDeletionPolicy to facilitate taking a +10. LUCENE-1073: Created SnapshotDeletionPolicy to facilitate taking a live backup of an index without pausing indexing. (Mike McCandless) diff --git a/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java b/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java index c98f8a0455c..56205306aa5 100644 --- a/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java +++ b/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java @@ -126,8 +126,7 @@ public class PerfRunData { FileUtils.fullyDelete(indexDir); } indexDir.mkdirs(); - final boolean doSync = config.get("fsdirectory.dosync", false); - directory = FSDirectory.getDirectory(indexDir, null, doSync); + directory = FSDirectory.getDirectory(indexDir); } else { directory = new RAMDirectory(); } diff --git a/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html b/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html index 53bb9710338..520c0f3ad01 100644 --- a/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html +++ b/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/package.html @@ -478,12 +478,6 @@ Some of the currently defined properties are: This tells which directory to use for the performance test. -
  • - fsdirectory.dosync - true/false - If "true", the FSDirectory will call sync() before close() on each - file. -
    • -
  • Index work parameters: Multi int/boolean values would be iterated with calls to NewRound. @@ -549,7 +543,6 @@ Here is a list of currently defined properties:
  • merge.factor
  • max.buffered
  • directory -
  • fsdirectory.dosync
  • ram.flush.mb
  • autocommit
    • @@ -683,4 +676,4 @@ the latter, elapsedSec would bring more insight.
     
    - + \ No newline at end of file diff --git a/src/java/org/apache/lucene/store/FSDirectory.java b/src/java/org/apache/lucene/store/FSDirectory.java index 17a0d72622a..6b23aca3c7a 100644 --- a/src/java/org/apache/lucene/store/FSDirectory.java +++ b/src/java/org/apache/lucene/store/FSDirectory.java @@ -62,14 +62,6 @@ public class FSDirectory extends Directory { private static boolean disableLocks = false; - private static boolean DEFAULT_DO_SYNC = false; - - // True if we should call sync() before closing a file. - // This improves chances that index will still be - // consistent if the machine or OS abruptly crashes. See - // LUCENE-1044. - private boolean doSync = DEFAULT_DO_SYNC; - // TODO: should this move up to the Directory base class? Also: should we // make a per-instance (in addition to the static "default") version? @@ -144,34 +136,17 @@ public class FSDirectory extends Directory { * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(String path) throws IOException { - return getDirectory(new File(path), null, DEFAULT_DO_SYNC); + return getDirectory(new File(path), null); } /** Returns the directory instance for the named location. * @param path the path to the directory. * @param lockFactory instance of {@link LockFactory} providing the - * locking implementation. If null, the default - * {@link SimpleFSLockFactory} is used. + * locking implementation. * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(String path, LockFactory lockFactory) throws IOException { - return getDirectory(new File(path), lockFactory, DEFAULT_DO_SYNC); - } - - /** Returns the directory instance for the named location. - * @param path the path to the directory. - * @param lockFactory instance of {@link LockFactory} providing the - * locking implementation. If null, the default - * {@link SimpleFSLockFactory} is used. - * @param doSync if true (the default), sync() is called - * on all file descriptors before close(). This - * improves the likelihood that the index will - * remain consistent when the OS or machine crashes - * or the power cord is pulled. - * @return the FSDirectory for the named file. */ - public static FSDirectory getDirectory(String path, LockFactory lockFactory, boolean doSync) - throws IOException { - return getDirectory(new File(path), lockFactory, doSync); + return getDirectory(new File(path), lockFactory); } /** Returns the directory instance for the named location. @@ -179,32 +154,15 @@ public class FSDirectory extends Directory { * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(File file) throws IOException { - return getDirectory(file, null, DEFAULT_DO_SYNC); + return getDirectory(file, null); } /** Returns the directory instance for the named location. * @param file the path to the directory. * @param lockFactory instance of {@link LockFactory} providing the - * locking implementation. If null, the default - * {@link SimpleFSLockFactory} is used. + * locking implementation. * @return the FSDirectory for the named file. */ public static FSDirectory getDirectory(File file, LockFactory lockFactory) - throws IOException { - return getDirectory(file, lockFactory, DEFAULT_DO_SYNC); - } - - /** Returns the directory instance for the named location. - * @param file the path to the directory. - * @param lockFactory instance of {@link LockFactory} providing the - * locking implementation. If null, the default - * {@link SimpleFSLockFactory} is used. - * @param doSync if true (the default), sync() is called - * on all file descriptors before close(). This - * improves the likelihood that the index will - * remain consistent when the OS or machine crashes - * or the power cord is pulled. - * @return the FSDirectory for the named file. */ - public static FSDirectory getDirectory(File file, LockFactory lockFactory, boolean doSync) throws IOException { file = new File(file.getCanonicalPath()); @@ -225,7 +183,7 @@ public class FSDirectory extends Directory { } catch (Exception e) { throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e); } - dir.init(file, lockFactory, doSync); + dir.init(file, lockFactory); DIRECTORIES.put(file, dir); } else { // Catch the case where a Directory is pulled from the cache, but has a @@ -296,7 +254,7 @@ public class FSDirectory extends Directory { protected FSDirectory() {}; // permit subclassing - private void init(File path, LockFactory lockFactory, boolean doSync) throws IOException { + private void init(File path, LockFactory lockFactory) throws IOException { // Set up lockFactory with cascaded defaults: if an instance was passed in, // use that; else if locks are disabled, use NoLockFactory; else if the @@ -304,7 +262,6 @@ public class FSDirectory extends Directory { // instantiate that; else, use SimpleFSLockFactory: directory = path; - this.doSync = doSync; boolean doClearLockID = false; @@ -475,7 +432,7 @@ public class FSDirectory extends Directory { if (file.exists() && !file.delete()) // delete existing, if any throw new IOException("Cannot overwrite: " + file); - return new FSIndexOutput(file, doSync); + return new FSIndexOutput(file); } // Inherit javadoc @@ -631,17 +588,10 @@ public class FSDirectory extends Directory { // remember if the file is open, so that we don't try to close it // more than once private boolean isOpen; - private boolean doSync; public FSIndexOutput(File path) throws IOException { - this(path, DEFAULT_DO_SYNC); - } - - public FSIndexOutput(File path, boolean doSync) throws IOException { file = new RandomAccessFile(path, "rw"); - isOpen = true; - this.doSync = doSync; } /** output methods: */ @@ -651,14 +601,9 @@ public class FSDirectory extends Directory { public void close() throws IOException { // only close the file if it has not been closed yet if (isOpen) { - try { - super.close(); - if (doSync) - file.getFD().sync(); - } finally { - file.close(); - isOpen = false; - } + super.close(); + file.close(); + isOpen = false; } } diff --git a/src/test/org/apache/lucene/index/TestAtomicUpdate.java b/src/test/org/apache/lucene/index/TestAtomicUpdate.java index 17c7e813c09..f6a698cb32a 100644 --- a/src/test/org/apache/lucene/index/TestAtomicUpdate.java +++ b/src/test/org/apache/lucene/index/TestAtomicUpdate.java @@ -177,7 +177,7 @@ public class TestAtomicUpdate extends LuceneTestCase { // Second in an FSDirectory: String tempDir = System.getProperty("java.io.tmpdir"); File dirPath = new File(tempDir, "lucene.test.atomic"); - directory = FSDirectory.getDirectory(dirPath, null, false); + directory = FSDirectory.getDirectory(dirPath); runTest(directory); directory.close(); _TestUtil.rmDir(dirPath); diff --git a/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index d85a352cc86..0e65f2729bb 100644 --- a/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -152,7 +152,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName, null, false); + Directory dir = FSDirectory.getDirectory(dirName); IndexSearcher searcher = new IndexSearcher(dir); Hits hits = searcher.search(new TermQuery(new Term("content", "aaa"))); @@ -172,7 +172,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName, null, false); + Directory dir = FSDirectory.getDirectory(dirName); // open writer IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false); @@ -232,7 +232,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName, null, false); + Directory dir = FSDirectory.getDirectory(dirName); // make sure searching sees right # hits IndexSearcher searcher = new IndexSearcher(dir); @@ -280,7 +280,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase dirName = fullDir(dirName); - Directory dir = FSDirectory.getDirectory(dirName, null, false); + Directory dir = FSDirectory.getDirectory(dirName); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true); writer.setUseCompoundFile(doCFS); @@ -311,7 +311,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase rmDir(outputDir); try { - Directory dir = FSDirectory.getDirectory(fullDir(outputDir), null, false); + Directory dir = FSDirectory.getDirectory(fullDir(outputDir)); boolean autoCommit = 0 == pass; diff --git a/src/test/org/apache/lucene/index/TestCompoundFile.java b/src/test/org/apache/lucene/index/TestCompoundFile.java index 26f96575e28..de0b617c39c 100644 --- a/src/test/org/apache/lucene/index/TestCompoundFile.java +++ b/src/test/org/apache/lucene/index/TestCompoundFile.java @@ -62,7 +62,7 @@ public class TestCompoundFile extends LuceneTestCase super.setUp(); File file = new File(System.getProperty("tempDir"), "testIndex"); _TestUtil.rmDir(file); - dir = FSDirectory.getDirectory(file, null, false); + dir = FSDirectory.getDirectory(file); } diff --git a/src/test/org/apache/lucene/index/TestIndexModifier.java b/src/test/org/apache/lucene/index/TestIndexModifier.java index 0822938e2a8..b882ae1cf78 100644 --- a/src/test/org/apache/lucene/index/TestIndexModifier.java +++ b/src/test/org/apache/lucene/index/TestIndexModifier.java @@ -147,7 +147,7 @@ public class TestIndexModifier extends LuceneTestCase { if (tempDir == null) throw new IOException("java.io.tmpdir undefined, cannot run test"); File indexDir = new File(tempDir, "lucenetestindex"); - Directory rd = FSDirectory.getDirectory(indexDir, null, false); + Directory rd = FSDirectory.getDirectory(indexDir); IndexThread.id = 0; IndexThread.idStack.clear(); IndexModifier index = new IndexModifier(rd, new StandardAnalyzer(), create); diff --git a/src/test/org/apache/lucene/index/TestNorms.java b/src/test/org/apache/lucene/index/TestNorms.java index 37d1d7f1431..1b662690c3b 100755 --- a/src/test/org/apache/lucene/index/TestNorms.java +++ b/src/test/org/apache/lucene/index/TestNorms.java @@ -81,7 +81,7 @@ public class TestNorms extends LuceneTestCase { // test with a single index: index1 File indexDir1 = new File(tempDir, "lucenetestindex1"); - Directory dir1 = FSDirectory.getDirectory(indexDir1, null, false); + Directory dir1 = FSDirectory.getDirectory(indexDir1); norms = new ArrayList(); modifiedNorms = new ArrayList(); @@ -99,14 +99,14 @@ public class TestNorms extends LuceneTestCase { numDocNorms = 0; File indexDir2 = new File(tempDir, "lucenetestindex2"); - Directory dir2 = FSDirectory.getDirectory(indexDir2, null, false); + Directory dir2 = FSDirectory.getDirectory(indexDir2); createIndex(dir2); doTestNorms(dir2); // add index1 and index2 to a third index: index3 File indexDir3 = new File(tempDir, "lucenetestindex3"); - Directory dir3 = FSDirectory.getDirectory(indexDir3, null, false); + Directory dir3 = FSDirectory.getDirectory(indexDir3); createIndex(dir3); IndexWriter iw = new IndexWriter(dir3,anlzr,false); diff --git a/src/test/org/apache/lucene/index/TestStressIndexing.java b/src/test/org/apache/lucene/index/TestStressIndexing.java index cfa8f73e78b..0f9868ecdf6 100644 --- a/src/test/org/apache/lucene/index/TestStressIndexing.java +++ b/src/test/org/apache/lucene/index/TestStressIndexing.java @@ -178,7 +178,7 @@ public class TestStressIndexing extends LuceneTestCase { // FSDir String tempDir = System.getProperty("java.io.tmpdir"); File dirPath = new File(tempDir, "lucene.test.stress"); - directory = FSDirectory.getDirectory(dirPath, null, false); + directory = FSDirectory.getDirectory(dirPath); runStressTest(directory, true, null); directory.close(); @@ -188,7 +188,7 @@ public class TestStressIndexing extends LuceneTestCase { directory.close(); // With ConcurrentMergeScheduler, in FSDir - directory = FSDirectory.getDirectory(dirPath, null, false); + directory = FSDirectory.getDirectory(dirPath); runStressTest(directory, true, new ConcurrentMergeScheduler()); directory.close(); @@ -198,7 +198,7 @@ public class TestStressIndexing extends LuceneTestCase { directory.close(); // With ConcurrentMergeScheduler and autoCommit=false, in FSDir - directory = FSDirectory.getDirectory(dirPath, null, false); + directory = FSDirectory.getDirectory(dirPath); runStressTest(directory, false, new ConcurrentMergeScheduler()); directory.close(); diff --git a/src/test/org/apache/lucene/index/TestThreadedOptimize.java b/src/test/org/apache/lucene/index/TestThreadedOptimize.java index e5fee5c8965..5ee7a76fead 100644 --- a/src/test/org/apache/lucene/index/TestThreadedOptimize.java +++ b/src/test/org/apache/lucene/index/TestThreadedOptimize.java @@ -149,7 +149,7 @@ public class TestThreadedOptimize extends LuceneTestCase { throw new IOException("tempDir undefined, cannot run test"); String dirName = tempDir + "/luceneTestThreadedOptimize"; - directory = FSDirectory.getDirectory(dirName, null, false); + directory = FSDirectory.getDirectory(dirName); runTest(directory, false, null); runTest(directory, true, null); runTest(directory, false, new ConcurrentMergeScheduler()); diff --git a/src/test/org/apache/lucene/store/TestLockFactory.java b/src/test/org/apache/lucene/store/TestLockFactory.java index 6d125bea518..e512a0f2908 100755 --- a/src/test/org/apache/lucene/store/TestLockFactory.java +++ b/src/test/org/apache/lucene/store/TestLockFactory.java @@ -325,7 +325,7 @@ public class TestLockFactory extends LuceneTestCase { } public void _testStressLocks(LockFactory lockFactory, String indexDirName) throws IOException { - FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory, false); + FSDirectory fs1 = FSDirectory.getDirectory(indexDirName, lockFactory); // First create a 1 doc index: IndexWriter w = new IndexWriter(fs1, new WhitespaceAnalyzer(), true);