Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-748: add javadoc around semantics of Exception during IndexWriter.close() 

LUCENE-129: fix finalizers to always call super.finalize
LUCENE-301: add new constructors IndexWriter({String,File,Directory}, Analyzer) that create index if it's not already there, else append
LUCENE-701: found two cases of "open IndexWriter for create while reader is open, on Windows" that I didn't properly fix; added new test cases
LUCENE-702: corrected some small javadoc issues


git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@488640 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2006-12-19 11:31:27 +00:00
parent d6823ef374
commit 7c4b667f71
9 changed files with 319 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
CHANGES.txt
View File

@ -238,7 +238,7 @@ Bug fixes
FSIndexInput/Output during finalize(). Besides sending an
IOException up to the GC, this may also be the cause intermittent
"The handle is invalid" IOExceptions on Windows when trying to
close readers or writers. (Michael Busch via Mike McCandless).
close readers or writers. (Michael Busch via Mike McCandless)
26. LUCENE-702: Fix IndexWriter.addIndexes(*) to not corrupt the index
on any exceptions (eg disk full). The semantics of these methods
@ -249,6 +249,10 @@ Bug fixes
instance state consistent to what's actually in the index (Mike
McCandless).
27. LUCENE-129: Change finalizers to do "try {...} finally
{super.finalize();}" to make sure we don't miss finalizers in
classes above us. (Esmond Pitt via Mike McCandless)
Optimizations
1. LUCENE-586: TermDocs.skipTo() is now more efficient for
@ -348,6 +352,9 @@ Documentation
7. Added in link to Clover Test Code Coverage Reports under the Develop section in Resources (Grant Ingersoll)
8. LUCENE-748: Added details for semantics of IndexWriter.close on
hitting an Exception. (Jed Wesley-Smith via Mike McCandless)
Build
1. Added in clover test code coverage per http://issues.apache.org/jira/browse/LUCENE-721 To enable clover code coverage, you must have clover.jar in the ANT classpath and specify -Drun.clover=true on the command line.(Michael Busch and Grant Ingersoll)
@ -370,6 +377,12 @@ API Changes
now throws an IllegalArgumentException
(Daniel Naber)
4. LUCENE-301: Added new IndexWriter({String,File,Directory},
Analyzer) constructors that do not take a boolean "create"
argument. These new constructors will create a new index if
necessary, else append to the existing one. (Dan Armbrust via
Mike McCandless)
New features
1. LUCENE-496: Command line tool for modifying the field norms of an

8
contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java
View File

@ -342,8 +342,12 @@ public class SpellChecker {
}
protected void finalize() throws Throwable {
if (reader != null) {
reader.close();
try {
if (reader != null) {
reader.close();
}
} finally {
super.finalize();
}
}
}

12
src/java/org/apache/lucene/index/IndexReader.java
View File

@ -721,10 +721,14 @@ public abstract class IndexReader {
protected abstract void doClose() throws IOException;
/** Release the write lock, if needed. */
protected void finalize() {
if (writeLock != null) {
writeLock.release(); // release write lock
writeLock = null;
protected void finalize() throws Throwable {
try {
if (writeLock != null) {
writeLock.release(); // release write lock
writeLock = null;
}
} finally {
super.finalize();
}
}

208
src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -202,7 +202,7 @@ public class IndexWriter {
*/
public IndexWriter(String path, Analyzer a, boolean create)
throws IOException {
this(FSDirectory.getDirectory(path, create), a, create, true);
init(path, a, create);
}
/**
@ -222,7 +222,7 @@ public class IndexWriter {
*/
public IndexWriter(File path, Analyzer a, boolean create)
throws IOException {
this(FSDirectory.getDirectory(path, create), a, create, true);
init(path, a, create);
}
/**
@ -242,49 +242,125 @@ public class IndexWriter {
*/
public IndexWriter(Directory d, Analyzer a, boolean create)
throws IOException {
this(d, a, create, false);
init(d, a, create, false);
}
/**
* Constructs an IndexWriter for the index in
* <code>path</code>, creating it first if it does not
* already exist, otherwise appending to the existing
* index. Text will be analyzed with <code>a</code>.
*
* @param path the path to the index directory
* @param a the analyzer to use
* @throws IOException if the directory cannot be
* created or read/written to
*/
public IndexWriter(String path, Analyzer a)
throws IOException {
if (IndexReader.indexExists(path)) {
init(path, a, false);
} else {
init(path, a, true);
}
}
/**
* Constructs an IndexWriter for the index in
* <code>path</code>, creating it first if it does not
* already exist, otherwise appending to the existing
* index. Text will be analyzed with
* <code>a</code>.
*
* @param path the path to the index directory
* @param a the analyzer to use
* @throws IOException if the directory cannot be
* created or read/written to
*/
public IndexWriter(File path, Analyzer a)
throws IOException {
if (IndexReader.indexExists(path)) {
init(path, a, false);
} else {
init(path, a, true);
}
}
/**
* Constructs an IndexWriter for the index in
* <code>d</code>, creating it first if it does not
* already exist, otherwise appending to the existing
* index. Text will be analyzed with <code>a</code>.
*
* @param d the index directory
* @param a the analyzer to use
* @throws IOException if the directory cannot be
* created or read/written to
*/
public IndexWriter(Directory d, Analyzer a)
throws IOException {
if (IndexReader.indexExists(d)) {
init(d, a, false, false);
} else {
init(d, a, true, false);
}
}
private IndexWriter(Directory d, Analyzer a, final boolean create, boolean closeDir)
throws IOException {
this.closeDir = closeDir;
directory = d;
analyzer = a;
init(d, a, create, closeDir);
}
Lock writeLock = directory.makeLock(IndexWriter.WRITE_LOCK_NAME);
if (!writeLock.obtain(writeLockTimeout)) // obtain write lock
throw new IOException("Index locked for write: " + writeLock);
this.writeLock = writeLock; // save it
private void init(String path, Analyzer a, final boolean create)
throws IOException {
init(FSDirectory.getDirectory(path, create, null, false), a, create, true);
}
try {
if (create) {
// Try to read first. This is to allow create
// against an index that's currently open for
// searching. In this case we write the next
// segments_N file with no segments:
try {
segmentInfos.read(directory);
segmentInfos.clear();
} catch (IOException e) {
// Likely this means it's a fresh directory
}
segmentInfos.write(directory);
} else {
private void init(File path, Analyzer a, final boolean create)
throws IOException {
init(FSDirectory.getDirectory(path, create, null, false), a, create, true);
}
private void init(Directory d, Analyzer a, final boolean create, boolean closeDir)
throws IOException {
this.closeDir = closeDir;
directory = d;
analyzer = a;
Lock writeLock = directory.makeLock(IndexWriter.WRITE_LOCK_NAME);
if (!writeLock.obtain(writeLockTimeout)) // obtain write lock
throw new IOException("Index locked for write: " + writeLock);
this.writeLock = writeLock; // save it
try {
if (create) {
// Try to read first. This is to allow create
// against an index that's currently open for
// searching. In this case we write the next
// segments_N file with no segments:
try {
segmentInfos.read(directory);
segmentInfos.clear();
} catch (IOException e) {
// Likely this means it's a fresh directory
}
// Create a deleter to keep track of which files can
// be deleted:
deleter = new IndexFileDeleter(segmentInfos, directory);
deleter.setInfoStream(infoStream);
deleter.findDeletableFiles();
deleter.deleteFiles();
} catch (IOException e) {
this.writeLock.release();
this.writeLock = null;
throw e;
segmentInfos.write(directory);
} else {
segmentInfos.read(directory);
}
// Create a deleter to keep track of which files can
// be deleted:
deleter = new IndexFileDeleter(segmentInfos, directory);
deleter.setInfoStream(infoStream);
deleter.findDeletableFiles();
deleter.deleteFiles();
} catch (IOException e) {
this.writeLock.release();
this.writeLock = null;
throw e;
}
}
/** Determines the largest number of documents ever merged by addDocument().
@ -418,7 +494,38 @@ public class IndexWriter {
return IndexWriter.WRITE_LOCK_TIMEOUT;
}
/** Flushes all changes to an index and closes all associated files. */
/**
* Flushes all changes to an index and closes all
* associated files.
*
* <p> If an Exception is hit during close, eg due to disk
* full or some other reason, then both the on-disk index
* and the internal state of the IndexWriter instance will
* be consistent. However, the close will not be complete
* even though part of it (flushing buffered documents)
* may have succeeded, so the write lock will still be
* held.</p>
*
* <p> If you can correct the underlying cause (eg free up
* some disk space) then you can call close() again.
* Failing that, if you want to force the write lock to be
* released (dangerous, because you may then lose buffered
* docs in the IndexWriter instance) then you can do
* something like this:</p>
*
* <pre>
* try {
* writer.close();
* } finally {
* if (IndexReader.isLocked(directory)) {
* IndexReader.unlock(directory);
* }
* }
* </pre>
*
* after which, you must be certain not to use the writer
* instance anymore.</p>
*/
public synchronized void close() throws IOException {
flushRamSegments();
ramDirectory.close();
@ -431,10 +538,14 @@ public class IndexWriter {
}
/** Release the write lock, if needed. */
protected void finalize() throws IOException {
if (writeLock != null) {
writeLock.release(); // release write lock
writeLock = null;
protected void finalize() throws Throwable {
try {
if (writeLock != null) {
writeLock.release(); // release write lock
writeLock = null;
}
} finally {
super.finalize();
}
}
@ -479,11 +590,12 @@ public class IndexWriter {
* {@link #setMaxFieldLength(int)} terms for a given field, the remainder are
* discarded.
*
* Note that if an Exception is hit (eg disk full) then
* the index will be consistent, but this document will
* not have been added. Furthermore, it's possible the
* index will have one segment in non-compound format even
* when using compound files.
* <p> Note that if an Exception is hit (eg disk full)
* then the index will be consistent, but this document
* may not have been added. Furthermore, it's possible
* the index will have one segment in non-compound format
* even when using compound files (when a merge has
* partially succeeded).</p>
*/
public void addDocument(Document doc) throws IOException {
addDocument(doc, analyzer);
@ -495,8 +607,8 @@ public class IndexWriter {
* {@link #setMaxFieldLength(int)} terms for a given field, the remainder are
* discarded.
*
* See @link #addDocument(Document) for details on index
* state after an IOException.
* <p>See {@link #addDocument(Document)} for details on
* index and IndexWriter state after an Exception.</p>
*/
public void addDocument(Document doc, Analyzer analyzer) throws IOException {
DocumentWriter dw =

11
src/java/org/apache/lucene/index/SegmentReader.java
View File

@ -187,10 +187,13 @@ class SegmentReader extends IndexReader {
}
}
protected void finalize() {
// patch for pre-1.4.2 JVMs, whose ThreadLocals leak
termVectorsLocal.set(null);
super.finalize();
protected void finalize() throws Throwable {
try {
// patch for pre-1.4.2 JVMs, whose ThreadLocals leak
termVectorsLocal.set(null);
} finally {
super.finalize();
}
}
protected void doCommit() throws IOException {

10
src/java/org/apache/lucene/index/TermInfosReader.java
View File

@ -55,9 +55,13 @@ final class TermInfosReader {
fieldInfos, true);
}
protected void finalize() {
// patch for pre-1.4.2 JVMs, whose ThreadLocals leak
enumerators.set(null);
protected void finalize() throws Throwable {
try {
// patch for pre-1.4.2 JVMs, whose ThreadLocals leak
enumerators.set(null);
} finally {
super.finalize();
}
}
public int getSkipInterval() {

16
src/java/org/apache/lucene/store/FSDirectory.java
View File

@ -554,8 +554,12 @@ class FSIndexInput extends BufferedIndexInput {
return length;
}
protected void finalize() throws IOException {
close(); // close the file
protected void finalize() throws Throwable {
try {
close(); // close the file
} finally {
super.finalize();
}
}
public Object clone() {
@ -607,8 +611,12 @@ class FSIndexOutput extends BufferedIndexOutput {
return file.length();
}
protected void finalize() throws IOException {
close(); // close the file
protected void finalize() throws Throwable {
try {
close(); // close the file
} finally {
super.finalize();
}
}
}

10
src/java/org/apache/lucene/store/NativeFSLockFactory.java
View File

@ -333,9 +333,13 @@ class NativeFSLock extends Lock {
return "NativeFSLock@" + path;
}
public void finalize() {
if (isLocked()) {
release();
public void finalize() throws Throwable {
try {
if (isLocked()) {
release();
}
} finally {
super.finalize();
}
}
}

121
src/test/org/apache/lucene/index/TestIndexWriter.java
View File

@ -39,7 +39,7 @@ public class TestIndexWriter extends TestCase
IndexWriter.setDefaultWriteLockTimeout(2000);
assertEquals(2000, IndexWriter.getDefaultWriteLockTimeout());
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
writer = new IndexWriter(dir, new WhitespaceAnalyzer());
IndexWriter.setDefaultWriteLockTimeout(1000);
@ -58,7 +58,7 @@ public class TestIndexWriter extends TestCase
reader.close();
// test doc count before segments are merged/index is optimized
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
writer = new IndexWriter(dir, new WhitespaceAnalyzer());
assertEquals(100, writer.docCount());
writer.close();
@ -68,7 +68,7 @@ public class TestIndexWriter extends TestCase
reader.close();
// optimize the index and check that the new doc count is correct
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
writer = new IndexWriter(dir, new WhitespaceAnalyzer());
writer.optimize();
assertEquals(60, writer.docCount());
writer.close();
@ -445,29 +445,102 @@ public class TestIndexWriter extends TestCase
if (tempDir == null)
throw new IOException("java.io.tmpdir undefined, cannot run test");
File indexDir = new File(tempDir, "lucenetestindexwriter");
Directory dir = FSDirectory.getDirectory(indexDir, true);
// add one document & close writer
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
addDoc(writer);
writer.close();
try {
Directory dir = FSDirectory.getDirectory(indexDir, true);
// now open reader:
IndexReader reader = IndexReader.open(dir);
assertEquals("should be one document", reader.numDocs(), 1);
// add one document & close writer
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
addDoc(writer);
writer.close();
// now open index for create:
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
assertEquals("should be zero documents", writer.docCount(), 0);
addDoc(writer);
writer.close();
// now open reader:
IndexReader reader = IndexReader.open(dir);
assertEquals("should be one document", reader.numDocs(), 1);
assertEquals("should be one document", reader.numDocs(), 1);
IndexReader reader2 = IndexReader.open(dir);
assertEquals("should be one document", reader2.numDocs(), 1);
reader.close();
reader2.close();
rmDir(indexDir);
// now open index for create:
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
assertEquals("should be zero documents", writer.docCount(), 0);
addDoc(writer);
writer.close();
assertEquals("should be one document", reader.numDocs(), 1);
IndexReader reader2 = IndexReader.open(dir);
assertEquals("should be one document", reader2.numDocs(), 1);
reader.close();
reader2.close();
} finally {
rmDir(indexDir);
}
}
// Same test as above, but use IndexWriter constructor
// that takes File:
public void testCreateWithReader2() throws IOException {
String tempDir = System.getProperty("java.io.tmpdir");
if (tempDir == null)
throw new IOException("java.io.tmpdir undefined, cannot run test");
File indexDir = new File(tempDir, "lucenetestindexwriter");
try {
// add one document & close writer
IndexWriter writer = new IndexWriter(indexDir, new WhitespaceAnalyzer(), true);
addDoc(writer);
writer.close();
// now open reader:
IndexReader reader = IndexReader.open(indexDir);
assertEquals("should be one document", reader.numDocs(), 1);
// now open index for create:
writer = new IndexWriter(indexDir, new WhitespaceAnalyzer(), true);
assertEquals("should be zero documents", writer.docCount(), 0);
addDoc(writer);
writer.close();
assertEquals("should be one document", reader.numDocs(), 1);
IndexReader reader2 = IndexReader.open(indexDir);
assertEquals("should be one document", reader2.numDocs(), 1);
reader.close();
reader2.close();
} finally {
rmDir(indexDir);
}
}
// Same test as above, but use IndexWriter constructor
// that takes String:
public void testCreateWithReader3() throws IOException {
String tempDir = System.getProperty("tempDir");
if (tempDir == null)
throw new IOException("java.io.tmpdir undefined, cannot run test");
String dirName = tempDir + "/lucenetestindexwriter";
try {
// add one document & close writer
IndexWriter writer = new IndexWriter(dirName, new WhitespaceAnalyzer(), true);
addDoc(writer);
writer.close();
// now open reader:
IndexReader reader = IndexReader.open(dirName);
assertEquals("should be one document", reader.numDocs(), 1);
// now open index for create:
writer = new IndexWriter(dirName, new WhitespaceAnalyzer(), true);
assertEquals("should be zero documents", writer.docCount(), 0);
addDoc(writer);
writer.close();
assertEquals("should be one document", reader.numDocs(), 1);
IndexReader reader2 = IndexReader.open(dirName);
assertEquals("should be one document", reader2.numDocs(), 1);
reader.close();
reader2.close();
} finally {
rmDir(new File(dirName));
}
}
// Simulate a writer that crashed while writing segments
@ -619,8 +692,10 @@ public class TestIndexWriter extends TestCase
private void rmDir(File dir) {
File[] files = dir.listFiles();
for (int i = 0; i < files.length; i++) {
if (files != null) {
for (int i = 0; i < files.length; i++) {
files[i].delete();
}
}
dir.delete();
}