Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-2805: SegmentInfos shouldn't blindly increment version on commit 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1043148 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Simon Willnauer 2010-12-07 17:57:14 +00:00
parent 620b2a0619
commit 6c9052d231
8 changed files with 87 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/CHANGES.txt
View File

@ -131,6 +131,11 @@ Changes in Runtime Behavior
to determine whether the passed in segment should be compound.
(Shai Erera, Earwin Burrfoot)
* LUCENE-2805: IndexWriter now increments the index version on every change to
the index instead of for every commit. Committing or closing the IndexWriter
without any changes to the index will not cause any index version increment.
(Simon Willnauer, Mike Mccandless)
API Changes
* LUCENE-2302, LUCENE-1458, LUCENE-2111, LUCENE-2514: Terms are no longer

2
lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
View File

@ -134,6 +134,7 @@ public class IndexSplitter {
int idx = getIdx(n);
infos.remove(idx);
}
infos.changed();
infos.commit(fsDir);
}
@ -152,6 +153,7 @@ public class IndexSplitter {
copyFile(srcFile, destFile);
}
}
destInfos.changed();
destInfos.commit(destFSDir);
// System.out.println("destDir:"+destDir.getAbsolutePath());
}

1
lucene/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -879,6 +879,7 @@ public class CheckIndex {
public void fixIndex(Status result) throws IOException {
if (result.partial)
throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)");
result.newSegments.changed();
result.newSegments.commit(result.dir);
}

2
lucene/src/java/org/apache/lucene/index/DirectoryReader.java
View File

@ -770,6 +770,7 @@ class DirectoryReader extends IndexReader implements Cloneable {
deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
segmentInfos, null, null, codecs);
segmentInfos.updateGeneration(deleter.getLastSegmentInfos());
segmentInfos.changed();
// Checkpoint the state we are about to change, in
// case we have to roll back:
@ -782,7 +783,6 @@ class DirectoryReader extends IndexReader implements Cloneable {
// Sync all files we just wrote
directory.sync(segmentInfos.files(directory, false));
segmentInfos.commit(directory);
success = true;
} finally {

22
lucene/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -741,6 +741,7 @@ public class IndexWriter implements Closeable {
// Record that we have a change (zero out all
// segments) pending:
changeCount++;
segmentInfos.changed();
} else {
segmentInfos.read(directory, codecs);
@ -757,6 +758,7 @@ public class IndexWriter implements Closeable {
oldInfos.read(directory, commit.getSegmentsFileName(), codecs);
segmentInfos.replace(oldInfos);
changeCount++;
segmentInfos.changed();
if (infoStream != null)
message("init: loaded commit \"" + commit.getSegmentsFileName() + "\"");
}
@ -774,12 +776,14 @@ public class IndexWriter implements Closeable {
conf.getIndexDeletionPolicy(),
segmentInfos, infoStream, docWriter, codecs);
if (deleter.startingCommitDeleted)
if (deleter.startingCommitDeleted) {
// Deletion policy deleted the "head" commit point.
// We have to mark ourself as changed so that if we
// are closed w/o any further changes we write a new
// segments_N file.
changeCount++;
segmentInfos.changed();
}
docWriter.setMaxBufferedDeleteTerms(conf.getMaxBufferedDeleteTerms());
docWriter.setRAMBufferSizeMB(conf.getRAMBufferSizeMB());
@ -1537,6 +1541,7 @@ public class IndexWriter implements Closeable {
// name that was previously returned which can cause
// problems at least with ConcurrentMergeScheduler.
changeCount++;
segmentInfos.changed();
return "_" + Integer.toString(segmentInfos.counter++, Character.MAX_RADIX);
}
}
@ -2038,6 +2043,7 @@ public class IndexWriter implements Closeable {
// Mark that the index has changed
++changeCount;
segmentInfos.changed();
} catch (OutOfMemoryError oom) {
handleOOM(oom, "deleteAll");
} finally {
@ -2119,6 +2125,7 @@ public class IndexWriter implements Closeable {
*/
private synchronized void checkpoint() throws IOException {
changeCount++;
segmentInfos.changed();
deleter.checkpoint(segmentInfos, false);
}
@ -3686,6 +3693,7 @@ public class IndexWriter implements Closeable {
}
toSync.remove(toSync.size()-1);
changeCount++;
segmentInfos.changed();
}
}
assert filesExist(toSync);
@ -3818,17 +3826,7 @@ public class IndexWriter implements Closeable {
}
synchronized boolean nrtIsCurrent(SegmentInfos infos) {
if (!infos.equals(segmentInfos)) {
// if any structural changes (new segments), we are
// stale
return false;
} else if (infos.getGeneration() != segmentInfos.getGeneration()) {
// if any commit took place since we were opened, we
// are stale
return false;
} else {
return !docWriter.anyChanges();
}
return infos.version == segmentInfos.version && !docWriter.anyChanges();
}
synchronized boolean isClosed() {

21
lucene/src/java/org/apache/lucene/index/SegmentInfos.java
View File

@ -696,7 +696,6 @@ public final class SegmentInfos extends Vector<SegmentInfo> {
void updateGeneration(SegmentInfos other) {
lastGeneration = other.lastGeneration;
generation = other.generation;
version = other.version;
}
final void rollbackCommit(Directory dir) throws IOException {
@ -727,7 +726,12 @@ public final class SegmentInfos extends Vector<SegmentInfo> {
* segments file, but writes an invalid checksum at the
* end, so that it is not visible to readers. Once this
* is called you must call {@link #finishCommit} to complete
* the commit or {@link #rollbackCommit} to abort it. */
* the commit or {@link #rollbackCommit} to abort it.
* <p>
* Note: {@link #changed()} should be called prior to this
* method if changes have been made to this {@link SegmentInfos} instance
* </p>
**/
final void prepareCommit(Directory dir) throws IOException {
if (pendingSegnOutput != null)
throw new IllegalStateException("prepareCommit was already called");
@ -811,7 +815,12 @@ public final class SegmentInfos extends Vector<SegmentInfo> {
}
/** Writes & syncs to the Directory dir, taking care to
* remove the segments file on exception */
* remove the segments file on exception
* <p>
* Note: {@link #changed()} should be called prior to this
* method if changes have been made to this {@link SegmentInfos} instance
* </p>
**/
final void commit(Directory dir) throws IOException {
prepareCommit(dir);
finishCommit(dir);
@ -862,4 +871,10 @@ public final class SegmentInfos extends Vector<SegmentInfo> {
}
return count;
}
/** Call this before committing if changes have been made to the
* segments. */
public void changed() {
version++;
}
}

3
lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
View File

@ -51,8 +51,7 @@ public class DefaultSegmentInfosWriter extends SegmentInfosWriter {
throws IOException {
IndexOutput out = createOutput(dir, segmentFileName);
out.writeInt(FORMAT_CURRENT); // write FORMAT
out.writeLong(++infos.version); // every write changes
// the index
out.writeLong(infos.version);
out.writeInt(infos.counter); // write counter
out.writeInt(infos.size()); // write infos
for (SegmentInfo si : infos) {

51
lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
View File

@ -61,7 +61,54 @@ public class TestIndexWriterReader extends LuceneTestCase {
}
return count;
}
public void testAddCloseOpen() throws IOException {
Directory dir1 = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
IndexWriter writer = new IndexWriter(dir1, iwc);
for (int i = 0; i < 97 ; i++) {
IndexReader reader = writer.getReader();
if (i == 0) {
writer.addDocument(createDocument(i, "x", 1 + random.nextInt(5)));
} else {
int previous = random.nextInt(i);
// a check if the reader is current here could fail since there might be
// merges going on.
switch (random.nextInt(5)) {
case 0:
case 1:
case 2:
writer.addDocument(createDocument(i, "x", 1 + random.nextInt(5)));
break;
case 3:
writer.updateDocument(new Term("id", "" + previous), createDocument(
previous, "x", 1 + random.nextInt(5)));
break;
case 4:
writer.deleteDocuments(new Term("id", "" + previous));
}
}
assertFalse(reader.isCurrent());
reader.close();
}
writer.optimize(); // make sure all merging is done etc.
IndexReader reader = writer.getReader();
writer.commit(); // no changes that are not visible to the reader
assertTrue(reader.isCurrent());
writer.close();
assertTrue(reader.isCurrent()); // all changes are visible to the reader
iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
writer = new IndexWriter(dir1, iwc);
assertTrue(reader.isCurrent());
writer.addDocument(createDocument(1, "x", 1+random.nextInt(5)));
assertTrue(reader.isCurrent()); // segments in ram but IW is different to the readers one
writer.close();
assertFalse(reader.isCurrent()); // segments written
reader.close();
dir1.close();
}
public void testUpdateDocument() throws Exception {
boolean optimize = true;
@ -171,7 +218,7 @@ public class TestIndexWriterReader extends LuceneTestCase {
assertTrue(r1.isCurrent());
writer.commit();
assertFalse(r1.isCurrent());
assertTrue(r1.isCurrent()); // we have seen all changes - no change after opening the NRT reader
assertEquals(200, r1.maxDoc());