Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4456: fix various bugs (see lucene/CHANGES.txt) 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1393408 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-10-03 11:01:13 +00:00
parent c98f935688
commit cb40231513
26 changed files with 442 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lucene/CHANGES.txt
View File

@ -226,6 +226,14 @@ Bug Fixes
CheckIndex that would report no deletions when a segment has
deletions, and vice/versa. (Uwe Schindler, Robert Muir, Mike McCandless)
* LUCENE-4456: Fixed double-counting sizeInBytes for a segment
(affects how merge policies pick merges); fixed CheckIndex's
incorrect reporting of whether a segment has deletions; fixed case
where on abort Lucene could remove files it didn't create; fixed
many cases where IndexWriter could leave leftover files (on
exception in various places, on reuse of a segment name after crash
and recovery. (Uwe Schindler, Robert Muir, Mike McCandless)
Optimizations
* LUCENE-4322: Decrease lucene-core JAR size. The core JAR size had increased a

1
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java
View File

@ -198,6 +198,7 @@ public class SimpleTextDocValuesConsumer extends DocValuesConsumer {
IOUtils.close(output);
} else {
IOUtils.closeWhileHandlingException(output);
dir.deleteFile(fileName);
}
}
}

1
lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
View File

@ -137,6 +137,7 @@ class DocumentsWriterPerThread {
* currently buffered docs. This resets our state,
* discarding any docs added since last flush. */
void abort() {
//System.out.println(Thread.currentThread().getName() + ": now abort seg=" + segmentInfo.name);
hasAborted = aborting = true;
try {
if (infoStream.isEnabled("DWPT")) {

14
lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
View File

@ -333,12 +333,16 @@ final class IndexFileDeleter {
segmentPrefix2 = null;
}
Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
for(int i=0;i<files.length;i++) {
String fileName = files[i];
m.reset(fileName);
if ((segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
!fileName.endsWith("write.lock") &&
!refCounts.containsKey(fileName) &&
!fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
!fileName.equals(IndexFileNames.SEGMENTS_GEN) &&
(m.matches() || fileName.startsWith(IndexFileNames.SEGMENTS))) {
// Unreferenced file, so remove it
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "refresh [prefix=" + segmentName + "]: removing newly created unreferenced file \"" + fileName + "\"");
@ -554,7 +558,13 @@ final class IndexFileDeleter {
void deleteNewFiles(Collection<String> files) throws IOException {
assert locked();
for (final String fileName: files) {
if (!refCounts.containsKey(fileName)) {
// NOTE: it's very unusual yet possible for the
// refCount to be present and 0: it can happen if you
// open IW on a crashed index, and it removes a bunch
// of unref'd files, and then you add new docs / do
// merging, and it reuses that segment name.
// TestCrash.testCrashAfterReopen can hit this:
if (!refCounts.containsKey(fileName) || refCounts.get(fileName).count == 0) {
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "delete new file \"" + fileName + "\"");
}

169
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -2145,7 +2145,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
// Now build compound file
Collection<String> oldFiles = createCompoundFile(infoStream, directory, MergeState.CheckAbort.NONE, newSegment.info, context);
newSegment.info.setUseCompoundFile(true);
synchronized(this) {
deleter.deleteNewFiles(oldFiles);
}
@ -2321,31 +2321,62 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
flush(false, true);
List<SegmentInfoPerCommit> infos = new ArrayList<SegmentInfoPerCommit>();
for (Directory dir : dirs) {
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "addIndexes: process directory " + dir);
}
SegmentInfos sis = new SegmentInfos(); // read infos from dir
sis.read(dir);
for (SegmentInfoPerCommit info : sis) {
assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" + info.info.name;
String newSegName = newSegmentName();
String dsName = info.info.name;
boolean success = false;
try {
for (Directory dir : dirs) {
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "addIndexes: process segment origName=" + info.info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
infoStream.message("IW", "addIndexes: process directory " + dir);
}
SegmentInfos sis = new SegmentInfos(); // read infos from dir
sis.read(dir);
IOContext context = new IOContext(new MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
for (SegmentInfoPerCommit info : sis) {
assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" + info.info.name;
String newSegName = newSegmentName();
String dsName = info.info.name;
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "addIndexes: process segment origName=" + info.info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
}
IOContext context = new IOContext(new MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
infos.add(copySegmentAsIs(info, newSegName, context));
infos.add(copySegmentAsIs(info, newSegName, context));
}
}
success = true;
} finally {
if (!success) {
for(SegmentInfoPerCommit sipc : infos) {
for(String file : sipc.files()) {
try {
directory.deleteFile(file);
} catch (Throwable t) {
}
}
}
}
}
synchronized (this) {
ensureOpen();
success = false;
try {
ensureOpen();
success = true;
} finally {
if (!success) {
for(SegmentInfoPerCommit sipc : infos) {
for(String file : sipc.files()) {
try {
directory.deleteFile(file);
} catch (Throwable t) {
}
}
}
}
}
segmentInfos.addAll(infos);
checkpoint();
}
@ -2420,7 +2451,18 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
merger.add(reader);
}
MergeState mergeState = merger.merge(); // merge 'em
MergeState mergeState;
boolean success = false;
try {
mergeState = merger.merge(); // merge 'em
success = true;
} finally {
if (!success) {
synchronized(this) {
deleter.refresh(info.name);
}
}
}
SegmentInfoPerCommit infoPerCommit = new SegmentInfoPerCommit(info, 0, -1L);
@ -2442,12 +2484,14 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
// Now create the compound file if needed
if (useCompoundFile) {
Collection<String> filesToDelete = infoPerCommit.files();
createCompoundFile(infoStream, directory, MergeState.CheckAbort.NONE, info, context);
// delete new non cfs files directly: they were never
// registered with IFD
synchronized(this) {
deleter.deleteNewFiles(filesToDelete);
try{
createCompoundFile(infoStream, directory, MergeState.CheckAbort.NONE, info, context);
} finally {
// delete new non cfs files directly: they were never
// registered with IFD
synchronized(this) {
deleter.deleteNewFiles(filesToDelete);
}
}
info.setUseCompoundFile(true);
}
@ -2456,7 +2500,18 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
// creating CFS so that 1) .si isn't slurped into CFS,
// and 2) .si reflects useCompoundFile=true change
// above:
codec.segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, info, mergeState.fieldInfos, context);
success = false;
try {
codec.segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, info, mergeState.fieldInfos, context);
success = true;
} finally {
if (!success) {
synchronized(this) {
deleter.refresh(info.name);
}
}
}
info.addFiles(trackingDir.getCreatedFiles());
// Register the new segment
@ -2512,23 +2567,38 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
// We must rewrite the SI file because it references segment name in its list of files, etc
TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory);
newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, newInfo, fis, context);
boolean success = false;
final Collection<String> siFiles = trackingDir.getCreatedFiles();
try {
// Copy the segment's files
for (String file: info.files()) {
newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, newInfo, fis, context);
final String newFileName = segName + IndexFileNames.stripSegmentName(file);
final Collection<String> siFiles = trackingDir.getCreatedFiles();
if (siFiles.contains(newFileName)) {
// We already rewrote this above
continue;
// Copy the segment's files
for (String file: info.files()) {
final String newFileName = segName + IndexFileNames.stripSegmentName(file);
if (siFiles.contains(newFileName)) {
// We already rewrote this above
continue;
}
assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists; siFiles=" + siFiles;
info.info.dir.copy(directory, file, newFileName, context);
}
success = true;
} finally {
if (!success) {
for(String file : newInfo.files()) {
try {
directory.deleteFile(file);
} catch (Throwable t) {
}
}
}
assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists; siFiles=" + siFiles;
info.info.dir.copy(directory, file, newFileName, context);
}
return newInfoPerCommit;
@ -3091,6 +3161,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "commitMerge: skip: it was aborted");
}
deleter.deleteNewFiles(merge.info.files());
return false;
}
@ -3136,6 +3207,11 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
}
}
if (dropSegment) {
assert !segmentInfos.contains(merge.info);
deleter.deleteNewFiles(merge.info.files());
}
// Must close before checkpoint, otherwise IFD won't be
// able to delete the held-open files from the merge
// readers:
@ -3407,7 +3483,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
setDiagnostics(si, "merge", details);
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "merge seg=" + merge.info.info.name);
infoStream.message("IW", "merge seg=" + merge.info.info.name + " " + segString(merge.segments));
}
assert merge.estimatedMergeBytes == 0;
@ -4116,7 +4192,22 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
} catch(IOException ex) {
prior = ex;
} finally {
IOUtils.closeWhileHandlingException(prior, cfsDir);
boolean success = false;
try {
IOUtils.closeWhileHandlingException(prior, cfsDir);
success = true;
} finally {
if (!success) {
try {
directory.deleteFile(fileName);
} catch (Throwable t) {
}
try {
directory.deleteFile(IndexFileNames.segmentFileName(info.name, "", IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
} catch (Throwable t) {
}
}
}
}
// Replace all previous files with the CFS/CFE files:

2
lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
View File

@ -547,7 +547,7 @@ public class LiveIndexWriterConfig {
sb.append("ramBufferSizeMB=").append(getRAMBufferSizeMB()).append("\n");
sb.append("maxBufferedDocs=").append(getMaxBufferedDocs()).append("\n");
sb.append("maxBufferedDeleteTerms=").append(getMaxBufferedDeleteTerms()).append("\n");
sb.append("mergedSegmentWarmer=").append(getMergeScheduler()).append("\n");
sb.append("mergedSegmentWarmer=").append(getMergedSegmentWarmer()).append("\n");
sb.append("readerTermsIndexDivisor=").append(getReaderTermsIndexDivisor()).append("\n");
sb.append("termIndexInterval=").append(getTermIndexInterval()).append("\n"); // TODO: this should be private to the codec, not settable here
sb.append("delPolicy=").append(getIndexDeletionPolicy().getClass().getName()).append("\n");

1
lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java
View File

@ -79,6 +79,7 @@ public class SegmentInfoPerCommit {
/** Returns all files in use by this segment. */
public Collection<String> files() throws IOException {
// Start from the wrapped info's files:
Collection<String> files = new HashSet<String>(info.files());
// Must separately add any live docs files:

13
lucene/core/src/java/org/apache/lucene/store/Directory.java
View File

@ -201,7 +201,18 @@ public abstract class Directory implements Closeable {
} catch (IOException ioe) {
priorException = ioe;
} finally {
IOUtils.closeWhileHandlingException(priorException, os, is);
boolean success = false;
try {
IOUtils.closeWhileHandlingException(priorException, os, is);
success = true;
} finally {
if (!success) {
try {
to.deleteFile(dest);
} catch (Throwable t) {
}
}
}
}
}

2
lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java
View File

@ -33,7 +33,7 @@ public class TestPerFieldPostingsFormat extends BasePostingsFormatTestCase {
@Override
public void setUp() throws Exception {
super.setUp();
codec = new RandomCodec(new Random(random().nextLong()), Collections.EMPTY_SET);
codec = new RandomCodec(new Random(random().nextLong()), Collections.<String>emptySet());
}
@Override

52
lucene/core/src/test/org/apache/lucene/index/TestCrash.java
View File

@ -20,12 +20,13 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Random;
import org.apache.lucene.document.Field;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.util.LuceneTestCase;
public class TestCrash extends LuceneTestCase {
@ -67,20 +68,42 @@ public class TestCrash extends LuceneTestCase {
// before any documents were added.
IndexWriter writer = initIndex(random(), true);
MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
// We create leftover files because merging could be
// running when we crash:
dir.setAssertNoUnrefencedFilesOnClose(false);
crash(writer);
IndexReader reader = DirectoryReader.open(dir);
assertTrue(reader.numDocs() < 157);
reader.close();
// Make a new dir, copying from the crashed dir, and
// open IW on it, to confirm IW "recovers" after a
// crash:
Directory dir2 = newDirectory(dir);
dir.close();
new RandomIndexWriter(random(), dir2).close();
dir2.close();
}
public void testWriterAfterCrash() throws IOException {
// This test relies on being able to open a reader before any commit
// happened, so we must create an initial commit just to allow that, but
// before any documents were added.
System.out.println("TEST: initIndex");
IndexWriter writer = initIndex(random(), true);
System.out.println("TEST: done initIndex");
MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
// We create leftover files because merging could be
// running / store files could be open when we crash:
dir.setAssertNoUnrefencedFilesOnClose(false);
dir.setPreventDoubleWrite(false);
System.out.println("TEST: now crash");
crash(writer);
writer = initIndex(random(), dir, false);
writer.close();
@ -88,12 +111,25 @@ public class TestCrash extends LuceneTestCase {
IndexReader reader = DirectoryReader.open(dir);
assertTrue(reader.numDocs() < 314);
reader.close();
// Make a new dir, copying from the crashed dir, and
// open IW on it, to confirm IW "recovers" after a
// crash:
Directory dir2 = newDirectory(dir);
dir.close();
new RandomIndexWriter(random(), dir2).close();
dir2.close();
}
public void testCrashAfterReopen() throws IOException {
IndexWriter writer = initIndex(random(), false);
MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
// We create leftover files because merging could be
// running when we crash:
dir.setAssertNoUnrefencedFilesOnClose(false);
writer.close();
writer = initIndex(random(), dir, false);
assertEquals(314, writer.maxDoc());
@ -111,7 +147,15 @@ public class TestCrash extends LuceneTestCase {
IndexReader reader = DirectoryReader.open(dir);
assertTrue(reader.numDocs() >= 157);
reader.close();
// Make a new dir, copying from the crashed dir, and
// open IW on it, to confirm IW "recovers" after a
// crash:
Directory dir2 = newDirectory(dir);
dir.close();
new RandomIndexWriter(random(), dir2).close();
dir2.close();
}
public void testCrashAfterClose() throws IOException {

1
lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
View File

@ -39,6 +39,7 @@ import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NoSuchDirectoryException;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;

1
lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
View File

@ -38,6 +38,7 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;

15
lucene/core/src/test/org/apache/lucene/index/TestDoc.java
View File

@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.InfoStream;
@ -112,6 +113,13 @@ public class TestDoc extends LuceneTestCase {
PrintWriter out = new PrintWriter(sw, true);
Directory directory = newFSDirectory(indexDir, null);
if (directory instanceof MockDirectoryWrapper) {
// We create unreferenced files (we don't even write
// a segments file):
((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
}
IndexWriter writer = new IndexWriter(
directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
@ -147,6 +155,13 @@ public class TestDoc extends LuceneTestCase {
out = new PrintWriter(sw, true);
directory = newFSDirectory(indexDir, null);
if (directory instanceof MockDirectoryWrapper) {
// We create unreferenced files (we don't even write
// a segments file):
((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
}
writer = new IndexWriter(
directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).

3
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
View File

@ -1070,6 +1070,9 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
fail("segmentInfos failed to retry fallback to correct segments_N file");
}
reader.close();
// should remove the corrumpted segments_N
new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
dir.close();
}

11
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
View File

@ -284,6 +284,9 @@ public class TestIndexWriterWithThreads extends LuceneTestCase {
failure.clearDoFail();
writer.close(false);
}
if (VERBOSE) {
System.out.println("TEST: success=" + success);
}
if (success) {
IndexReader reader = DirectoryReader.open(dir);
@ -341,6 +344,12 @@ public class TestIndexWriterWithThreads extends LuceneTestCase {
}
@Override
public void eval(MockDirectoryWrapper dir) throws IOException {
// Since we throw exc during abort, eg when IW is
// attempting to delete files, we will leave
// leftovers:
dir.setAssertNoUnrefencedFilesOnClose(false);
if (doFail) {
StackTraceElement[] trace = new Exception().getStackTrace();
boolean sawAbortOrFlushDoc = false;
@ -405,6 +414,8 @@ public class TestIndexWriterWithThreads extends LuceneTestCase {
if ("flush".equals(trace[i].getMethodName()) && "org.apache.lucene.index.DocFieldProcessor".equals(trace[i].getClassName())) {
if (onlyOnce)
doFail = false;
//System.out.println(Thread.currentThread().getName() + ": NOW FAIL: onlyOnce=" + onlyOnce);
//new Throwable().printStackTrace(System.out);
throw new IOException("now failing on purpose");
}
}

2
lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java
View File

@ -21,10 +21,12 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.Arrays;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;

1
lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
View File

@ -25,6 +25,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

6
lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
View File

@ -39,7 +39,6 @@ public class TestRollingUpdates extends LuceneTestCase {
public void testRollingUpdates() throws Exception {
Random random = new Random(random().nextLong());
final BaseDirectoryWrapper dir = newDirectory();
dir.setCheckIndexOnClose(false); // we use a custom codec provider
final LineFileDocs docs = new LineFileDocs(random, true);
//provider.register(new MemoryCodec());
@ -130,10 +129,11 @@ public class TestRollingUpdates extends LuceneTestCase {
assertEquals(SIZE, w.numDocs());
w.close();
TestIndexWriter.assertNoUnreferencedFiles(dir, "leftover files after rolling updates");
docs.close();
_TestUtil.checkIndex(dir);
// LUCENE-4455:
SegmentInfos infos = new SegmentInfos();
infos.read(dir);

10
lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
View File

@ -17,17 +17,18 @@ package org.apache.lucene.index;
* limitations under the License.
*/
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.Random;
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThreadInterruptedException;
import org.junit.Test;
@ -313,6 +314,7 @@ public class TestSnapshotDeletionPolicy extends LuceneTestCase {
public void testRollbackToOldSnapshot() throws Exception {
int numSnapshots = 2;
Directory dir = newDirectory();
SnapshotDeletionPolicy sdp = getDeletionPolicy();
IndexWriter writer = new IndexWriter(dir, getConfig(random(), sdp));
prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
@ -325,10 +327,11 @@ public class TestSnapshotDeletionPolicy extends LuceneTestCase {
writer.deleteUnusedFiles();
assertSnapshotExists(dir, sdp, numSnapshots - 1);
writer.close();
// but 'snapshot1' files will still exist (need to release snapshot before they can be deleted).
String segFileName = sdp.getSnapshot("snapshot1").getSegmentsFileName();
assertTrue("snapshot files should exist in the directory: " + segFileName, dir.fileExists(segFileName));
dir.close();
}
@ -385,6 +388,7 @@ public class TestSnapshotDeletionPolicy extends LuceneTestCase {
@Test
public void testSnapshotLastCommitTwice() throws Exception {
Directory dir = newDirectory();
SnapshotDeletionPolicy sdp = getDeletionPolicy();
IndexWriter writer = new IndexWriter(dir, getConfig(random(), sdp));
writer.addDocument(new Document());

26
lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java
View File

@ -21,17 +21,18 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import org.apache.lucene.document.Field;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
/**
* Test class to illustrate using IndexDeletionPolicy to provide multi-level rollback capability.
@ -55,13 +56,16 @@ public class TestTransactionRollback extends LuceneTestCase {
for (Iterator<IndexCommit> iterator = commits.iterator(); iterator.hasNext();) {
IndexCommit commit = iterator.next();
Map<String,String> ud=commit.getUserData();
if (ud.size() > 0)
if (ud.get("index").endsWith(ids))
last=commit;
if (ud.size() > 0) {
if (ud.get("index").endsWith(ids)) {
last = commit;
}
}
}
if (last==null)
if (last==null) {
throw new RuntimeException("Couldn't find commit point "+id);
}
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random())).setIndexDeletionPolicy(
@ -74,13 +78,13 @@ public class TestTransactionRollback extends LuceneTestCase {
public void testRepeatedRollBacks() throws Exception {
int expectedLastRecordId=100;
int expectedLastRecordId = 100;
while (expectedLastRecordId>10) {
expectedLastRecordId -=10;
expectedLastRecordId -= 10;
rollBackLast(expectedLastRecordId);
BitSet expecteds = new BitSet(100);
expecteds.set(1,(expectedLastRecordId+1),true);
expecteds.set(1, (expectedLastRecordId+1), true);
checkExpecteds(expecteds);
}
}
@ -125,6 +129,7 @@ public class TestTransactionRollback extends LuceneTestCase {
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
//Build index, of records 1 to 100, committing after each batch of 10
IndexDeletionPolicy sdp=new KeepAllDeletionPolicy();
IndexWriter w=new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setIndexDeletionPolicy(sdp));
@ -199,6 +204,7 @@ public class TestTransactionRollback extends LuceneTestCase {
}
public void testRollbackDeletionPolicy() throws Exception {
for(int i=0;i<2;i++) {
// Unless you specify a prior commit point, rollback
// should not work:

5
lucene/core/src/test/org/apache/lucene/index/TestTransactions.java
View File

@ -215,6 +215,11 @@ public class TestTransactions extends LuceneTestCase {
dir1.setFailOnOpenInput(false);
dir2.setFailOnOpenInput(false);
// We throw exceptions in deleteFile, which creates
// leftover files:
dir1.setAssertNoUnrefencedFilesOnClose(false);
dir2.setAssertNoUnrefencedFilesOnClose(false);
initIndex(dir1);
initIndex(dir2);

58
lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java Normal file
View File

@ -0,0 +1,58 @@
package org.apache.lucene.util.junitcompat;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.Collections;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.JUnitCore;
import org.junit.runner.Result;
// LUCENE-4456: Test that we fail if there are unreferenced files
public class TestFailIfUnreferencedFiles extends WithNestedTests {
public TestFailIfUnreferencedFiles() {
super(true);
}
public static class Nested1 extends WithNestedTests.AbstractNestedTest {
public void testDummy() throws Exception {
Directory dir = newMockDirectory();
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
iw.addDocument(new Document());
iw.close();
IndexOutput output = dir.createOutput("_hello.world", IOContext.DEFAULT);
output.writeString("i am unreferenced!");
output.close();
dir.sync(Collections.singleton("_hello.world"));
dir.close();
}
}
@Test
public void testFailIfUnreferencedFiles() {
Result r = JUnitCore.runClasses(Nested1.class);
Assert.assertEquals(1, r.getFailureCount());
}
}

6
lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
View File

@ -22,6 +22,7 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@ -34,6 +35,11 @@ public class TestIndexSplitter extends LuceneTestCase {
_TestUtil.rmDir(destDir);
destDir.mkdirs();
Directory fsDir = newFSDirectory(dir);
// IndexSplitter.split makes its own commit directly with SIPC/SegmentInfos,
// so the unreferenced files are expected.
if (fsDir instanceof MockDirectoryWrapper) {
((MockDirectoryWrapper)fsDir).setAssertNoUnrefencedFilesOnClose(false);
}
LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
mergePolicy.setNoCFSRatio(1.0);

6
lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
View File

@ -402,10 +402,16 @@ public class RandomIndexWriter implements Closeable {
final int segCount = w.getSegmentCount();
if (r.nextBoolean() || segCount == 0) {
// full forceMerge
if (LuceneTestCase.VERBOSE) {
System.out.println("RIW: doRandomForceMerge(1)");
}
w.forceMerge(1);
} else {
// partial forceMerge
final int limit = _TestUtil.nextInt(r, 1, segCount);
if (LuceneTestCase.VERBOSE) {
System.out.println("RIW: doRandomForceMerge(" + limit + ")");
}
w.forceMerge(limit);
assert !doRandomForceMergeAssert || w.getSegmentCount() <= limit: "limit=" + limit + " actual=" + w.getSegmentCount();
}

100
lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
View File

@ -28,14 +28,18 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.NoDeletionPolicy;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@ -204,15 +208,17 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
// first force-close all files, so we can corrupt on windows etc.
// clone the file map, as these guys want to remove themselves on close.
Map<Closeable,Exception> m = new IdentityHashMap<Closeable,Exception>(openFileHandles);
for (Closeable f : m.keySet())
for (Closeable f : m.keySet()) {
try {
f.close();
} catch (Exception ignored) {}
}
while(it.hasNext()) {
String name = it.next();
int damage = randomState.nextInt(5);
String action = null;
if (damage == 0) {
action = "deleted";
deleteFile(name, true);
@ -544,6 +550,9 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
@Override
public synchronized void close() throws IOException {
// files that we tried to delete, but couldn't because readers were open.
// all that matters is that we tried! (they will eventually go away)
Set<String> pendingDeletions = new HashSet<String>(openFilesDeleted);
maybeYield();
if (openFiles == null) {
openFiles = new HashMap<String,Integer>();
@ -574,17 +583,91 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
}
_TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose());
// TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
if (assertNoUnreferencedFilesOnClose) {
// now look for unreferenced files:
String[] startFiles = listAll();
new IndexWriter(this, new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null)).rollback();
String[] endFiles = listAll();
// now look for unreferenced files: discount ones that we tried to delete but could not
Set<String> allFiles = new HashSet<String>(Arrays.asList(listAll()));
allFiles.removeAll(pendingDeletions);
String[] startFiles = allFiles.toArray(new String[0]);
IndexWriterConfig iwc = new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null);
iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
new IndexWriter(delegate, iwc).rollback();
String[] endFiles = delegate.listAll();
Arrays.sort(startFiles);
Arrays.sort(endFiles);
Set<String> startSet = new TreeSet<String>(Arrays.asList(startFiles));
Set<String> endSet = new TreeSet<String>(Arrays.asList(endFiles));
if (pendingDeletions.contains("segments.gen") && endSet.contains("segments.gen")) {
// this is possible if we hit an exception while writing segments.gen, we try to delete it
// and it ends out in pendingDeletions (but IFD wont remove this).
startSet.add("segments.gen");
if (LuceneTestCase.VERBOSE) {
System.out.println("MDW: Unreferenced check: Ignoring segments.gen that we could not delete.");
}
}
// its possible we cannot delete the segments_N on windows if someone has it open and
// maybe other files too, depending on timing. normally someone on windows wouldnt have
// an issue (IFD would nuke this stuff eventually), but we pass NoDeletionPolicy...
for (String file : pendingDeletions) {
if (file.startsWith("segments") && !file.equals("segments.gen") && endSet.contains(file)) {
startSet.add(file);
if (LuceneTestCase.VERBOSE) {
System.out.println("MDW: Unreferenced check: Ignoring segments file: " + file + " that we could not delete.");
}
try {
SegmentInfos sis = new SegmentInfos();
sis.read(delegate, file);
Set<String> ghosts = new HashSet<String>(sis.files(delegate, false));
for (String s : ghosts) {
if (endSet.contains(s) && !startSet.contains(s)) {
assert pendingDeletions.contains(s);
if (LuceneTestCase.VERBOSE) {
System.out.println("MDW: Unreferenced check: Ignoring referenced file: " + s + " " +
"from " + file + " that we could not delete.");
}
startSet.add(s);
}
}
} catch (Throwable ignore) {}
}
}
startFiles = startSet.toArray(new String[0]);
endFiles = endSet.toArray(new String[0]);
if (!Arrays.equals(startFiles, endFiles)) {
assert false : "unreferenced files: before delete:\n " + Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles);
StringBuilder sb = new StringBuilder();
List<String> removed = new ArrayList<String>();
for(String fileName : startFiles) {
if (!endSet.contains(fileName)) {
removed.add(fileName);
}
}
List<String> added = new ArrayList<String>();
for(String fileName : endFiles) {
if (!startSet.contains(fileName)) {
added.add(fileName);
}
}
String extras;
if (removed.size() != 0) {
extras = "\n\nThese files were removed: " + removed;
} else {
extras = "";
}
if (added.size() != 0) {
extras += "\n\nThese files were added (waaaaaaaaaat!): " + added;
}
if (pendingDeletions.size() != 0) {
extras += "\n\nThese files we had previously tried to delete, but couldn't: " + pendingDeletions;
}
assert false : "unreferenced files: before delete:\n " + Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles) + extras;
}
DirectoryReader ir1 = DirectoryReader.open(this);
@ -607,7 +690,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
if (v != null) {
if (v.intValue() == 1) {
openFiles.remove(name);
openFilesDeleted.remove(name);
} else {
v = Integer.valueOf(v.intValue()-1);
openFiles.put(name, v);

1
lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -948,6 +948,7 @@ public abstract class LuceneTestCase extends Assert {
if (rarely(random)) {
directory = new NRTCachingDirectory(directory, random.nextDouble(), random.nextDouble());
}
if (bare) {
BaseDirectoryWrapper base = new BaseDirectoryWrapper(directory);
closeAfterSuite(new CloseableDirectory(base, suiteFailureMarker));