Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5684: best effort detection of (illegal) reopen after index was blown away 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1597050 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2014-05-23 09:51:12 +00:00
parent 8a1587d6d8
commit d3bee7cb7f
3 changed files with 220 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lucene/CHANGES.txt
View File

@ -214,6 +214,12 @@ Bug fixes
doc-values updates. Now doc-values field updates are written in separate file
per field. (Shai Erera, Robert Muir)
* LUCENE-5684: Make best effort to detect invalid usage of Lucene,
when IndexReader is reopened after all files in its index were
removed and recreated by the application (the proper way to do
this is IndexWriter.deleteAll, or opening an IndexWriter with
OpenMode.CREATE) (Mike McCandless)
Test Framework
* LUCENE-5622: Fail tests if they print over the given limit of bytes to

89
lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
View File

@ -54,7 +54,6 @@ final class StandardDirectoryReader extends DirectoryReader {
sis.read(directory, segmentFileName);
final SegmentReader[] readers = new SegmentReader[sis.size()];
for (int i = sis.size()-1; i >= 0; i--) {
IOException prior = null;
boolean success = false;
try {
readers[i] = new SegmentReader(sis.info(i), IOContext.READ);
@ -145,84 +144,84 @@ final class StandardDirectoryReader extends DirectoryReader {
SegmentReader[] newReaders = new SegmentReader[infos.size()];
// remember which readers are shared between the old and the re-opened
// DirectoryReader - we have to incRef those readers
boolean[] readerShared = new boolean[infos.size()];
for (int i = infos.size() - 1; i>=0; i--) {
SegmentCommitInfo commitInfo = infos.info(i);
// find SegmentReader for this segment
Integer oldReaderIndex = segmentReaders.get(infos.info(i).info.name);
Integer oldReaderIndex = segmentReaders.get(commitInfo.info.name);
SegmentReader oldReader;
if (oldReaderIndex == null) {
// this is a new segment, no old SegmentReader can be reused
newReaders[i] = null;
oldReader = null;
} else {
// there is an old reader for this segment - we'll try to reopen it
newReaders[i] = (SegmentReader) oldReaders.get(oldReaderIndex.intValue());
oldReader = (SegmentReader) oldReaders.get(oldReaderIndex.intValue());
}
boolean success = false;
Throwable prior = null;
try {
SegmentReader newReader;
if (newReaders[i] == null || infos.info(i).info.getUseCompoundFile() != newReaders[i].getSegmentInfo().info.getUseCompoundFile()) {
if (oldReader == null || commitInfo.info.getUseCompoundFile() != oldReader.getSegmentInfo().info.getUseCompoundFile()) {
// this is a new reader; in case we hit an exception we can close it safely
newReader = new SegmentReader(infos.info(i), IOContext.READ);
readerShared[i] = false;
// this is a new reader; in case we hit an exception we can decRef it safely
newReader = new SegmentReader(commitInfo, IOContext.READ);
newReaders[i] = newReader;
} else {
if (newReaders[i].getSegmentInfo().getDelGen() == infos.info(i).getDelGen()
&& newReaders[i].getSegmentInfo().getFieldInfosGen() == infos.info(i).getFieldInfosGen()) {
if (oldReader.getSegmentInfo().getDelGen() == commitInfo.getDelGen()
&& oldReader.getSegmentInfo().getFieldInfosGen() == commitInfo.getFieldInfosGen()) {
// No change; this reader will be shared between
// the old and the new one, so we must incRef
// it:
readerShared[i] = true;
newReaders[i].incRef();
oldReader.incRef();
newReaders[i] = oldReader;
} else {
// there are changes to the reader, either liveDocs or DV updates
readerShared[i] = false;
// Steal the ref returned by SegmentReader ctor:
assert infos.info(i).info.dir == newReaders[i].getSegmentInfo().info.dir;
assert infos.info(i).hasDeletions() || infos.info(i).hasFieldUpdates();
if (newReaders[i].getSegmentInfo().getDelGen() == infos.info(i).getDelGen()) {
assert commitInfo.info.dir == oldReader.getSegmentInfo().info.dir;
// Make a best effort to detect when the app illegally "rm -rf" their
// index while a reader was open, and then called openIfChanged:
boolean illegalDocCountChange = commitInfo.info.getDocCount() != oldReader.getSegmentInfo().info.getDocCount();
boolean hasNeitherDeletionsNorUpdates = commitInfo.hasDeletions()== false && commitInfo.hasFieldUpdates() == false;
boolean deletesWereLost = commitInfo.getDelGen() == -1 && oldReader.getSegmentInfo().getDelGen() != -1;
if (illegalDocCountChange || hasNeitherDeletionsNorUpdates || deletesWereLost) {
throw new IllegalStateException("same segment " + commitInfo.info.name + " has invalid changes; likely you are re-opening a reader after illegally removing index files yourself and building a new index in their place. Use IndexWriter.deleteAll or OpenMode.CREATE instead");
}
if (oldReader.getSegmentInfo().getDelGen() == commitInfo.getDelGen()) {
// only DV updates
newReaders[i] = new SegmentReader(infos.info(i), newReaders[i], newReaders[i].getLiveDocs(), newReaders[i].numDocs());
newReaders[i] = new SegmentReader(commitInfo, oldReader, oldReader.getLiveDocs(), oldReader.numDocs());
} else {
// both DV and liveDocs have changed
newReaders[i] = new SegmentReader(infos.info(i), newReaders[i]);
newReaders[i] = new SegmentReader(commitInfo, oldReader);
}
}
}
success = true;
} catch (Throwable ex) {
prior = ex;
} finally {
if (!success) {
for (i++; i < infos.size(); i++) {
if (newReaders[i] != null) {
try {
if (!readerShared[i]) {
// this is a new subReader that is not used by the old one,
// we can close it
newReaders[i].close();
} else {
// this subReader is also used by the old reader, so instead
// closing we must decRef it
newReaders[i].decRef();
}
} catch (Throwable t) {
if (prior == null) prior = t;
}
}
}
decRefWhileHandlingException(newReaders);
}
// throw the first exception
IOUtils.reThrow(prior);
}
}
return new StandardDirectoryReader(directory, newReaders, null, infos, false);
}
// TODO: move somewhere shared if it's useful elsewhere
private static void decRefWhileHandlingException(SegmentReader[] readers) {
for(SegmentReader reader : readers) {
if (reader != null) {
try {
reader.decRef();
} catch (Throwable t) {
// Ignore so we keep throwing original exception
}
}
}
}
@Override
public String toString() {
final StringBuilder buffer = new StringBuilder();

170
lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
View File

@ -28,15 +28,20 @@ import java.util.Random;
import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@ -618,4 +623,169 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
r2.close();
dir.close();
}
public void testOverDecRefDuringReopen() throws Exception {
MockDirectoryWrapper dir = newMockDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwc.setCodec(Codec.forName("Lucene46"));
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
doc.add(newStringField("id", "id", Field.Store.NO));
w.addDocument(doc);
doc = new Document();
doc.add(newStringField("id", "id2", Field.Store.NO));
w.addDocument(doc);
w.commit();
// Open reader w/ one segment w/ 2 docs:
DirectoryReader r = DirectoryReader.open(dir);
// Delete 1 doc from the segment:
//System.out.println("TEST: now delete");
w.deleteDocuments(new Term("id", "id"));
//System.out.println("TEST: now commit");
w.commit();
// Fail when reopen tries to open the live docs file:
dir.failOn(new MockDirectoryWrapper.Failure() {
int failCount;
@Override
public void eval(MockDirectoryWrapper dir) throws IOException {
// Need to throw exc three times so the logic in
// SegmentInfos.FindSegmentsFile "really believes" us:
if (failCount >= 3) {
return;
}
//System.out.println("failOn: ");
//new Throwable().printStackTrace(System.out);
StackTraceElement[] trace = new Exception().getStackTrace();
for (int i = 0; i < trace.length; i++) {
if ("readLiveDocs".equals(trace[i].getMethodName())) {
if (VERBOSE) {
System.out.println("TEST: now fail; exc:");
new Throwable().printStackTrace(System.out);
}
failCount++;
throw new FakeIOException();
}
}
}
});
// Now reopen:
//System.out.println("TEST: now reopen");
try {
IndexReader r2 = DirectoryReader.openIfChanged(r);
//System.out.println("got " + r2);
fail("didn't hit exception");
} catch (FakeIOException fio) {
// expected
}
IndexSearcher s = newSearcher(r);
assertEquals(1, s.search(new TermQuery(new Term("id", "id")), 1).totalHits);
r.close();
w.close();
dir.close();
}
public void testNPEAfterInvalidReindex1() throws Exception {
Directory dir = new RAMDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
Document doc = new Document();
doc.add(newStringField("id", "id", Field.Store.NO));
w.addDocument(doc);
doc = new Document();
doc.add(newStringField("id", "id2", Field.Store.NO));
w.addDocument(doc);
w.deleteDocuments(new Term("id", "id"));
w.commit();
w.close();
// Open reader w/ one segment w/ 2 docs, 1 deleted:
DirectoryReader r = DirectoryReader.open(dir);
// Blow away the index:
for(String fileName : dir.listAll()) {
dir.deleteFile(fileName);
}
w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
doc = new Document();
doc.add(newStringField("id", "id", Field.Store.NO));
doc.add(new NumericDocValuesField("ndv", 13));
w.addDocument(doc);
doc = new Document();
doc.add(newStringField("id", "id2", Field.Store.NO));
w.addDocument(doc);
w.commit();
doc = new Document();
doc.add(newStringField("id", "id2", Field.Store.NO));
w.addDocument(doc);
w.updateNumericDocValue(new Term("id", "id"), "ndv", 17L);
w.commit();
w.close();
try {
DirectoryReader.openIfChanged(r);
fail("didn't hit expected exception");
} catch (IllegalStateException ise) {
// expected
}
r.close();
w.close();
dir.close();
}
public void testNPEAfterInvalidReindex2() throws Exception {
Directory dir = new RAMDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
Document doc = new Document();
doc.add(newStringField("id", "id", Field.Store.NO));
w.addDocument(doc);
doc = new Document();
doc.add(newStringField("id", "id2", Field.Store.NO));
w.addDocument(doc);
w.deleteDocuments(new Term("id", "id"));
w.commit();
w.close();
// Open reader w/ one segment w/ 2 docs, 1 deleted:
DirectoryReader r = DirectoryReader.open(dir);
// Blow away the index:
for(String fileName : dir.listAll()) {
dir.deleteFile(fileName);
}
w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
doc = new Document();
doc.add(newStringField("id", "id", Field.Store.NO));
doc.add(new NumericDocValuesField("ndv", 13));
w.addDocument(doc);
w.commit();
doc = new Document();
doc.add(newStringField("id", "id2", Field.Store.NO));
w.addDocument(doc);
w.commit();
w.close();
try {
DirectoryReader.openIfChanged(r);
fail("didn't hit expected exception");
} catch (IllegalStateException ise) {
// expected
}
r.close();
w.close();
dir.close();
}
}