LUCENE-3606: Reactivate parts of the reopen tests

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3606@1210037 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Uwe Schindler 2011-12-04 00:25:35 +00:00
parent 498cba2fac
commit 04f8f2850a
1 changed files with 597 additions and 76 deletions

View File

@ -49,6 +49,99 @@ import org.apache.lucene.util._TestUtil;
public class TestIndexReaderReopen extends LuceneTestCase {
public void testReopen() throws Exception {
final Directory dir1 = newDirectory();
createIndex(random, dir1, false);
performDefaultTests(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir1);
}
@Override
protected IndexReader openReader() throws IOException {
return IndexReader.open(dir1);
}
});
dir1.close();
final Directory dir2 = newDirectory();
createIndex(random, dir2, true);
performDefaultTests(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir2);
}
@Override
protected IndexReader openReader() throws IOException {
return IndexReader.open(dir2);
}
});
dir2.close();
}
public void testParallelReaderReopen() throws Exception {
final Directory dir1 = newDirectory();
createIndex(random, dir1, true);
final Directory dir2 = newDirectory();
createIndex(random, dir2, true);
performDefaultTests(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir1);
TestIndexReaderReopen.modifyIndex(i, dir2);
}
@Override
protected IndexReader openReader() throws IOException {
ParallelReader pr = new ParallelReader();
pr.add(IndexReader.open(dir1));
pr.add(IndexReader.open(dir2));
return pr;
}
});
dir1.close();
dir2.close();
final Directory dir3 = newDirectory();
createIndex(random, dir3, true);
final Directory dir4 = newDirectory();
createIndex(random, dir4, true);
performTestsWithExceptionInReopen(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir3);
TestIndexReaderReopen.modifyIndex(i, dir4);
}
@Override
protected IndexReader openReader() throws IOException {
ParallelReader pr = new ParallelReader();
pr.add(IndexReader.open(dir3));
pr.add(IndexReader.open(dir4));
// Does not implement reopen, so
// hits exception:
pr.add(new FilterIndexReader(IndexReader.open(dir3)));
return pr;
}
});
dir3.close();
dir4.close();
}
// LUCENE-1228: IndexWriter.commit() does not update the index version
// populate an index in iterations.
// at the end of every iteration, commit the index and reopen/recreate the reader.
@ -70,7 +163,7 @@ public class TestIndexReaderReopen extends LuceneTestCase {
TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(
OpenMode.CREATE).setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(newLogMergePolicy()));
iwriter.commit();
IndexReader reader = IndexReader.open(dir, false);
IndexReader reader = IndexReader.open(dir);
try {
int M = 3;
FieldType customType = new FieldType(TextField.TYPE_STORED);
@ -107,7 +200,7 @@ public class TestIndexReaderReopen extends LuceneTestCase {
} else {
// recreate
reader.close();
reader = IndexReader.open(dir, false);
reader = IndexReader.open(dir);
}
}
} finally {
@ -116,6 +209,469 @@ public class TestIndexReaderReopen extends LuceneTestCase {
}
}
public void testMultiReaderReopen() throws Exception {
final Directory dir1 = newDirectory();
createIndex(random, dir1, true);
final Directory dir2 = newDirectory();
createIndex(random, dir2, true);
performDefaultTests(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir1);
TestIndexReaderReopen.modifyIndex(i, dir2);
}
@Override
protected IndexReader openReader() throws IOException {
return new MultiReader(IndexReader.open(dir1),
IndexReader.open(dir2));
}
});
dir1.close();
dir2.close();
final Directory dir3 = newDirectory();
createIndex(random, dir3, true);
final Directory dir4 = newDirectory();
createIndex(random, dir4, true);
performTestsWithExceptionInReopen(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir3);
TestIndexReaderReopen.modifyIndex(i, dir4);
}
@Override
protected IndexReader openReader() throws IOException {
return new MultiReader(IndexReader.open(dir3),
IndexReader.open(dir4),
// Does not implement reopen, so
// hits exception:
new FilterIndexReader(IndexReader.open(dir3)));
}
});
dir3.close();
dir4.close();
}
public void testMixedReaders() throws Exception {
final Directory dir1 = newDirectory();
createIndex(random, dir1, true);
final Directory dir2 = newDirectory();
createIndex(random, dir2, true);
final Directory dir3 = newDirectory();
createIndex(random, dir3, false);
final Directory dir4 = newDirectory();
createIndex(random, dir4, true);
final Directory dir5 = newDirectory();
createIndex(random, dir5, false);
performDefaultTests(new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
TestIndexReaderReopen.modifyIndex(i, dir4);
TestIndexReaderReopen.modifyIndex(i, dir5);
}
@Override
protected IndexReader openReader() throws IOException {
MultiReader mr1 = new MultiReader(IndexReader.open(dir1), IndexReader.open(dir2));
MultiReader mr2 = new MultiReader(IndexReader.open(dir3), IndexReader.open(dir4));
return new MultiReader(mr1, mr2, IndexReader.open(dir5));
}
});
dir1.close();
dir2.close();
dir3.close();
dir4.close();
dir5.close();
}
private void performDefaultTests(TestReopen test) throws Exception {
IndexReader index1 = test.openReader();
IndexReader index2 = test.openReader();
TestIndexReader.assertIndexEquals(index1, index2);
// verify that reopen() does not return a new reader instance
// in case the index has no changes
ReaderCouple couple = refreshReader(index2, false);
assertTrue(couple.refreshedReader == index2);
couple = refreshReader(index2, test, 0, true);
index1.close();
index1 = couple.newReader;
IndexReader index2_refreshed = couple.refreshedReader;
index2.close();
// test if refreshed reader and newly opened reader return equal results
TestIndexReader.assertIndexEquals(index1, index2_refreshed);
index2_refreshed.close();
assertReaderClosed(index2, true, true);
assertReaderClosed(index2_refreshed, true, true);
index2 = test.openReader();
for (int i = 1; i < 4; i++) {
index1.close();
couple = refreshReader(index2, test, i, true);
// refresh IndexReader
index2.close();
index2 = couple.refreshedReader;
index1 = couple.newReader;
TestIndexReader.assertIndexEquals(index1, index2);
}
index1.close();
index2.close();
assertReaderClosed(index1, true, true);
assertReaderClosed(index2, true, true);
}
public void testReferenceCountingMultiReader() throws IOException {
for (int mode = 0; mode <=1; mode++) {
Directory dir1 = newDirectory();
createIndex(random, dir1, false);
Directory dir2 = newDirectory();
createIndex(random, dir2, true);
IndexReader reader1 = IndexReader.open(dir1);
assertRefCountEquals(1, reader1);
IndexReader initReader2 = IndexReader.open(dir2);
IndexReader multiReader1 = new MultiReader(new IndexReader[] {reader1, initReader2}, (mode == 0));
modifyIndex(0, dir2);
assertRefCountEquals(1 + mode, reader1);
IndexReader multiReader2 = IndexReader.openIfChanged(multiReader1);
assertNotNull(multiReader2);
// index1 hasn't changed, so multiReader2 should share reader1 now with multiReader1
assertRefCountEquals(2 + mode, reader1);
modifyIndex(0, dir1);
IndexReader reader2 = IndexReader.openIfChanged(reader1);
assertNotNull(reader2);
assertNull(IndexReader.openIfChanged(reader2));
assertRefCountEquals(2 + mode, reader1);
if (mode == 1) {
initReader2.close();
}
modifyIndex(1, dir1);
IndexReader reader3 = IndexReader.openIfChanged(reader2);
assertNotNull(reader3);
assertRefCountEquals(2 + mode, reader1);
assertRefCountEquals(1, reader2);
multiReader1.close();
assertRefCountEquals(1 + mode, reader1);
multiReader1.close();
assertRefCountEquals(1 + mode, reader1);
if (mode == 1) {
initReader2.close();
}
reader1.close();
assertRefCountEquals(1, reader1);
multiReader2.close();
assertRefCountEquals(0, reader1);
multiReader2.close();
assertRefCountEquals(0, reader1);
reader3.close();
assertRefCountEquals(0, reader1);
assertReaderClosed(reader1, true, false);
reader2.close();
assertRefCountEquals(0, reader1);
assertReaderClosed(reader1, true, false);
reader2.close();
assertRefCountEquals(0, reader1);
reader3.close();
assertRefCountEquals(0, reader1);
assertReaderClosed(reader1, true, true);
dir1.close();
dir2.close();
}
}
private void performTestsWithExceptionInReopen(TestReopen test) throws Exception {
IndexReader index1 = test.openReader();
IndexReader index2 = test.openReader();
TestIndexReader.assertIndexEquals(index1, index2);
try {
refreshReader(index1, test, 0, true);
fail("Expected exception not thrown.");
} catch (Exception e) {
// expected exception
}
// index2 should still be usable and unaffected by the failed reopen() call
TestIndexReader.assertIndexEquals(index1, index2);
index1.close();
index2.close();
}
public void testThreadSafety() throws Exception {
final Directory dir = newDirectory();
// NOTE: this also controls the number of threads!
final int n = _TestUtil.nextInt(random, 20, 40);
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random)));
for (int i = 0; i < n; i++) {
writer.addDocument(createDocument(i, 3));
}
writer.forceMerge(1);
writer.close();
final TestReopen test = new TestReopen() {
@Override
protected void modifyIndex(int i) throws IOException {
IndexWriter modifier = new IndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random)));
modifier.addDocument(createDocument(n + i, 6));
modifier.close();
}
@Override
protected IndexReader openReader() throws IOException {
return IndexReader.open(dir);
}
};
final List<ReaderCouple> readers = Collections.synchronizedList(new ArrayList<ReaderCouple>());
IndexReader firstReader = IndexReader.open(dir);
IndexReader reader = firstReader;
final Random rnd = random;
ReaderThread[] threads = new ReaderThread[n];
final Set<IndexReader> readersToClose = Collections.synchronizedSet(new HashSet<IndexReader>());
for (int i = 0; i < n; i++) {
if (i % 2 == 0) {
IndexReader refreshed = IndexReader.openIfChanged(reader);
if (refreshed != null) {
readersToClose.add(reader);
reader = refreshed;
}
}
final IndexReader r = reader;
final int index = i;
ReaderThreadTask task;
if (i < 4 || (i >=10 && i < 14) || i > 18) {
task = new ReaderThreadTask() {
@Override
public void run() throws Exception {
while (!stopped) {
if (index % 2 == 0) {
// refresh reader synchronized
ReaderCouple c = (refreshReader(r, test, index, true));
readersToClose.add(c.newReader);
readersToClose.add(c.refreshedReader);
readers.add(c);
// prevent too many readers
break;
} else {
// not synchronized
IndexReader refreshed = IndexReader.openIfChanged(r);
if (refreshed == null) {
refreshed = r;
}
IndexSearcher searcher = newSearcher(refreshed);
ScoreDoc[] hits = searcher.search(
new TermQuery(new Term("field1", "a" + rnd.nextInt(refreshed.maxDoc()))),
null, 1000).scoreDocs;
if (hits.length > 0) {
searcher.doc(hits[0].doc);
}
searcher.close();
if (refreshed != r) {
refreshed.close();
}
}
synchronized(this) {
wait(_TestUtil.nextInt(random, 1, 100));
}
}
}
};
} else {
task = new ReaderThreadTask() {
@Override
public void run() throws Exception {
while (!stopped) {
int numReaders = readers.size();
if (numReaders > 0) {
ReaderCouple c = readers.get(rnd.nextInt(numReaders));
TestIndexReader.assertIndexEquals(c.newReader, c.refreshedReader);
}
synchronized(this) {
wait(_TestUtil.nextInt(random, 1, 100));
}
}
}
};
}
threads[i] = new ReaderThread(task);
threads[i].start();
}
synchronized(this) {
wait(1000);
}
for (int i = 0; i < n; i++) {
if (threads[i] != null) {
threads[i].stopThread();
}
}
for (int i = 0; i < n; i++) {
if (threads[i] != null) {
threads[i].join();
if (threads[i].error != null) {
String msg = "Error occurred in thread " + threads[i].getName() + ":\n" + threads[i].error.getMessage();
fail(msg);
}
}
}
for (final IndexReader readerToClose : readersToClose) {
readerToClose.close();
}
firstReader.close();
reader.close();
for (final IndexReader readerToClose : readersToClose) {
assertReaderClosed(readerToClose, true, true);
}
assertReaderClosed(reader, true, true);
assertReaderClosed(firstReader, true, true);
dir.close();
}
private static class ReaderCouple {
ReaderCouple(IndexReader r1, IndexReader r2) {
newReader = r1;
refreshedReader = r2;
}
IndexReader newReader;
IndexReader refreshedReader;
}
private abstract static class ReaderThreadTask {
protected volatile boolean stopped;
public void stop() {
this.stopped = true;
}
public abstract void run() throws Exception;
}
private static class ReaderThread extends Thread {
private ReaderThreadTask task;
private Throwable error;
ReaderThread(ReaderThreadTask task) {
this.task = task;
}
public void stopThread() {
this.task.stop();
}
@Override
public void run() {
try {
this.task.run();
} catch (Throwable r) {
r.printStackTrace(System.out);
this.error = r;
}
}
}
private Object createReaderMutex = new Object();
private ReaderCouple refreshReader(IndexReader reader, boolean hasChanges) throws IOException {
return refreshReader(reader, null, -1, hasChanges);
}
ReaderCouple refreshReader(IndexReader reader, TestReopen test, int modify, boolean hasChanges) throws IOException {
synchronized (createReaderMutex) {
IndexReader r = null;
if (test != null) {
test.modifyIndex(modify);
r = test.openReader();
}
IndexReader refreshed = null;
try {
refreshed = IndexReader.openIfChanged(reader);
if (refreshed == null) {
refreshed = reader;
}
} finally {
if (refreshed == null && r != null) {
// Hit exception -- close opened reader
r.close();
}
}
if (hasChanges) {
if (refreshed == reader) {
fail("No new IndexReader instance created during refresh.");
}
} else {
if (refreshed != reader) {
fail("New IndexReader instance created during refresh even though index had no changes.");
}
}
return new ReaderCouple(r, refreshed);
}
}
public static void createIndex(Random random, Directory dir, boolean multiSegment) throws IOException {
IndexWriter.unlock(dir);
IndexWriter w = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(random,
@ -135,7 +691,7 @@ public class TestIndexReaderReopen extends LuceneTestCase {
w.close();
IndexReader r = IndexReader.open(dir, false);
IndexReader r = IndexReader.open(dir);
if (multiSegment) {
assertTrue(r.getSequentialSubReaders().length > 1);
} else {
@ -165,6 +721,42 @@ public class TestIndexReaderReopen extends LuceneTestCase {
return doc;
}
static void modifyIndex(int i, Directory dir) throws IOException {
switch (i) {
case 0: {
if (VERBOSE) {
System.out.println("TEST: modify index");
}
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
w.deleteDocuments(new Term("field2", "a11"));
w.deleteDocuments(new Term("field2", "b30"));
w.close();
break;
}
case 1: {
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
w.forceMerge(1);
w.close();
break;
}
case 2: {
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
w.addDocument(createDocument(101, 4));
w.forceMerge(1);
w.addDocument(createDocument(102, 4));
w.addDocument(createDocument(103, 4));
w.close();
break;
}
case 3: {
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
w.addDocument(createDocument(101, 4));
w.close();
break;
}
}
}
private void assertReaderClosed(IndexReader reader, boolean checkSubReaders, boolean checkNormsClosed) {
assertEquals(0, reader.getRefCount());
@ -216,28 +808,7 @@ public class TestIndexReaderReopen extends LuceneTestCase {
private abstract static class TestReopen {
protected abstract IndexReader openReader() throws IOException;
}
public void testCloseOrig() throws Throwable {
Directory dir = newDirectory();
createIndex(random, dir, false);
IndexReader r1 = IndexReader.open(dir, false);
IndexReader r2 = IndexReader.open(dir, false);
r2.deleteDocument(0);
r2.close();
IndexReader r3 = IndexReader.openIfChanged(r1);
assertNotNull(r3);
assertTrue(r1 != r3);
r1.close();
try {
r1.document(2);
fail("did not hit exception");
} catch (AlreadyClosedException ace) {
// expected
}
r3.close();
dir.close();
protected abstract void modifyIndex(int i) throws IOException;
}
private static class KeepAllCommits implements IndexDeletionPolicy {
@ -272,7 +843,7 @@ public class TestIndexReaderReopen extends LuceneTestCase {
}
writer.close();
IndexReader r = IndexReader.open(dir, false);
IndexReader r = IndexReader.open(dir);
assertEquals(0, r.numDocs());
Collection<IndexCommit> commits = IndexReader.listCommits(dir);
@ -308,54 +879,4 @@ public class TestIndexReaderReopen extends LuceneTestCase {
r.close();
dir.close();
}
// LUCENE-1579: Make sure all SegmentReaders are new when
// reopen switches readOnly
public void testReopenChangeReadonly() throws Exception {
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setMaxBufferedDocs(-1).
setMergePolicy(newLogMergePolicy(10))
);
Document doc = new Document();
doc.add(newField("number", "17", StringField.TYPE_UNSTORED));
writer.addDocument(doc);
writer.commit();
// Open reader1
IndexReader r = IndexReader.open(dir, false);
assertTrue(r instanceof DirectoryReader);
IndexReader r1 = getOnlySegmentReader(r);
final int[] ints = FieldCache.DEFAULT.getInts(r1, "number", false);
assertEquals(1, ints.length);
assertEquals(17, ints[0]);
// Reopen to readonly w/ no chnages
IndexReader r3 = IndexReader.openIfChanged(r, true);
assertNotNull(r3);
assertTrue(((DirectoryReader) r3).readOnly);
r3.close();
// Add new segment
writer.addDocument(doc);
writer.commit();
// Reopen reader1 --> reader2
IndexReader r2 = IndexReader.openIfChanged(r, true);
assertNotNull(r2);
r.close();
assertTrue(((DirectoryReader) r2).readOnly);
IndexReader[] subs = r2.getSequentialSubReaders();
final int[] ints2 = FieldCache.DEFAULT.getInts(subs[0], "number", false);
r2.close();
assertTrue(((SegmentReader) subs[0]).readOnly);
assertTrue(((SegmentReader) subs[1]).readOnly);
assertTrue(ints == ints2);
writer.close();
dir.close();
}
}