Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix tests to survive nightly runs with many documents

This commit is contained in:
Simon Willnauer 2020-04-29 22:11:42 +02:00
parent 5e6d91eec0
commit 207d240ae2
5 changed files with 59 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/core/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java
View File

@ -36,9 +36,9 @@ public class TestNRTReaderWithThreads extends LuceneTestCase {
}
IndexWriter writer = new IndexWriter(
mainDir,
newIndexWriterConfig(new MockAnalyzer(random()))
ensureSaneIWCOnNightly(newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(10)
.setMergePolicy(newLogMergePolicy(false,2))
.setMergePolicy(newLogMergePolicy(false,2)))
);
IndexReader reader = writer.getReader(); // start pooling readers
reader.close();

4
lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
View File

@ -556,7 +556,7 @@ public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase {
AtomicReference<IndexWriter> writerRef = new AtomicReference<>();
final MockAnalyzer analyzer = new MockAnalyzer(random());
analyzer.setMaxTokenLength(IndexWriter.MAX_TERM_LENGTH);
writerRef.set(new IndexWriter(dir, ensureSaneIWCOnNigtly(newIndexWriterConfig(analyzer))));
writerRef.set(new IndexWriter(dir, ensureSaneIWCOnNightly(newIndexWriterConfig(analyzer))));
AtomicReference<SearcherManager> mgrRef = new AtomicReference<>();
mgrRef.set(new SearcherManager(writerRef.get(), null));
@ -580,7 +580,7 @@ public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase {
} else {
w.rollback();
}
writerRef.set(new IndexWriter(dir, ensureSaneIWCOnNigtly(newIndexWriterConfig(analyzer))));
writerRef.set(new IndexWriter(dir, ensureSaneIWCOnNightly(newIndexWriterConfig(analyzer))));
}
}
if (VERBOSE) {

54
lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
View File

@ -53,6 +53,7 @@ public class RandomIndexWriter implements Closeable {
private boolean getReaderCalled;
private final Analyzer analyzer; // only if WE created it (then we close it)
private final double softDeletesRatio;
private final LiveIndexWriterConfig config;
/** Returns an indexwriter that randomly mixes up thread scheduling (by yielding at test points) */
public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException {
@ -131,6 +132,7 @@ public class RandomIndexWriter implements Closeable {
softDeletesRatio = 0d;
}
w = mockIndexWriter(dir, c, r);
config = w.getConfig();
flushAt = TestUtil.nextInt(r, 10, 1000);
if (closeAnalyzer) {
analyzer = w.getAnalyzer();
@ -151,7 +153,7 @@ public class RandomIndexWriter implements Closeable {
* @see IndexWriter#addDocument(Iterable)
*/
public <T extends IndexableField> long addDocument(final Iterable<T> doc) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
long seqNo;
if (r.nextInt(5) == 3) {
// TODO: maybe, we should simply buffer up added docs
@ -197,7 +199,7 @@ public class RandomIndexWriter implements Closeable {
}
private void maybeFlushOrCommit() throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
if (docCount++ == flushAt) {
if (r.nextBoolean()) {
flushAllBuffersSequentially();
@ -234,17 +236,17 @@ public class RandomIndexWriter implements Closeable {
}
public long addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
long seqNo = w.addDocuments(docs);
maybeFlushOrCommit();
return seqNo;
}
public long updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
long seqNo;
if (useSoftDeletes()) {
seqNo = w.softUpdateDocuments(delTerm, docs, new NumericDocValuesField(w.getConfig().getSoftDeletesField(), 1));
seqNo = w.softUpdateDocuments(delTerm, docs, new NumericDocValuesField(config.getSoftDeletesField(), 1));
} else {
seqNo = w.updateDocuments(delTerm, docs);
}
@ -261,13 +263,13 @@ public class RandomIndexWriter implements Closeable {
* @see IndexWriter#updateDocument(Term, Iterable)
*/
public <T extends IndexableField> long updateDocument(Term t, final Iterable<T> doc) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
final long seqNo;
if (useSoftDeletes()) {
if (r.nextInt(5) == 3) {
seqNo = w.softUpdateDocuments(t, Arrays.asList(doc), new NumericDocValuesField(w.getConfig().getSoftDeletesField(), 1));
seqNo = w.softUpdateDocuments(t, Arrays.asList(doc), new NumericDocValuesField(config.getSoftDeletesField(), 1));
} else {
seqNo = w.softUpdateDocument(t, doc, new NumericDocValuesField(w.getConfig().getSoftDeletesField(), 1));
seqNo = w.softUpdateDocument(t, doc, new NumericDocValuesField(config.getSoftDeletesField(), 1));
}
} else {
if (r.nextInt(5) == 3) {
@ -282,37 +284,37 @@ public class RandomIndexWriter implements Closeable {
}
public long addIndexes(Directory... dirs) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
return w.addIndexes(dirs);
}
public long addIndexes(CodecReader... readers) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
return w.addIndexes(readers);
}
public long updateNumericDocValue(Term term, String field, Long value) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
return w.updateNumericDocValue(term, field, value);
}
public long updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
return w.updateBinaryDocValue(term, field, value);
}
public long updateDocValues(Term term, Field... updates) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
return w.updateDocValues(term, updates);
}
public long deleteDocuments(Term term) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
return w.deleteDocuments(term);
}
public long deleteDocuments(Query q) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
return w.deleteDocuments(q);
}
@ -321,7 +323,7 @@ public class RandomIndexWriter implements Closeable {
}
public long commit(boolean flushConcurrently) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
if (flushConcurrently) {
List<Throwable> throwableList = new CopyOnWriteArrayList<>();
Thread thread = new Thread(() -> {
@ -372,7 +374,7 @@ public class RandomIndexWriter implements Closeable {
}
public DirectoryReader getReader() throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
return getReader(true, false);
}
@ -380,12 +382,12 @@ public class RandomIndexWriter implements Closeable {
private boolean doRandomForceMergeAssert;
public void forceMergeDeletes(boolean doWait) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
w.forceMergeDeletes(doWait);
}
public void forceMergeDeletes() throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
w.forceMergeDeletes();
}
@ -413,7 +415,7 @@ public class RandomIndexWriter implements Closeable {
System.out.println("RIW: doRandomForceMerge(" + limit + ")");
}
w.forceMerge(limit);
if (limit == 1 || (w.getConfig().getMergePolicy() instanceof TieredMergePolicy) == false) {
if (limit == 1 || (config.getMergePolicy() instanceof TieredMergePolicy) == false) {
assert !doRandomForceMergeAssert || w.getSegmentCount() <= limit : "limit=" + limit + " actual=" + w.getSegmentCount();
}
} else {
@ -426,7 +428,7 @@ public class RandomIndexWriter implements Closeable {
}
public DirectoryReader getReader(boolean applyDeletions, boolean writeAllDeletes) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
getReaderCalled = true;
if (r.nextInt(20) == 2) {
doRandomForceMerge();
@ -447,8 +449,8 @@ public class RandomIndexWriter implements Closeable {
w.commit();
if (r.nextBoolean()) {
DirectoryReader reader = DirectoryReader.open(w.getDirectory());
if (w.getConfig().getSoftDeletesField() != null) {
return new SoftDeletesDirectoryReaderWrapper(reader, w.getConfig().getSoftDeletesField());
if (config.getSoftDeletesField() != null) {
return new SoftDeletesDirectoryReaderWrapper(reader, config.getSoftDeletesField());
} else {
return reader;
}
@ -467,13 +469,13 @@ public class RandomIndexWriter implements Closeable {
boolean success = false;
try {
if (w.isClosed() == false) {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
}
// if someone isn't using getReader() API, we want to be sure to
// forceMerge since presumably they might open a reader on the dir.
if (getReaderCalled == false && r.nextInt(8) == 2 && w.isClosed() == false) {
doRandomForceMerge();
if (w.getConfig().getCommitOnClose() == false) {
if (config.getCommitOnClose() == false) {
// index may have changed, must commit the changes, or otherwise they are discarded by the call to close()
w.commit();
}
@ -496,7 +498,7 @@ public class RandomIndexWriter implements Closeable {
* @see IndexWriter#forceMerge(int)
*/
public void forceMerge(int maxSegmentCount) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
w.forceMerge(maxSegmentCount);
}

24
lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
View File

@ -424,28 +424,6 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas
private final Map<SegmentCoreReaders,Boolean> warmed = Collections.synchronizedMap(new WeakHashMap<SegmentCoreReaders,Boolean>());
protected static IndexWriterConfig ensureSaneIWCOnNigtly(IndexWriterConfig conf) {
if (LuceneTestCase.TEST_NIGHTLY) {
// newIWConfig makes smallish max seg size, which
// results in tons and tons of segments for this test
// when run nightly:
MergePolicy mp = conf.getMergePolicy();
if (mp instanceof TieredMergePolicy) {
((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.);
} else if (mp instanceof LogByteSizeMergePolicy) {
((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.);
} else if (mp instanceof LogMergePolicy) {
((LogMergePolicy) mp).setMaxMergeDocs(100000);
}
// when running nightly, merging can still have crazy parameters,
// and might use many per-field codecs. turn on CFS for IW flushes
// and ensure CFS ratio is reasonable to keep it contained.
conf.setUseCompoundFile(true);
mp.setNoCFSRatio(Math.max(0.25d, mp.getNoCFSRatio()));
}
return conf;
}
public void runTest(String testName) throws Exception {
failed.set(false);
@ -470,7 +448,7 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas
((MockRandomMergePolicy)conf.getMergePolicy()).setDoNonBulkMerges(false);
}
ensureSaneIWCOnNigtly(conf);
ensureSaneIWCOnNightly(conf);
conf.setMergedSegmentWarmer((reader) -> {
if (VERBOSE) {

26
lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -2989,4 +2989,30 @@ public abstract class LuceneTestCase extends Assert {
assumeTrue("hit JDK collator bug", Integer.signum(v1) == Integer.signum(v2));
return v1;
}
/**
* Ensures that the MergePolicy has sane values for tests that test with lots of documents.
*/
protected static IndexWriterConfig ensureSaneIWCOnNightly(IndexWriterConfig conf) {
if (LuceneTestCase.TEST_NIGHTLY) {
// newIWConfig makes smallish max seg size, which
// results in tons and tons of segments for this test
// when run nightly:
MergePolicy mp = conf.getMergePolicy();
if (mp instanceof TieredMergePolicy) {
((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.);
} else if (mp instanceof LogByteSizeMergePolicy) {
((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.);
} else if (mp instanceof LogMergePolicy) {
((LogMergePolicy) mp).setMaxMergeDocs(100000);
}
// when running nightly, merging can still have crazy parameters,
// and might use many per-field codecs. turn on CFS for IW flushes
// and ensure CFS ratio is reasonable to keep it contained.
conf.setUseCompoundFile(true);
mp.setNoCFSRatio(Math.max(0.25d, mp.getNoCFSRatio()));
}
return conf;
}
}