From 207d240ae2427e15af929fa44f1b36d4f8e5a250 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 29 Apr 2020 22:11:42 +0200 Subject: [PATCH] Fix tests to survive nightly runs with many documents --- .../index/TestNRTReaderWithThreads.java | 4 +- .../lucene/search/TestSearcherManager.java | 4 +- .../lucene/index/RandomIndexWriter.java | 54 ++++++++++--------- .../ThreadedIndexingAndSearchingTestCase.java | 24 +-------- .../apache/lucene/util/LuceneTestCase.java | 26 +++++++++ 5 files changed, 59 insertions(+), 53 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java b/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java index 45ceb5a8513..3a41d3ad0fc 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java @@ -36,9 +36,9 @@ public class TestNRTReaderWithThreads extends LuceneTestCase { } IndexWriter writer = new IndexWriter( mainDir, - newIndexWriterConfig(new MockAnalyzer(random())) + ensureSaneIWCOnNightly(newIndexWriterConfig(new MockAnalyzer(random())) .setMaxBufferedDocs(10) - .setMergePolicy(newLogMergePolicy(false,2)) + .setMergePolicy(newLogMergePolicy(false,2))) ); IndexReader reader = writer.getReader(); // start pooling readers reader.close(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java b/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java index 9d5e087db7d..b9238666916 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java @@ -556,7 +556,7 @@ public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase { AtomicReference writerRef = new AtomicReference<>(); final MockAnalyzer analyzer = new MockAnalyzer(random()); analyzer.setMaxTokenLength(IndexWriter.MAX_TERM_LENGTH); - writerRef.set(new IndexWriter(dir, ensureSaneIWCOnNigtly(newIndexWriterConfig(analyzer)))); + writerRef.set(new IndexWriter(dir, ensureSaneIWCOnNightly(newIndexWriterConfig(analyzer)))); AtomicReference mgrRef = new AtomicReference<>(); mgrRef.set(new SearcherManager(writerRef.get(), null)); @@ -580,7 +580,7 @@ public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase { } else { w.rollback(); } - writerRef.set(new IndexWriter(dir, ensureSaneIWCOnNigtly(newIndexWriterConfig(analyzer)))); + writerRef.set(new IndexWriter(dir, ensureSaneIWCOnNightly(newIndexWriterConfig(analyzer)))); } } if (VERBOSE) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java index e2915e06d2a..7c70480a865 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java @@ -53,6 +53,7 @@ public class RandomIndexWriter implements Closeable { private boolean getReaderCalled; private final Analyzer analyzer; // only if WE created it (then we close it) private final double softDeletesRatio; + private final LiveIndexWriterConfig config; /** Returns an indexwriter that randomly mixes up thread scheduling (by yielding at test points) */ public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException { @@ -131,6 +132,7 @@ public class RandomIndexWriter implements Closeable { softDeletesRatio = 0d; } w = mockIndexWriter(dir, c, r); + config = w.getConfig(); flushAt = TestUtil.nextInt(r, 10, 1000); if (closeAnalyzer) { analyzer = w.getAnalyzer(); @@ -151,7 +153,7 @@ public class RandomIndexWriter implements Closeable { * @see IndexWriter#addDocument(Iterable) */ public long addDocument(final Iterable doc) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); long seqNo; if (r.nextInt(5) == 3) { // TODO: maybe, we should simply buffer up added docs @@ -197,7 +199,7 @@ public class RandomIndexWriter implements Closeable { } private void maybeFlushOrCommit() throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); if (docCount++ == flushAt) { if (r.nextBoolean()) { flushAllBuffersSequentially(); @@ -234,17 +236,17 @@ public class RandomIndexWriter implements Closeable { } public long addDocuments(Iterable> docs) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); long seqNo = w.addDocuments(docs); maybeFlushOrCommit(); return seqNo; } public long updateDocuments(Term delTerm, Iterable> docs) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); long seqNo; if (useSoftDeletes()) { - seqNo = w.softUpdateDocuments(delTerm, docs, new NumericDocValuesField(w.getConfig().getSoftDeletesField(), 1)); + seqNo = w.softUpdateDocuments(delTerm, docs, new NumericDocValuesField(config.getSoftDeletesField(), 1)); } else { seqNo = w.updateDocuments(delTerm, docs); } @@ -261,13 +263,13 @@ public class RandomIndexWriter implements Closeable { * @see IndexWriter#updateDocument(Term, Iterable) */ public long updateDocument(Term t, final Iterable doc) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); final long seqNo; if (useSoftDeletes()) { if (r.nextInt(5) == 3) { - seqNo = w.softUpdateDocuments(t, Arrays.asList(doc), new NumericDocValuesField(w.getConfig().getSoftDeletesField(), 1)); + seqNo = w.softUpdateDocuments(t, Arrays.asList(doc), new NumericDocValuesField(config.getSoftDeletesField(), 1)); } else { - seqNo = w.softUpdateDocument(t, doc, new NumericDocValuesField(w.getConfig().getSoftDeletesField(), 1)); + seqNo = w.softUpdateDocument(t, doc, new NumericDocValuesField(config.getSoftDeletesField(), 1)); } } else { if (r.nextInt(5) == 3) { @@ -282,37 +284,37 @@ public class RandomIndexWriter implements Closeable { } public long addIndexes(Directory... dirs) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); return w.addIndexes(dirs); } public long addIndexes(CodecReader... readers) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); return w.addIndexes(readers); } public long updateNumericDocValue(Term term, String field, Long value) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); return w.updateNumericDocValue(term, field, value); } public long updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); return w.updateBinaryDocValue(term, field, value); } public long updateDocValues(Term term, Field... updates) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); return w.updateDocValues(term, updates); } public long deleteDocuments(Term term) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); return w.deleteDocuments(term); } public long deleteDocuments(Query q) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); return w.deleteDocuments(q); } @@ -321,7 +323,7 @@ public class RandomIndexWriter implements Closeable { } public long commit(boolean flushConcurrently) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); if (flushConcurrently) { List throwableList = new CopyOnWriteArrayList<>(); Thread thread = new Thread(() -> { @@ -372,7 +374,7 @@ public class RandomIndexWriter implements Closeable { } public DirectoryReader getReader() throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); return getReader(true, false); } @@ -380,12 +382,12 @@ public class RandomIndexWriter implements Closeable { private boolean doRandomForceMergeAssert; public void forceMergeDeletes(boolean doWait) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); w.forceMergeDeletes(doWait); } public void forceMergeDeletes() throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); w.forceMergeDeletes(); } @@ -413,7 +415,7 @@ public class RandomIndexWriter implements Closeable { System.out.println("RIW: doRandomForceMerge(" + limit + ")"); } w.forceMerge(limit); - if (limit == 1 || (w.getConfig().getMergePolicy() instanceof TieredMergePolicy) == false) { + if (limit == 1 || (config.getMergePolicy() instanceof TieredMergePolicy) == false) { assert !doRandomForceMergeAssert || w.getSegmentCount() <= limit : "limit=" + limit + " actual=" + w.getSegmentCount(); } } else { @@ -426,7 +428,7 @@ public class RandomIndexWriter implements Closeable { } public DirectoryReader getReader(boolean applyDeletions, boolean writeAllDeletes) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); getReaderCalled = true; if (r.nextInt(20) == 2) { doRandomForceMerge(); @@ -447,8 +449,8 @@ public class RandomIndexWriter implements Closeable { w.commit(); if (r.nextBoolean()) { DirectoryReader reader = DirectoryReader.open(w.getDirectory()); - if (w.getConfig().getSoftDeletesField() != null) { - return new SoftDeletesDirectoryReaderWrapper(reader, w.getConfig().getSoftDeletesField()); + if (config.getSoftDeletesField() != null) { + return new SoftDeletesDirectoryReaderWrapper(reader, config.getSoftDeletesField()); } else { return reader; } @@ -467,13 +469,13 @@ public class RandomIndexWriter implements Closeable { boolean success = false; try { if (w.isClosed() == false) { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); } // if someone isn't using getReader() API, we want to be sure to // forceMerge since presumably they might open a reader on the dir. if (getReaderCalled == false && r.nextInt(8) == 2 && w.isClosed() == false) { doRandomForceMerge(); - if (w.getConfig().getCommitOnClose() == false) { + if (config.getCommitOnClose() == false) { // index may have changed, must commit the changes, or otherwise they are discarded by the call to close() w.commit(); } @@ -496,7 +498,7 @@ public class RandomIndexWriter implements Closeable { * @see IndexWriter#forceMerge(int) */ public void forceMerge(int maxSegmentCount) throws IOException { - LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); + LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config); w.forceMerge(maxSegmentCount); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java index 56919793755..1a35ee725dd 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java @@ -424,28 +424,6 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas private final Map warmed = Collections.synchronizedMap(new WeakHashMap()); - protected static IndexWriterConfig ensureSaneIWCOnNigtly(IndexWriterConfig conf) { - if (LuceneTestCase.TEST_NIGHTLY) { - // newIWConfig makes smallish max seg size, which - // results in tons and tons of segments for this test - // when run nightly: - MergePolicy mp = conf.getMergePolicy(); - if (mp instanceof TieredMergePolicy) { - ((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.); - } else if (mp instanceof LogByteSizeMergePolicy) { - ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.); - } else if (mp instanceof LogMergePolicy) { - ((LogMergePolicy) mp).setMaxMergeDocs(100000); - } - // when running nightly, merging can still have crazy parameters, - // and might use many per-field codecs. turn on CFS for IW flushes - // and ensure CFS ratio is reasonable to keep it contained. - conf.setUseCompoundFile(true); - mp.setNoCFSRatio(Math.max(0.25d, mp.getNoCFSRatio())); - } - return conf; - } - public void runTest(String testName) throws Exception { failed.set(false); @@ -470,7 +448,7 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas ((MockRandomMergePolicy)conf.getMergePolicy()).setDoNonBulkMerges(false); } - ensureSaneIWCOnNigtly(conf); + ensureSaneIWCOnNightly(conf); conf.setMergedSegmentWarmer((reader) -> { if (VERBOSE) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java index cb32dc26659..9f2cd27c8c7 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java @@ -2989,4 +2989,30 @@ public abstract class LuceneTestCase extends Assert { assumeTrue("hit JDK collator bug", Integer.signum(v1) == Integer.signum(v2)); return v1; } + + + /** + * Ensures that the MergePolicy has sane values for tests that test with lots of documents. + */ + protected static IndexWriterConfig ensureSaneIWCOnNightly(IndexWriterConfig conf) { + if (LuceneTestCase.TEST_NIGHTLY) { + // newIWConfig makes smallish max seg size, which + // results in tons and tons of segments for this test + // when run nightly: + MergePolicy mp = conf.getMergePolicy(); + if (mp instanceof TieredMergePolicy) { + ((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.); + } else if (mp instanceof LogByteSizeMergePolicy) { + ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.); + } else if (mp instanceof LogMergePolicy) { + ((LogMergePolicy) mp).setMaxMergeDocs(100000); + } + // when running nightly, merging can still have crazy parameters, + // and might use many per-field codecs. turn on CFS for IW flushes + // and ensure CFS ratio is reasonable to keep it contained. + conf.setUseCompoundFile(true); + mp.setNoCFSRatio(Math.max(0.25d, mp.getNoCFSRatio())); + } + return conf; + } }