Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Lower number of hits in TestLargeNumHitsTopDocsCollector (#13775) 

There's two tests where we use 250_000 as number of collected hits, but we only
ever index max 2000 docs. That makes use create a priority queue of size
250_000 for each segment partition which causes out of memory errors when the
number of partitions is higher than a few.

With this commit I propose that we lower the threshold to 2000 for those tests
that need a high number of collected hits. The assumption that a priority queue
is not built within the LargeNumHitsTopDocsCollector still holds so this change
should not defeat the purpose of the tests.
This commit is contained in:
Luca Cavanna 2024-09-12 16:19:22 +02:00 committed by GitHub
parent 5359d36ba6
commit cb48c7121a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/sandbox/src/java/org/apache/lucene/sandbox/search/LargeNumHitsTopDocsCollector.java
View File

@ -129,7 +129,7 @@ public final class LargeNumHitsTopDocsCollector implements Collector {
*/
protected void populateResults(ScoreDoc[] results, int howMany) {
if (pq != null) {
assert totalHits >= requestedHitCount;
assert totalHits > requestedHitCount;
for (int i = howMany - 1; i >= 0; i--) {
results[i] = pq.pop();
}
@ -137,7 +137,7 @@ public final class LargeNumHitsTopDocsCollector implements Collector {
}
// Total number of hits collected were less than requestedHitCount
assert totalHits < requestedHitCount;
assert totalHits <= requestedHitCount;
Collections.sort(
hits,
Comparator.comparing((ScoreDoc scoreDoc) -> scoreDoc.score)

4
lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestLargeNumHitsTopDocsCollector.java
View File

@ -105,7 +105,7 @@ public class TestLargeNumHitsTopDocsCollector extends LuceneTestCase {
IndexSearcher searcher = newSearcher(reader);
LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(250_000);
TopScoreDocCollectorManager regularCollectorManager =
new TopScoreDocCollectorManager(250_000, Integer.MAX_VALUE);
new TopScoreDocCollectorManager(reader.numDocs(), Integer.MAX_VALUE);
searcher.search(testQuery, largeCollector);
TopDocs topDocs = searcher.search(testQuery, regularCollectorManager);
@ -135,7 +135,7 @@ public class TestLargeNumHitsTopDocsCollector extends LuceneTestCase {
IndexSearcher searcher = newSearcher(reader);
LargeNumHitsTopDocsCollector largeCollector = new LargeNumHitsTopDocsCollector(250_000);
TopScoreDocCollectorManager regularCollectorManager =
new TopScoreDocCollectorManager(250_000, Integer.MAX_VALUE);
new TopScoreDocCollectorManager(reader.numDocs(), Integer.MAX_VALUE);
searcher.search(testQuery, largeCollector);
TopDocs topDocs = searcher.search(testQuery, regularCollectorManager);