Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8144: Moved QueryCachingPolicy.ALWAYS_CACHE to the test framework.

This commit is contained in:
Adrien Grand 2018-05-25 08:47:00 +02:00
parent 0e4e0096fd
commit 24d79de796
4 changed files with 43 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -44,6 +44,9 @@ API Changes
* LUCENE-8267: Memory codecs removed from the codebase (MemoryPostings, * LUCENE-8267: Memory codecs removed from the codebase (MemoryPostings,
MemoryDocValues). (Dawid Weiss) MemoryDocValues). (Dawid Weiss)
* LUCENE-8144: Moved QueryCachingPolicy.ALWAYS_CACHE to the test framework.
(Nhat Nguyen via Adrien Grand)
Changes in Runtime Behavior Changes in Runtime Behavior
* LUCENE-7837: Indices that were created before the previous major version * LUCENE-7837: Indices that were created before the previous major version

5
lucene/MIGRATE.txt
View File

@ -47,3 +47,8 @@ a boost of 1f.
## Memory codecs removed (LUCENE-8267) ## ## Memory codecs removed (LUCENE-8267) ##
Memory codecs have been removed from the codebase (MemoryPostings, MemoryDocValues). Memory codecs have been removed from the codebase (MemoryPostings, MemoryDocValues).
## QueryCachingPolicy.ALWAYS_CACHE removed (LUCENE-8144) ##
Caching everything is discouraged as it disables the ability to skip non-interesting documents.
ALWAYS_CACHE can be replaced by a UsageTrackingQueryCachingPolicy with an appropriate config.

13
lucene/core/src/java/org/apache/lucene/search/QueryCachingPolicy.java
View File

@ -31,19 +31,6 @@ import java.io.IOException;
// TODO: add APIs for integration with IndexWriter.IndexReaderWarmer // TODO: add APIs for integration with IndexWriter.IndexReaderWarmer
public interface QueryCachingPolicy { public interface QueryCachingPolicy {
/** A simple policy that caches all the provided filters on all segments. */
public static final QueryCachingPolicy ALWAYS_CACHE = new QueryCachingPolicy() {
@Override
public void onUse(Query query) {}
@Override
public boolean shouldCache(Query query) throws IOException {
return true;
}
};
/** Callback that is called every time that a cached filter is used. /** Callback that is called every time that a cached filter is used.
* This is typically useful if the policy wants to track usage statistics * This is typically useful if the policy wants to track usage statistics
* in order to make decisions. */ * in order to make decisions. */

58
lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
View File

@ -65,6 +65,18 @@ import org.apache.lucene.util.TestUtil;
public class TestLRUQueryCache extends LuceneTestCase { public class TestLRUQueryCache extends LuceneTestCase {
private static final QueryCachingPolicy ALWAYS_CACHE = new QueryCachingPolicy() {
@Override
public void onUse(Query query) {}
@Override
public boolean shouldCache(Query query) throws IOException {
return true;
}
};
private static final QueryCachingPolicy NEVER_CACHE = new QueryCachingPolicy() { private static final QueryCachingPolicy NEVER_CACHE = new QueryCachingPolicy() {
@Override @Override
@ -195,7 +207,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
searcher.search(new ConstantScoreQuery(green), 1); searcher.search(new ConstantScoreQuery(green), 1);
assertEquals(Collections.emptyList(), queryCache.cachedQueries()); assertEquals(Collections.emptyList(), queryCache.cachedQueries());
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
searcher.search(new ConstantScoreQuery(red), 1); searcher.search(new ConstantScoreQuery(red), 1);
assertEquals(Collections.singletonList(red), queryCache.cachedQueries()); assertEquals(Collections.singletonList(red), queryCache.cachedQueries());
@ -244,7 +256,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
final LRUQueryCache queryCache = new LRUQueryCache(Integer.MAX_VALUE, Long.MAX_VALUE, context -> true); final LRUQueryCache queryCache = new LRUQueryCache(Integer.MAX_VALUE, Long.MAX_VALUE, context -> true);
searcher.setQueryCache(queryCache); searcher.setQueryCache(queryCache);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
searcher.search(new BoostQuery(new ConstantScoreQuery(query1), random().nextFloat()), 1); searcher.search(new BoostQuery(new ConstantScoreQuery(query1), random().nextFloat()), 1);
assertEquals(1, queryCache.cachedQueries().size()); assertEquals(1, queryCache.cachedQueries().size());
@ -414,7 +426,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
final DirectoryReader reader = w.getReader(); final DirectoryReader reader = w.getReader();
final IndexSearcher searcher = new IndexSearcher(reader); final IndexSearcher searcher = new IndexSearcher(reader);
searcher.setQueryCache(queryCache); searcher.setQueryCache(queryCache);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
final int numQueries = atLeast(1000); final int numQueries = atLeast(1000);
for (int i = 0; i < numQueries; ++i) { for (int i = 0; i < numQueries; ++i) {
@ -532,7 +544,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
assertEquals(0, queryCache.getCacheSize()); assertEquals(0, queryCache.getCacheSize());
// second pass, lookups + caching, only the first one is a miss // second pass, lookups + caching, only the first one is a miss
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
for (int i = 0; i < 10; ++i) { for (int i = 0; i < 10; ++i) {
searcher.search(new ConstantScoreQuery(query), 1); searcher.search(new ConstantScoreQuery(query), 1);
} }
@ -556,7 +568,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
assertEquals(1 * segmentCount, queryCache.getCacheSize()); assertEquals(1 * segmentCount, queryCache.getCacheSize());
// fourth pass with a different filter which will trigger evictions since the size is 1 // fourth pass with a different filter which will trigger evictions since the size is 1
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
for (int i = 0; i < 10; ++i) { for (int i = 0; i < 10; ++i) {
searcher.search(new ConstantScoreQuery(query2), 1); searcher.search(new ConstantScoreQuery(query2), 1);
} }
@ -698,7 +710,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
for (IndexSearcher searcher : Arrays.asList(searcher1, searcher2)) { for (IndexSearcher searcher : Arrays.asList(searcher1, searcher2)) {
searcher.setQueryCache(queryCache); searcher.setQueryCache(queryCache);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
} }
// search on searcher1 // search on searcher1
@ -801,7 +813,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
final LRUQueryCache queryCache = new LRUQueryCache(1000000, 10000000, context -> true); final LRUQueryCache queryCache = new LRUQueryCache(1000000, 10000000, context -> true);
searcher.setQueryCache(queryCache); searcher.setQueryCache(queryCache);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
BooleanQuery.Builder bq = new BooleanQuery.Builder(); BooleanQuery.Builder bq = new BooleanQuery.Builder();
TermQuery should = new TermQuery(new Term("foo", "baz")); TermQuery should = new TermQuery(new Term("foo", "baz"));
@ -920,7 +932,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
uncachedSearcher.setQueryCache(null); uncachedSearcher.setQueryCache(null);
cachedSearcher = newSearcher(reader); cachedSearcher = newSearcher(reader);
cachedSearcher.setQueryCache(queryCache); cachedSearcher.setQueryCache(queryCache);
cachedSearcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); cachedSearcher.setQueryCachingPolicy(ALWAYS_CACHE);
} }
final Query q = buildRandomQuery(0); final Query q = buildRandomQuery(0);
assertEquals(uncachedSearcher.count(q), cachedSearcher.count(q)); assertEquals(uncachedSearcher.count(q), cachedSearcher.count(q));
@ -983,7 +995,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
final LRUQueryCache queryCache = new LRUQueryCache(1, 10000, context -> true); final LRUQueryCache queryCache = new LRUQueryCache(1, 10000, context -> true);
final IndexSearcher searcher = newSearcher(reader); final IndexSearcher searcher = newSearcher(reader);
searcher.setQueryCache(queryCache); searcher.setQueryCache(queryCache);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
BadQuery query = new BadQuery(); BadQuery query = new BadQuery();
searcher.count(query); searcher.count(query);
@ -1017,7 +1029,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
final LRUQueryCache queryCache = new LRUQueryCache(1, 1, context -> random().nextBoolean()); final LRUQueryCache queryCache = new LRUQueryCache(1, 1, context -> random().nextBoolean());
final IndexSearcher searcher = newSearcher(reader); final IndexSearcher searcher = newSearcher(reader);
searcher.setQueryCache(queryCache); searcher.setQueryCache(queryCache);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
searcher.count(new MatchAllDocsQuery()); searcher.count(new MatchAllDocsQuery());
assertEquals(0, queryCache.getCacheCount()); assertEquals(0, queryCache.getCacheCount());
@ -1152,7 +1164,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
bulkScorerCalled.set(false); bulkScorerCalled.set(false);
weight = searcher.createWeight(new MatchAllDocsQuery(), ScoreMode.COMPLETE_NO_SCORES, 1); weight = searcher.createWeight(new MatchAllDocsQuery(), ScoreMode.COMPLETE_NO_SCORES, 1);
weight = new WeightWrapper(weight, scorerCalled, bulkScorerCalled); weight = new WeightWrapper(weight, scorerCalled, bulkScorerCalled);
weight = cache.doCache(weight, QueryCachingPolicy.ALWAYS_CACHE); weight = cache.doCache(weight, ALWAYS_CACHE);
weight.scorer(leaf); weight.scorer(leaf);
assertEquals(true, bulkScorerCalled.get()); assertEquals(true, bulkScorerCalled.get());
assertEquals(false, scorerCalled.get()); assertEquals(false, scorerCalled.get());
@ -1177,7 +1189,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
}; };
searcher.setQueryCache(queryCache); searcher.setQueryCache(queryCache);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
Query query = new DummyQuery(); Query query = new DummyQuery();
searcher.count(query); searcher.count(query);
@ -1196,7 +1208,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
w.addDocument(new Document()); w.addDocument(new Document());
DirectoryReader reader = w.getReader(); DirectoryReader reader = w.getReader();
IndexSearcher searcher = newSearcher(reader); IndexSearcher searcher = newSearcher(reader);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
LRUQueryCache cache = new LRUQueryCache(2, 10000, new LRUQueryCache.MinSegmentSizePredicate(2, 0f)); LRUQueryCache cache = new LRUQueryCache(2, 10000, new LRUQueryCache.MinSegmentSizePredicate(2, 0f));
searcher.setQueryCache(cache); searcher.setQueryCache(cache);
@ -1217,7 +1229,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
reader.close(); reader.close();
reader = w.getReader(); reader = w.getReader();
searcher = newSearcher(reader); searcher = newSearcher(reader);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
cache = new LRUQueryCache(2, 10000, new LRUQueryCache.MinSegmentSizePredicate(0, .6f)); cache = new LRUQueryCache(2, 10000, new LRUQueryCache.MinSegmentSizePredicate(0, .6f));
searcher.setQueryCache(cache); searcher.setQueryCache(cache);
searcher.count(new DummyQuery()); searcher.count(new DummyQuery());
@ -1267,7 +1279,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
w.addDocument(new Document()); w.addDocument(new Document());
DirectoryReader reader = new DummyDirectoryReader(w.getReader()); DirectoryReader reader = new DummyDirectoryReader(w.getReader());
IndexSearcher searcher = newSearcher(reader); IndexSearcher searcher = newSearcher(reader);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
// don't cache if the reader does not expose a cache helper // don't cache if the reader does not expose a cache helper
assertNull(reader.leaves().get(0).reader().getCoreCacheHelper()); assertNull(reader.leaves().get(0).reader().getCoreCacheHelper());
@ -1331,7 +1343,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
w.addDocument(new Document()); w.addDocument(new Document());
DirectoryReader reader = w.getReader(); DirectoryReader reader = w.getReader();
IndexSearcher searcher = newSearcher(reader); IndexSearcher searcher = newSearcher(reader);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
LRUQueryCache cache = new LRUQueryCache(2, 10000, context -> true); LRUQueryCache cache = new LRUQueryCache(2, 10000, context -> true);
searcher.setQueryCache(cache); searcher.setQueryCache(cache);
@ -1491,7 +1503,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
// we need to be single threaded to ensure that LRUQueryCache doesn't skip the cache // we need to be single threaded to ensure that LRUQueryCache doesn't skip the cache
// due to thread contention // due to thread contention
IndexSearcher searcher = new AssertingIndexSearcher(random(), reader); IndexSearcher searcher = new AssertingIndexSearcher(random(), reader);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
LRUQueryCache cache = new LRUQueryCache(1, 10000, context -> true); LRUQueryCache cache = new LRUQueryCache(1, 10000, context -> true);
searcher.setQueryCache(cache); searcher.setQueryCache(cache);
@ -1509,7 +1521,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
reader.close(); reader.close();
reader = DirectoryReader.open(w); reader = DirectoryReader.open(w);
searcher = new AssertingIndexSearcher(random(), reader); // no newSearcher(reader) - see comment above searcher = new AssertingIndexSearcher(random(), reader); // no newSearcher(reader) - see comment above
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
searcher.setQueryCache(cache); searcher.setQueryCache(cache);
assertEquals(2, searcher.count(query)); assertEquals(2, searcher.count(query));
@ -1518,7 +1530,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
reader.close(); reader.close();
reader = DirectoryReader.open(w); reader = DirectoryReader.open(w);
searcher = new AssertingIndexSearcher(random(), reader); // no newSearcher(reader) - see comment above searcher = new AssertingIndexSearcher(random(), reader); // no newSearcher(reader) - see comment above
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
searcher.setQueryCache(cache); searcher.setQueryCache(cache);
assertEquals(2, searcher.count(query)); assertEquals(2, searcher.count(query));
@ -1529,7 +1541,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
reader.close(); reader.close();
reader = DirectoryReader.open(w); reader = DirectoryReader.open(w);
searcher = new AssertingIndexSearcher(random(), reader); // no newSearcher(reader) - see comment above searcher = new AssertingIndexSearcher(random(), reader); // no newSearcher(reader) - see comment above
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
searcher.setQueryCache(cache); searcher.setQueryCache(cache);
assertEquals(2, searcher.count(query)); assertEquals(2, searcher.count(query));
@ -1550,7 +1562,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
IndexWriter w = new IndexWriter(dir, iwc); IndexWriter w = new IndexWriter(dir, iwc);
LRUQueryCache queryCache = new LRUQueryCache(10, 1000 * 1000, ctx -> true); LRUQueryCache queryCache = new LRUQueryCache(10, 1000 * 1000, ctx -> true);
IndexSearcher.setDefaultQueryCache(queryCache); IndexSearcher.setDefaultQueryCache(queryCache);
IndexSearcher.setDefaultQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); IndexSearcher.setDefaultQueryCachingPolicy(ALWAYS_CACHE);
SearcherManager sm = new SearcherManager(w, new SearcherFactory()); SearcherManager sm = new SearcherManager(w, new SearcherFactory());
@ -1622,7 +1634,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
LRUQueryCache cache = new LRUQueryCache(1, 100000, context -> true); LRUQueryCache cache = new LRUQueryCache(1, 100000, context -> true);
IndexSearcher searcher = new AssertingIndexSearcher(random(), reader); IndexSearcher searcher = new AssertingIndexSearcher(random(), reader);
searcher.setQueryCache(cache); searcher.setQueryCache(cache);
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); searcher.setQueryCachingPolicy(ALWAYS_CACHE);
Query query = new ConstantScoreQuery(new BooleanQuery.Builder() Query query = new ConstantScoreQuery(new BooleanQuery.Builder()
.add(new BoostQuery(new TermQuery(new Term("field", "foo")), 3), Occur.SHOULD) .add(new BoostQuery(new TermQuery(new Term("field", "foo")), 3), Occur.SHOULD)
@ -1634,7 +1646,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
IndexSearcher noCacheHelperSearcher = new AssertingIndexSearcher(random(), noCacheReader); IndexSearcher noCacheHelperSearcher = new AssertingIndexSearcher(random(), noCacheReader);
noCacheHelperSearcher.setQueryCache(cache); noCacheHelperSearcher.setQueryCache(cache);
noCacheHelperSearcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE); noCacheHelperSearcher.setQueryCachingPolicy(ALWAYS_CACHE);
noCacheHelperSearcher.search(query, 1); noCacheHelperSearcher.search(query, 1);
Thread t = new Thread(() -> { Thread t = new Thread(() -> {