LUCENE-6106: Improve tracking of filter usage in LRUFilterCache.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1645613 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2014-12-15 10:32:09 +00:00
parent 692826b906
commit 4ecff9bf89
5 changed files with 68 additions and 7 deletions

View File

@ -38,7 +38,7 @@ public interface FilterCachingPolicy {
public static final FilterCachingPolicy ALWAYS_CACHE = new FilterCachingPolicy() { public static final FilterCachingPolicy ALWAYS_CACHE = new FilterCachingPolicy() {
@Override @Override
public void onCache(Filter filter) {} public void onUse(Filter filter) {}
@Override @Override
public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException { public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
@ -78,7 +78,7 @@ public interface FilterCachingPolicy {
} }
@Override @Override
public void onCache(Filter filter) {} public void onUse(Filter filter) {}
@Override @Override
public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException { public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
@ -89,10 +89,10 @@ public interface FilterCachingPolicy {
}; };
/** Callback that is called on every call to {@link FilterCache#doCache}. /** Callback that is called every time that a cached filter is used.
* This is typically useful if the policy wants to track usage statistics * This is typically useful if the policy wants to track usage statistics
* in order to make decisions. */ * in order to make decisions. */
void onCache(Filter filter); void onUse(Filter filter);
/** Whether the given {@link DocIdSet} should be cached on a given segment. /** Whether the given {@link DocIdSet} should be cached on a given segment.
* This method will be called on each leaf context to know if the filter * This method will be called on each leaf context to know if the filter

View File

@ -339,6 +339,10 @@ public class LRUFilterCache implements FilterCache, Accountable {
@Override @Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException { public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
if (context.ord == 0) {
policy.onUse(in);
}
DocIdSet set = get(in, context); DocIdSet set = get(in, context);
if (set == null) { if (set == null) {
// do not apply acceptDocs yet, we want the cached filter to not take them into account // do not apply acceptDocs yet, we want the cached filter to not take them into account

View File

@ -103,7 +103,7 @@ public final class UsageTrackingFilterCachingPolicy implements FilterCachingPoli
} }
@Override @Override
public void onCache(Filter filter) { public void onUse(Filter filter) {
// Using the filter hash codes might help keep memory usage a bit lower // Using the filter hash codes might help keep memory usage a bit lower
// since some filters might have non-negligible memory usage? // since some filters might have non-negligible memory usage?
recentlyUsedFilters.add(filter.hashCode()); recentlyUsedFilters.add(filter.hashCode());

View File

@ -22,6 +22,7 @@ import java.lang.reflect.Field;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -49,7 +50,7 @@ public class TestLRUFilterCache extends LuceneTestCase {
private static final FilterCachingPolicy NEVER_CACHE = new FilterCachingPolicy() { private static final FilterCachingPolicy NEVER_CACHE = new FilterCachingPolicy() {
@Override @Override
public void onCache(Filter filter) {} public void onUse(Filter filter) {}
@Override @Override
public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException { public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
@ -401,4 +402,60 @@ public class TestLRUFilterCache extends LuceneTestCase {
dir.close(); dir.close();
} }
public void testOnUse() throws IOException {
final LRUFilterCache filterCache = new LRUFilterCache(1 + random().nextInt(5), 1 + random().nextInt(1000));
Directory dir = newDirectory();
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
Document doc = new Document();
StringField f = new StringField("color", "", Store.NO);
doc.add(f);
final int numDocs = atLeast(10);
for (int i = 0; i < numDocs; ++i) {
f.setStringValue(RandomPicks.randomFrom(random(), Arrays.asList("red", "blue", "green", "yellow")));
w.addDocument(doc);
if (random().nextBoolean()) {
w.getReader().close();
}
}
final DirectoryReader reader = w.getReader();
final IndexSearcher searcher = new IndexSearcher(reader);
final Map<Filter, Integer> actualCounts = new HashMap<>();
final Map<Filter, Integer> expectedCounts = new HashMap<>();
final FilterCachingPolicy countingPolicy = new FilterCachingPolicy() {
@Override
public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
return random().nextBoolean();
}
@Override
public void onUse(Filter filter) {
expectedCounts.put(filter, expectedCounts.getOrDefault(filter, 0));
}
};
Filter[] filters = new Filter[10 + random().nextInt(10)];
Filter[] cachedFilters = new Filter[filters.length];
for (int i = 0; i < filters.length; ++i) {
filters[i] = new QueryWrapperFilter(new TermQuery(new Term("color", RandomPicks.randomFrom(random(), Arrays.asList("red", "blue", "green", "yellow")))));
cachedFilters[i] = filterCache.doCache(filters[i], countingPolicy);
}
for (int i = 0; i < 20; ++i) {
final int idx = random().nextInt(filters.length);
searcher.search(new ConstantScoreQuery(cachedFilters[idx]), 1);
actualCounts.put(filters[idx], actualCounts.getOrDefault(filters[idx], 0));
}
assertEquals(actualCounts, expectedCounts);
reader.close();
w.close();
dir.close();
}
} }

View File

@ -474,7 +474,7 @@ public abstract class LuceneTestCase extends Assert {
public static final FilterCachingPolicy MAYBE_CACHE_POLICY = new FilterCachingPolicy() { public static final FilterCachingPolicy MAYBE_CACHE_POLICY = new FilterCachingPolicy() {
@Override @Override
public void onCache(Filter filter) {} public void onUse(Filter filter) {}
@Override @Override
public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException { public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {