mirror of https://github.com/apache/lucene.git
LUCENE-6303: Disable auto-caching in IndexSearcher.
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1670006 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
8c88f40869
commit
92a7c7e4ee
|
@ -74,8 +74,8 @@ import org.apache.lucene.util.ThreadInterruptedException;
|
||||||
*/
|
*/
|
||||||
public class IndexSearcher {
|
public class IndexSearcher {
|
||||||
|
|
||||||
// 32MB and at most 10,000 queries
|
// disabled by default
|
||||||
private static QueryCache DEFAULT_QUERY_CACHE = new LRUQueryCache(10000, 1 << 25);
|
private static QueryCache DEFAULT_QUERY_CACHE = null;
|
||||||
private static QueryCachingPolicy DEFAULT_CACHING_POLICY = new UsageTrackingQueryCachingPolicy();
|
private static QueryCachingPolicy DEFAULT_CACHING_POLICY = new UsageTrackingQueryCachingPolicy();
|
||||||
|
|
||||||
final IndexReader reader; // package private for testing!
|
final IndexReader reader; // package private for testing!
|
||||||
|
@ -187,7 +187,10 @@ public class IndexSearcher {
|
||||||
* A value of {@code null} indicates that query matches should never be
|
* A value of {@code null} indicates that query matches should never be
|
||||||
* cached. This method should be called <b>before</b> starting using this
|
* cached. This method should be called <b>before</b> starting using this
|
||||||
* {@link IndexSearcher}.
|
* {@link IndexSearcher}.
|
||||||
|
* <p>NOTE: When using a query cache, queries should not be modified after
|
||||||
|
* they have been passed to IndexSearcher.
|
||||||
* @see QueryCache
|
* @see QueryCache
|
||||||
|
* @lucene.experimental
|
||||||
*/
|
*/
|
||||||
public void setQueryCache(QueryCache queryCache) {
|
public void setQueryCache(QueryCache queryCache) {
|
||||||
this.queryCache = queryCache;
|
this.queryCache = queryCache;
|
||||||
|
@ -198,6 +201,7 @@ public class IndexSearcher {
|
||||||
* This method should be called <b>before</b> starting using this
|
* This method should be called <b>before</b> starting using this
|
||||||
* {@link IndexSearcher}.
|
* {@link IndexSearcher}.
|
||||||
* @see QueryCachingPolicy
|
* @see QueryCachingPolicy
|
||||||
|
* @lucene.experimental
|
||||||
*/
|
*/
|
||||||
public void setQueryCachingPolicy(QueryCachingPolicy queryCachingPolicy) {
|
public void setQueryCachingPolicy(QueryCachingPolicy queryCachingPolicy) {
|
||||||
this.queryCachingPolicy = Objects.requireNonNull(queryCachingPolicy);
|
this.queryCachingPolicy = Objects.requireNonNull(queryCachingPolicy);
|
||||||
|
|
|
@ -21,6 +21,7 @@ import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
import java.util.ConcurrentModificationException;
|
||||||
import java.util.IdentityHashMap;
|
import java.util.IdentityHashMap;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
|
@ -269,10 +270,20 @@ public class LRUQueryCache implements QueryCache, Accountable {
|
||||||
synchronized void evictIfNecessary() {
|
synchronized void evictIfNecessary() {
|
||||||
// under a lock to make sure that mostRecentlyUsedQueries and cache keep sync'ed
|
// under a lock to make sure that mostRecentlyUsedQueries and cache keep sync'ed
|
||||||
if (requiresEviction()) {
|
if (requiresEviction()) {
|
||||||
|
|
||||||
Iterator<Query> iterator = mostRecentlyUsedQueries.iterator();
|
Iterator<Query> iterator = mostRecentlyUsedQueries.iterator();
|
||||||
do {
|
do {
|
||||||
final Query query = iterator.next();
|
final Query query = iterator.next();
|
||||||
|
final int size = mostRecentlyUsedQueries.size();
|
||||||
iterator.remove();
|
iterator.remove();
|
||||||
|
if (size == mostRecentlyUsedQueries.size()) {
|
||||||
|
// size did not decrease, because the hash of the query changed since it has been
|
||||||
|
// put into the cache
|
||||||
|
throw new ConcurrentModificationException("Removal from the cache failed! This " +
|
||||||
|
"is probably due to a query which has been modified after having been put into " +
|
||||||
|
" the cache or a badly implemented clone(). Query class: [" + query.getClass() +
|
||||||
|
"], query: [" + query + "]");
|
||||||
|
}
|
||||||
onEviction(query);
|
onEviction(query);
|
||||||
} while (iterator.hasNext() && requiresEviction());
|
} while (iterator.hasNext() && requiresEviction());
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,6 +22,7 @@ import java.lang.reflect.Field;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
import java.util.ConcurrentModificationException;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.LinkedHashMap;
|
import java.util.LinkedHashMap;
|
||||||
|
@ -45,6 +46,7 @@ import org.apache.lucene.index.Term;
|
||||||
import org.apache.lucene.search.BooleanClause.Occur;
|
import org.apache.lucene.search.BooleanClause.Occur;
|
||||||
import org.apache.lucene.store.Directory;
|
import org.apache.lucene.store.Directory;
|
||||||
import org.apache.lucene.util.Bits;
|
import org.apache.lucene.util.Bits;
|
||||||
|
import org.apache.lucene.util.IOUtils;
|
||||||
import org.apache.lucene.util.LuceneTestCase;
|
import org.apache.lucene.util.LuceneTestCase;
|
||||||
import org.apache.lucene.util.RamUsageTester;
|
import org.apache.lucene.util.RamUsageTester;
|
||||||
import org.apache.lucene.util.TestUtil;
|
import org.apache.lucene.util.TestUtil;
|
||||||
|
@ -913,4 +915,56 @@ public class TestLRUQueryCache extends LuceneTestCase {
|
||||||
queryCache.assertConsistent();
|
queryCache.assertConsistent();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static class BadQuery extends Query {
|
||||||
|
|
||||||
|
int[] i = new int[] {42}; // an array so that clone keeps the reference
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
|
||||||
|
return new ConstantScoreWeight(this) {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
Scorer scorer(LeafReaderContext context, Bits acceptDocs, float score) throws IOException {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString(String field) {
|
||||||
|
return "BadQuery";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return super.hashCode() ^ i[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testDetectMutatedQueries() throws IOException {
|
||||||
|
Directory dir = newDirectory();
|
||||||
|
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
|
||||||
|
w.addDocument(new Document());
|
||||||
|
IndexReader reader = w.getReader();
|
||||||
|
|
||||||
|
// size of 1 so that 2nd query evicts from the cache
|
||||||
|
final LRUQueryCache queryCache = new LRUQueryCache(1, 10000);
|
||||||
|
final IndexSearcher searcher = newSearcher(reader);
|
||||||
|
searcher.setQueryCache(queryCache);
|
||||||
|
searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE);
|
||||||
|
|
||||||
|
BadQuery query = new BadQuery();
|
||||||
|
searcher.count(query);
|
||||||
|
query.i[0] += 1; // change the hashCode!
|
||||||
|
try {
|
||||||
|
// trigger an eviction
|
||||||
|
searcher.count(new MatchAllDocsQuery());
|
||||||
|
fail();
|
||||||
|
} catch (ConcurrentModificationException e) {
|
||||||
|
// expected
|
||||||
|
}
|
||||||
|
|
||||||
|
IOUtils.close(w, reader, dir);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,24 +21,23 @@ import java.util.Random;
|
||||||
|
|
||||||
import org.apache.lucene.index.Term;
|
import org.apache.lucene.index.Term;
|
||||||
import org.apache.lucene.util.TestUtil;
|
import org.apache.lucene.util.TestUtil;
|
||||||
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* random sloppy phrase query tests
|
* random sloppy phrase query tests
|
||||||
*/
|
*/
|
||||||
@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-6369")
|
|
||||||
public class TestSloppyPhraseQuery2 extends SearchEquivalenceTestBase {
|
public class TestSloppyPhraseQuery2 extends SearchEquivalenceTestBase {
|
||||||
/** "A B"~N ⊆ "A B"~N+1 */
|
/** "A B"~N ⊆ "A B"~N+1 */
|
||||||
public void testIncreasingSloppiness() throws Exception {
|
public void testIncreasingSloppiness() throws Exception {
|
||||||
Term t1 = randomTerm();
|
Term t1 = randomTerm();
|
||||||
Term t2 = randomTerm();
|
Term t2 = randomTerm();
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
PhraseQuery q1 = new PhraseQuery();
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
q1.add(t1);
|
q1.add(t1);
|
||||||
q1.add(t2);
|
q1.add(t2);
|
||||||
|
q1.setSlop(i);
|
||||||
PhraseQuery q2 = new PhraseQuery();
|
PhraseQuery q2 = new PhraseQuery();
|
||||||
q2.add(t1);
|
q2.add(t1);
|
||||||
q2.add(t2);
|
q2.add(t2);
|
||||||
for (int i = 0; i < 10; i++) {
|
|
||||||
q1.setSlop(i);
|
q1.setSlop(i);
|
||||||
q2.setSlop(i+1);
|
q2.setSlop(i+1);
|
||||||
assertSubsetOf(q1, q2);
|
assertSubsetOf(q1, q2);
|
||||||
|
@ -49,14 +48,14 @@ public class TestSloppyPhraseQuery2 extends SearchEquivalenceTestBase {
|
||||||
public void testIncreasingSloppinessWithHoles() throws Exception {
|
public void testIncreasingSloppinessWithHoles() throws Exception {
|
||||||
Term t1 = randomTerm();
|
Term t1 = randomTerm();
|
||||||
Term t2 = randomTerm();
|
Term t2 = randomTerm();
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
PhraseQuery q1 = new PhraseQuery();
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
q1.add(t1);
|
q1.add(t1);
|
||||||
q1.add(t2, 2);
|
q1.add(t2, 2);
|
||||||
|
q1.setSlop(i);
|
||||||
PhraseQuery q2 = new PhraseQuery();
|
PhraseQuery q2 = new PhraseQuery();
|
||||||
q2.add(t1);
|
q2.add(t1);
|
||||||
q2.add(t2, 2);
|
q2.add(t2, 2);
|
||||||
for (int i = 0; i < 10; i++) {
|
|
||||||
q1.setSlop(i);
|
|
||||||
q2.setSlop(i+1);
|
q2.setSlop(i+1);
|
||||||
assertSubsetOf(q1, q2);
|
assertSubsetOf(q1, q2);
|
||||||
}
|
}
|
||||||
|
@ -67,16 +66,16 @@ public class TestSloppyPhraseQuery2 extends SearchEquivalenceTestBase {
|
||||||
Term t1 = randomTerm();
|
Term t1 = randomTerm();
|
||||||
Term t2 = randomTerm();
|
Term t2 = randomTerm();
|
||||||
Term t3 = randomTerm();
|
Term t3 = randomTerm();
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
PhraseQuery q1 = new PhraseQuery();
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
q1.add(t1);
|
q1.add(t1);
|
||||||
q1.add(t2);
|
q1.add(t2);
|
||||||
q1.add(t3);
|
q1.add(t3);
|
||||||
|
q1.setSlop(i);
|
||||||
PhraseQuery q2 = new PhraseQuery();
|
PhraseQuery q2 = new PhraseQuery();
|
||||||
q2.add(t1);
|
q2.add(t1);
|
||||||
q2.add(t2);
|
q2.add(t2);
|
||||||
q2.add(t3);
|
q2.add(t3);
|
||||||
for (int i = 0; i < 10; i++) {
|
|
||||||
q1.setSlop(i);
|
|
||||||
q2.setSlop(i+1);
|
q2.setSlop(i+1);
|
||||||
assertSubsetOf(q1, q2);
|
assertSubsetOf(q1, q2);
|
||||||
}
|
}
|
||||||
|
@ -89,16 +88,16 @@ public class TestSloppyPhraseQuery2 extends SearchEquivalenceTestBase {
|
||||||
Term t3 = randomTerm();
|
Term t3 = randomTerm();
|
||||||
int pos1 = 1 + random().nextInt(3);
|
int pos1 = 1 + random().nextInt(3);
|
||||||
int pos2 = pos1 + 1 + random().nextInt(3);
|
int pos2 = pos1 + 1 + random().nextInt(3);
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
PhraseQuery q1 = new PhraseQuery();
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
q1.add(t1);
|
q1.add(t1);
|
||||||
q1.add(t2, pos1);
|
q1.add(t2, pos1);
|
||||||
q1.add(t3, pos2);
|
q1.add(t3, pos2);
|
||||||
|
q1.setSlop(i);
|
||||||
PhraseQuery q2 = new PhraseQuery();
|
PhraseQuery q2 = new PhraseQuery();
|
||||||
q2.add(t1);
|
q2.add(t1);
|
||||||
q2.add(t2, pos1);
|
q2.add(t2, pos1);
|
||||||
q2.add(t3, pos2);
|
q2.add(t3, pos2);
|
||||||
for (int i = 0; i < 10; i++) {
|
|
||||||
q1.setSlop(i);
|
|
||||||
q2.setSlop(i+1);
|
q2.setSlop(i+1);
|
||||||
assertSubsetOf(q1, q2);
|
assertSubsetOf(q1, q2);
|
||||||
}
|
}
|
||||||
|
@ -107,14 +106,14 @@ public class TestSloppyPhraseQuery2 extends SearchEquivalenceTestBase {
|
||||||
/** "A A"~N ⊆ "A A"~N+1 */
|
/** "A A"~N ⊆ "A A"~N+1 */
|
||||||
public void testRepetitiveIncreasingSloppiness() throws Exception {
|
public void testRepetitiveIncreasingSloppiness() throws Exception {
|
||||||
Term t = randomTerm();
|
Term t = randomTerm();
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
PhraseQuery q1 = new PhraseQuery();
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
q1.add(t);
|
q1.add(t);
|
||||||
q1.add(t);
|
q1.add(t);
|
||||||
|
q1.setSlop(i);
|
||||||
PhraseQuery q2 = new PhraseQuery();
|
PhraseQuery q2 = new PhraseQuery();
|
||||||
q2.add(t);
|
q2.add(t);
|
||||||
q2.add(t);
|
q2.add(t);
|
||||||
for (int i = 0; i < 10; i++) {
|
|
||||||
q1.setSlop(i);
|
|
||||||
q2.setSlop(i+1);
|
q2.setSlop(i+1);
|
||||||
assertSubsetOf(q1, q2);
|
assertSubsetOf(q1, q2);
|
||||||
}
|
}
|
||||||
|
@ -123,14 +122,14 @@ public class TestSloppyPhraseQuery2 extends SearchEquivalenceTestBase {
|
||||||
/** same as the above with posincr */
|
/** same as the above with posincr */
|
||||||
public void testRepetitiveIncreasingSloppinessWithHoles() throws Exception {
|
public void testRepetitiveIncreasingSloppinessWithHoles() throws Exception {
|
||||||
Term t = randomTerm();
|
Term t = randomTerm();
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
PhraseQuery q1 = new PhraseQuery();
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
q1.add(t);
|
q1.add(t);
|
||||||
q1.add(t, 2);
|
q1.add(t, 2);
|
||||||
|
q1.setSlop(i);
|
||||||
PhraseQuery q2 = new PhraseQuery();
|
PhraseQuery q2 = new PhraseQuery();
|
||||||
q2.add(t);
|
q2.add(t);
|
||||||
q2.add(t, 2);
|
q2.add(t, 2);
|
||||||
for (int i = 0; i < 10; i++) {
|
|
||||||
q1.setSlop(i);
|
|
||||||
q2.setSlop(i+1);
|
q2.setSlop(i+1);
|
||||||
assertSubsetOf(q1, q2);
|
assertSubsetOf(q1, q2);
|
||||||
}
|
}
|
||||||
|
@ -139,16 +138,16 @@ public class TestSloppyPhraseQuery2 extends SearchEquivalenceTestBase {
|
||||||
/** "A A A"~N ⊆ "A A A"~N+1 */
|
/** "A A A"~N ⊆ "A A A"~N+1 */
|
||||||
public void testRepetitiveIncreasingSloppiness3() throws Exception {
|
public void testRepetitiveIncreasingSloppiness3() throws Exception {
|
||||||
Term t = randomTerm();
|
Term t = randomTerm();
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
PhraseQuery q1 = new PhraseQuery();
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
q1.add(t);
|
q1.add(t);
|
||||||
q1.add(t);
|
q1.add(t);
|
||||||
q1.add(t);
|
q1.add(t);
|
||||||
|
q1.setSlop(i);
|
||||||
PhraseQuery q2 = new PhraseQuery();
|
PhraseQuery q2 = new PhraseQuery();
|
||||||
q2.add(t);
|
q2.add(t);
|
||||||
q2.add(t);
|
q2.add(t);
|
||||||
q2.add(t);
|
q2.add(t);
|
||||||
for (int i = 0; i < 10; i++) {
|
|
||||||
q1.setSlop(i);
|
|
||||||
q2.setSlop(i+1);
|
q2.setSlop(i+1);
|
||||||
assertSubsetOf(q1, q2);
|
assertSubsetOf(q1, q2);
|
||||||
}
|
}
|
||||||
|
@ -159,16 +158,16 @@ public class TestSloppyPhraseQuery2 extends SearchEquivalenceTestBase {
|
||||||
Term t = randomTerm();
|
Term t = randomTerm();
|
||||||
int pos1 = 1 + random().nextInt(3);
|
int pos1 = 1 + random().nextInt(3);
|
||||||
int pos2 = pos1 + 1 + random().nextInt(3);
|
int pos2 = pos1 + 1 + random().nextInt(3);
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
PhraseQuery q1 = new PhraseQuery();
|
PhraseQuery q1 = new PhraseQuery();
|
||||||
q1.add(t);
|
q1.add(t);
|
||||||
q1.add(t, pos1);
|
q1.add(t, pos1);
|
||||||
q1.add(t, pos2);
|
q1.add(t, pos2);
|
||||||
|
q1.setSlop(i);
|
||||||
PhraseQuery q2 = new PhraseQuery();
|
PhraseQuery q2 = new PhraseQuery();
|
||||||
q2.add(t);
|
q2.add(t);
|
||||||
q2.add(t, pos1);
|
q2.add(t, pos1);
|
||||||
q2.add(t, pos2);
|
q2.add(t, pos2);
|
||||||
for (int i = 0; i < 10; i++) {
|
|
||||||
q1.setSlop(i);
|
|
||||||
q2.setSlop(i+1);
|
q2.setSlop(i+1);
|
||||||
assertSubsetOf(q1, q2);
|
assertSubsetOf(q1, q2);
|
||||||
}
|
}
|
||||||
|
@ -177,9 +176,9 @@ public class TestSloppyPhraseQuery2 extends SearchEquivalenceTestBase {
|
||||||
/** MultiPhraseQuery~N ⊆ MultiPhraseQuery~N+1 */
|
/** MultiPhraseQuery~N ⊆ MultiPhraseQuery~N+1 */
|
||||||
public void testRandomIncreasingSloppiness() throws Exception {
|
public void testRandomIncreasingSloppiness() throws Exception {
|
||||||
long seed = random().nextLong();
|
long seed = random().nextLong();
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
MultiPhraseQuery q1 = randomPhraseQuery(seed);
|
MultiPhraseQuery q1 = randomPhraseQuery(seed);
|
||||||
MultiPhraseQuery q2 = randomPhraseQuery(seed);
|
MultiPhraseQuery q2 = randomPhraseQuery(seed);
|
||||||
for (int i = 0; i < 10; i++) {
|
|
||||||
q1.setSlop(i);
|
q1.setSlop(i);
|
||||||
q2.setSlop(i+1);
|
q2.setSlop(i+1);
|
||||||
assertSubsetOf(q1, q2);
|
assertSubsetOf(q1, q2);
|
||||||
|
|
Loading…
Reference in New Issue