diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 875753e745c..1db4e413719 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -511,6 +511,15 @@ Bug fixes ArrayIndexOutOfBoundsException (selckin, Robert Muir, Mike McCandless) +API Changes + +* LUCENE-3208: Renamed protected IndexSearcher.createWeight() to expert + public method IndexSearcher.createNormalizedWeight() as this better describes + what this method does. The old method is still there for backwards + compatibility. Query.weight() was deprecated and simply delegates to + IndexSearcher. Both deprecated methods will be removed in Lucene 4.0. + (Uwe Schindler, Robert Muir, Yonik Seeley) + New Features * LUCENE-3140: Added experimental FST implementation to Lucene. diff --git a/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java b/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java index 745117daec0..6a9303c3c80 100644 --- a/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java +++ b/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java @@ -28,10 +28,10 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; +import org.apache.lucene.search.QueryWrapperFilter; /* Tracks the stream of {@link BufferedDeletes}. * When DocumentsWriterPerThread flushes, its buffered @@ -434,18 +434,16 @@ class BufferedDeletesStream { // Delete by query private synchronized long applyQueryDeletes(Iterable queriesIter, SegmentReader reader) throws IOException { long delCount = 0; - IndexSearcher searcher = new IndexSearcher(reader); - assert searcher.getTopReaderContext().isAtomic; - final AtomicReaderContext readerContext = (AtomicReaderContext) searcher.getTopReaderContext(); - try { - for (QueryAndLimit ent : queriesIter) { - Query query = ent.query; - int limit = ent.limit; - Weight weight = query.weight(searcher); - Scorer scorer = weight.scorer(readerContext, Weight.ScorerContext.def()); - if (scorer != null) { + final AtomicReaderContext readerContext = (AtomicReaderContext) reader.getTopReaderContext(); + for (QueryAndLimit ent : queriesIter) { + Query query = ent.query; + int limit = ent.limit; + final DocIdSet docs = new QueryWrapperFilter(query).getDocIdSet(readerContext); + if (docs != null) { + final DocIdSetIterator it = docs.iterator(); + if (it != null) { while(true) { - int doc = scorer.nextDoc(); + int doc = it.nextDoc(); if (doc >= limit) break; @@ -459,8 +457,6 @@ class BufferedDeletesStream { } } } - } finally { - searcher.close(); } return delCount; diff --git a/lucene/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/src/java/org/apache/lucene/search/IndexSearcher.java index 1be99dedf65..ce404f9ff4a 100644 --- a/lucene/src/java/org/apache/lucene/search/IndexSearcher.java +++ b/lucene/src/java/org/apache/lucene/search/IndexSearcher.java @@ -289,7 +289,7 @@ public class IndexSearcher { */ public TopDocs search(Query query, Filter filter, int n) throws IOException { - return search(createWeight(query), filter, n); + return search(createNormalizedWeight(query), filter, n); } /** Lower-level search API. @@ -310,7 +310,7 @@ public class IndexSearcher { */ public void search(Query query, Filter filter, Collector results) throws IOException { - search(leafContexts, createWeight(query), filter, results); + search(leafContexts, createNormalizedWeight(query), filter, results); } /** Lower-level search API. @@ -328,7 +328,7 @@ public class IndexSearcher { */ public void search(Query query, Collector results) throws IOException { - search(leafContexts, createWeight(query), null, results); + search(leafContexts, createNormalizedWeight(query), null, results); } /** Search implementation with arbitrary sorting. Finds @@ -344,7 +344,7 @@ public class IndexSearcher { */ public TopFieldDocs search(Query query, Filter filter, int n, Sort sort) throws IOException { - return search(createWeight(query), filter, n, sort); + return search(createNormalizedWeight(query), filter, n, sort); } /** @@ -357,7 +357,7 @@ public class IndexSearcher { */ public TopFieldDocs search(Query query, int n, Sort sort) throws IOException { - return search(createWeight(query), null, n, sort); + return search(createNormalizedWeight(query), null, n, sort); } /** Expert: Low-level search implementation. Finds the top n @@ -623,7 +623,7 @@ public class IndexSearcher { * entire index. */ public Explanation explain(Query query, int doc) throws IOException { - return explain(createWeight(query), doc); + return explain(createNormalizedWeight(query), doc); } /** Expert: low-level implementation method @@ -665,13 +665,23 @@ public class IndexSearcher { } /** - * creates a weight for query - * @return new weight + * Creates a normalized weight for a top-level {@link Query}. + * The query is rewritten by this method and {@link Query#createWeight} called, + * afterwards the {@link Weight} is normalized. The returned {@code Weight} + * can then directly be used to get a {@link Scorer}. + * @lucene.internal */ - protected Weight createWeight(Query query) throws IOException { - return query.weight(this); + public Weight createNormalizedWeight(Query query) throws IOException { + query = rewrite(query); + Weight weight = query.createWeight(this); + float sum = weight.sumOfSquaredWeights(); + float norm = getSimilarityProvider().queryNorm(sum); + if (Float.isInfinite(norm) || Float.isNaN(norm)) + norm = 1.0f; + weight.normalize(norm); + return weight; } - + /** * Returns this searchers the top-level {@link ReaderContext}. * @see IndexReader#getTopReaderContext() diff --git a/lucene/src/java/org/apache/lucene/search/Query.java b/lucene/src/java/org/apache/lucene/search/Query.java index 40ec80d44a5..714b62836a4 100644 --- a/lucene/src/java/org/apache/lucene/search/Query.java +++ b/lucene/src/java/org/apache/lucene/search/Query.java @@ -91,21 +91,6 @@ public abstract class Query implements Cloneable { throw new UnsupportedOperationException(); } - /** - * Expert: Constructs and initializes a Weight for a top-level query. - */ - public Weight weight(IndexSearcher searcher) throws IOException { - Query query = searcher.rewrite(this); - Weight weight = query.createWeight(searcher); - float sum = weight.sumOfSquaredWeights(); - float norm = searcher.getSimilarityProvider().queryNorm(sum); - if (Float.isInfinite(norm) || Float.isNaN(norm)) - norm = 1.0f; - weight.normalize(norm); - return weight; - } - - /** Expert: called to re-write queries into primitive queries. For example, * a PrefixQuery will be rewritten into a BooleanQuery that consists * of TermQuerys. diff --git a/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java b/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java index 175e36d2d45..c0cb638fb1f 100644 --- a/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java +++ b/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java @@ -52,7 +52,7 @@ public class QueryWrapperFilter extends Filter { // get a private context that is used to rewrite, createWeight and score eventually assert context.reader.getTopReaderContext().isAtomic; final AtomicReaderContext privateContext = (AtomicReaderContext) context.reader.getTopReaderContext(); - final Weight weight = query.weight(new IndexSearcher(privateContext)); + final Weight weight = new IndexSearcher(privateContext).createNormalizedWeight(query); return new DocIdSet() { @Override public DocIdSetIterator iterator() throws IOException { diff --git a/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java b/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java index 8a5ba9abf41..9ea258f30c6 100755 --- a/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java +++ b/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java @@ -187,7 +187,7 @@ public class CustomScoreQuery extends Query { boolean qStrict; public CustomWeight(IndexSearcher searcher) throws IOException { - this.subQueryWeight = subQuery.weight(searcher); + this.subQueryWeight = subQuery.createWeight(searcher); this.valSrcWeights = new Weight[valSrcQueries.length]; for(int i = 0; i < valSrcQueries.length; i++) { this.valSrcWeights[i] = valSrcQueries[i].createWeight(searcher); diff --git a/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java b/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java new file mode 100644 index 00000000000..41541264955 --- /dev/null +++ b/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java @@ -0,0 +1,90 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.concurrent.ExecutorService; +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.IndexReader.ReaderContext; + +/** + * Helper class that adds some extra checks to ensure correct + * usage of {@code IndexSearcher} and {@code Weight}. + * TODO: Extend this by more checks, that's just a start. + */ +public class AssertingIndexSearcher extends IndexSearcher { + public AssertingIndexSearcher(IndexReader r) { + super(r); + } + + public AssertingIndexSearcher(ReaderContext context) { + super(context); + } + + public AssertingIndexSearcher(IndexReader r, ExecutorService ex) { + super(r, ex); + } + + public AssertingIndexSearcher(ReaderContext context, ExecutorService ex) { + super(context, ex); + } + + /** Ensures, that the returned {@code Weight} is not normalized again, which may produce wrong scores. */ + @Override + public Weight createNormalizedWeight(Query query) throws IOException { + final Weight w = super.createNormalizedWeight(query); + return new Weight() { + @Override + public Explanation explain(AtomicReaderContext context, int doc) throws IOException { + return w.explain(context, doc); + } + + @Override + public Query getQuery() { + return w.getQuery(); + } + + @Override + public float getValue() { + return w.getValue(); + } + + @Override + public void normalize(float norm) { + throw new IllegalStateException("Weight already normalized."); + } + + @Override + public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException { + return w.scorer(context, scorerContext); + } + + @Override + public float sumOfSquaredWeights() throws IOException { + throw new IllegalStateException("Weight already normalized."); + } + + @Override + public boolean scoresDocsOutOfOrder() { + return w.scoresDocsOutOfOrder(); + } + }; + } +} diff --git a/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java b/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java index 9cdec64ad23..699af49f51c 100644 --- a/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java +++ b/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java @@ -198,7 +198,7 @@ public class QueryUtils { public static void checkSkipTo(final Query q, final IndexSearcher s) throws IOException { //System.out.println("Checking "+q); final AtomicReaderContext[] readerContextArray = ReaderUtil.leaves(s.getTopReaderContext()); - if (q.weight(s).scoresDocsOutOfOrder()) return; // in this case order of skipTo() might differ from that of next(). + if (s.createNormalizedWeight(q).scoresDocsOutOfOrder()) return; // in this case order of skipTo() might differ from that of next(). final int skip_op = 0; final int next_op = 1; @@ -241,7 +241,7 @@ public class QueryUtils { lastDoc[0] = doc; try { if (scorer == null) { - Weight w = q.weight(s); + Weight w = s.createNormalizedWeight(q); scorer = w.scorer(readerContextArray[leafPtr], ScorerContext.def()); } @@ -286,7 +286,7 @@ public class QueryUtils { if (lastReader[0] != null) { final IndexReader previousReader = lastReader[0]; IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); - Weight w = q.weight(indexSearcher); + Weight w = indexSearcher.createNormalizedWeight(q); Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def()); if (scorer != null) { boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; @@ -312,7 +312,7 @@ public class QueryUtils { // previous reader, hits NO_MORE_DOCS final IndexReader previousReader = lastReader[0]; IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader, false); - Weight w = q.weight(indexSearcher); + Weight w = indexSearcher.createNormalizedWeight(q); Scorer scorer = w.scorer((AtomicReaderContext)previousReader.getTopReaderContext(), ScorerContext.def()); if (scorer != null) { boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; @@ -343,7 +343,7 @@ public class QueryUtils { try { long startMS = System.currentTimeMillis(); for (int i=lastDoc[0]+1; i<=doc; i++) { - Weight w = q.weight(s); + Weight w = s.createNormalizedWeight(q); Scorer scorer = w.scorer(context[leafPtr], ScorerContext.def()); Assert.assertTrue("query collected "+doc+" but skipTo("+i+") says no more docs!",scorer.advance(i) != DocIdSetIterator.NO_MORE_DOCS); Assert.assertEquals("query collected "+doc+" but skipTo("+i+") got to "+scorer.docID(),doc,scorer.docID()); @@ -370,7 +370,7 @@ public class QueryUtils { if (lastReader[0] != null) { final IndexReader previousReader = lastReader[0]; IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); - Weight w = q.weight(indexSearcher); + Weight w = indexSearcher.createNormalizedWeight(q); Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def()); if (scorer != null) { boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; @@ -394,7 +394,7 @@ public class QueryUtils { // previous reader, hits NO_MORE_DOCS final IndexReader previousReader = lastReader[0]; IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); - Weight w = q.weight(indexSearcher); + Weight w = indexSearcher.createNormalizedWeight(q); Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def()); if (scorer != null) { boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; diff --git a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java index 1df2902507b..e4243c332c2 100644 --- a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java @@ -55,6 +55,7 @@ import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.FieldCache.CacheEntry; +import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; @@ -1231,13 +1232,11 @@ public abstract class LuceneTestCase extends Assert { * with one that returns null for getSequentialSubReaders. */ public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) throws IOException { - if (random.nextBoolean()) { if (maybeWrap && rarely()) { - return new IndexSearcher(new SlowMultiReaderWrapper(r)); - } else { - return new IndexSearcher(r); + r = new SlowMultiReaderWrapper(r); } + return random.nextBoolean() ? new AssertingIndexSearcher(r) : new AssertingIndexSearcher(r.getTopReaderContext()); } else { int threads = 0; final ExecutorService ex = (random.nextBoolean()) ? null @@ -1246,20 +1245,31 @@ public abstract class LuceneTestCase extends Assert { if (ex != null && VERBOSE) { System.out.println("NOTE: newSearcher using ExecutorService with " + threads + " threads"); } - return new IndexSearcher(r.getTopReaderContext(), ex) { - @Override - public void close() throws IOException { - super.close(); - if (ex != null) { - ex.shutdown(); - try { - ex.awaitTermination(1000, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } + return random.nextBoolean() ? + new AssertingIndexSearcher(r, ex) { + @Override + public void close() throws IOException { + super.close(); + shutdownExecutorService(ex); } - } - }; + } : new AssertingIndexSearcher(r.getTopReaderContext(), ex) { + @Override + public void close() throws IOException { + super.close(); + shutdownExecutorService(ex); + } + }; + } + } + + static void shutdownExecutorService(ExecutorService ex) { + if (ex != null) { + ex.shutdown(); + try { + ex.awaitTermination(1000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } } } diff --git a/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java b/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java index 272384b0e01..e8a6b69a948 100644 --- a/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java +++ b/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java @@ -173,7 +173,7 @@ public class TestDisjunctionMaxQuery extends LuceneTestCase { QueryUtils.check(random, dq, s); assertTrue(s.getTopReaderContext().isAtomic); - final Weight dw = dq.weight(s); + final Weight dw = s.createNormalizedWeight(dq); final Scorer ds = dw.scorer((AtomicReaderContext)s.getTopReaderContext(), ScorerContext.def()); final boolean skipOk = ds.advance(3) != DocIdSetIterator.NO_MORE_DOCS; if (skipOk) { @@ -188,7 +188,7 @@ public class TestDisjunctionMaxQuery extends LuceneTestCase { dq.add(tq("dek", "DOES_NOT_EXIST")); assertTrue(s.getTopReaderContext().isAtomic); QueryUtils.check(random, dq, s); - final Weight dw = dq.weight(s); + final Weight dw = s.createNormalizedWeight(dq); final Scorer ds = dw.scorer((AtomicReaderContext)s.getTopReaderContext(), ScorerContext.def()); assertTrue("firsttime skipTo found no match", ds.advance(3) != DocIdSetIterator.NO_MORE_DOCS); diff --git a/lucene/src/test/org/apache/lucene/search/TestTermScorer.java b/lucene/src/test/org/apache/lucene/search/TestTermScorer.java index d4f919856b4..fbdbb0c0b03 100644 --- a/lucene/src/test/org/apache/lucene/search/TestTermScorer.java +++ b/lucene/src/test/org/apache/lucene/search/TestTermScorer.java @@ -73,7 +73,7 @@ public class TestTermScorer extends LuceneTestCase { Term allTerm = new Term(FIELD, "all"); TermQuery termQuery = new TermQuery(allTerm); - Weight weight = termQuery.weight(indexSearcher); + Weight weight = indexSearcher.createNormalizedWeight(termQuery); assertTrue(indexSearcher.getTopReaderContext().isAtomic); Scorer ts = weight.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def().scoreDocsInOrder(true).topScorer(true)); // we have 2 documents with the term all in them, one document for all the @@ -134,7 +134,7 @@ public class TestTermScorer extends LuceneTestCase { Term allTerm = new Term(FIELD, "all"); TermQuery termQuery = new TermQuery(allTerm); - Weight weight = termQuery.weight(indexSearcher); + Weight weight = indexSearcher.createNormalizedWeight(termQuery); assertTrue(indexSearcher.getTopReaderContext().isAtomic); Scorer ts = weight.scorer((AtomicReaderContext) indexSearcher.getTopReaderContext(), ScorerContext.def().scoreDocsInOrder(true).topScorer(true)); assertTrue("next did not return a doc", @@ -152,7 +152,7 @@ public class TestTermScorer extends LuceneTestCase { Term allTerm = new Term(FIELD, "all"); TermQuery termQuery = new TermQuery(allTerm); - Weight weight = termQuery.weight(indexSearcher); + Weight weight = indexSearcher.createNormalizedWeight(termQuery); assertTrue(indexSearcher.getTopReaderContext().isAtomic); Scorer ts = weight.scorer((AtomicReaderContext) indexSearcher.getTopReaderContext(), ScorerContext.def().scoreDocsInOrder(true).topScorer(true)); diff --git a/lucene/src/test/org/apache/lucene/search/TestTopDocsMerge.java b/lucene/src/test/org/apache/lucene/search/TestTopDocsMerge.java index 906a9dac1e9..7c459f7fd16 100644 --- a/lucene/src/test/org/apache/lucene/search/TestTopDocsMerge.java +++ b/lucene/src/test/org/apache/lucene/search/TestTopDocsMerge.java @@ -197,7 +197,7 @@ public class TestTopDocsMerge extends LuceneTestCase { } // ... then all shards: - final Weight w = query.weight(searcher); + final Weight w = searcher.createNormalizedWeight(query); final TopDocs[] shardHits = new TopDocs[subSearchers.length]; for(int shardIDX=0;shardIDX