LUCENE-6527: Queries now get a dummy Similarity when scores are not needed in order to not load unnecessary information like norms.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1684502 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -92,6 +92,9 @@ Bug fixes
 * LUCENE-6482: Fix class loading deadlock relating to Codec initialization,
   default codec and SPI discovery.  (Shikhar Bhushan, Uwe Schindler)
 
+* LUCENE-6527: Queries now get a dummy Similarity when scores are not needed
+  in order to not load unnecessary information like norms. (Adrien Grand)
+
 Changes in Runtime Behavior
 
 * LUCENE-6501: The subreader structure in ParallelCompositeReader

lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java

@@ -50,7 +50,7 @@ public class BooleanWeight extends Weight {
     super(query);
     this.query = query;
     this.needsScores = needsScores;
-    this.similarity = searcher.getSimilarity();
+    this.similarity = searcher.getSimilarity(needsScores);
     weights = new ArrayList<>(query.clauses().size());
     for (int i = 0 ; i < query.clauses().size(); i++) {
       BooleanClause c = query.clauses().get(i);

lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java

@@ -31,6 +31,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
 import org.apache.lucene.index.DirectoryReader; // javadocs
+import org.apache.lucene.index.FieldInvertState;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.IndexWriter; // javadocs
@@ -45,6 +46,7 @@ import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.store.NIOFSDirectory;    // javadoc
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.ThreadInterruptedException;
 
 /** Implements search over a single IndexReader.
@@ -74,6 +76,54 @@ import org.apache.lucene.util.ThreadInterruptedException;
  */
 public class IndexSearcher {
 
+  /** A search-time {@link Similarity} that does not make use of scoring factors
+   *  and may be used when scores are not needed. */
+  private static final Similarity NON_SCORING_SIMILARITY = new Similarity() {
+
+    @Override
+    public long computeNorm(FieldInvertState state) {
+      throw new UnsupportedOperationException("This Similarity may only be used for searching, not indexing");
+    }
+
+    @Override
+    public SimWeight computeWeight(float queryBoost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+      return new SimWeight() {
+
+        @Override
+        public float getValueForNormalization() {
+          return 1f;
+        }
+
+        @Override
+        public void normalize(float queryNorm, float topLevelBoost) {}
+
+      };
+    }
+
+    @Override
+    public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
+      return new SimScorer() {
+
+        @Override
+        public float score(int doc, float freq) {
+          return 0f;
+        }
+
+        @Override
+        public float computeSlopFactor(int distance) {
+          return 1f;
+        }
+
+        @Override
+        public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
+          return 1f;
+        }
+
+      };
+    }
+
+  };
+
   // disabled by default
   private static QueryCache DEFAULT_QUERY_CACHE = null;
   private static QueryCachingPolicy DEFAULT_CACHING_POLICY = new UsageTrackingQueryCachingPolicy();
@@ -100,7 +150,7 @@ public class IndexSearcher {
    * Expert: returns a default Similarity instance.
    * In general, this method is only called to initialize searchers and writers.
    * User code and query implementations should respect
-   * {@link IndexSearcher#getSimilarity()}.
+   * {@link IndexSearcher#getSimilarity(boolean)}.
    * @lucene.internal
    */
   public static Similarity getDefaultSimilarity() {
@@ -273,8 +323,15 @@ public class IndexSearcher {
     this.similarity = similarity;
   }
 
-  public Similarity getSimilarity() {
+  /** Expert: Get the {@link Similarity} to use to compute scores. When
-    return similarity;
+   *  {@code needsScores} is {@code false}, this method will return a simple
+   *  {@link Similarity} that does not leverage scoring factors such as norms.
+   *  When {@code needsScores} is {@code true}, this returns the
+   *  {@link Similarity} that has been set through {@link #setSimilarity(Similarity)}
+   *  or the {@link #getDefaultSimilarity()} default {@link Similarity} if none
+   *  has been set explicitely. */
+  public Similarity getSimilarity(boolean needsScores) {
+    return needsScores ? similarity : NON_SCORING_SIMILARITY;
   }
 
   /**
@@ -625,7 +682,7 @@ public class IndexSearcher {
     query = rewrite(query);
     Weight weight = createWeight(query, needsScores);
     float v = weight.getValueForNormalization();
-    float norm = getSimilarity().queryNorm(v);
+    float norm = getSimilarity(needsScores).queryNorm(v);
     if (Float.isInfinite(norm) || Float.isNaN(norm)) {
       norm = 1.0f;
     }

lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java

@@ -138,7 +138,7 @@ public class MultiPhraseQuery extends Query {
       throws IOException {
       super(MultiPhraseQuery.this);
       this.needsScores = needsScores;
-      this.similarity = searcher.getSimilarity();
+      this.similarity = searcher.getSimilarity(needsScores);
       final IndexReaderContext context = searcher.getTopReaderContext();
       
       // compute idf

lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java

@@ -255,7 +255,7 @@ public class PhraseQuery extends Query {
         throw new IllegalStateException("PhraseWeight requires that the first position is 0, call rewrite first");
       }
       this.needsScores = needsScores;
-      this.similarity = searcher.getSimilarity();
+      this.similarity = searcher.getSimilarity(needsScores);
       final IndexReaderContext context = searcher.getTopReaderContext();
       states = new TermContext[terms.size()];
       TermStatistics termStats[] = new TermStatistics[terms.size()];

lucene/core/src/java/org/apache/lucene/search/TermQuery.java

@@ -58,7 +58,7 @@ public class TermQuery extends Query {
       // checked with a real exception in TermQuery constructor
       assert termStates.hasOnlyRealTerms();
       this.termStates = termStates;
-      this.similarity = searcher.getSimilarity();
+      this.similarity = searcher.getSimilarity(needsScores);
 
       final CollectionStatistics collectionStats;
       final TermStatistics termStats;

lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java

@@ -84,10 +84,6 @@ public abstract class SpanPositionCheckQuery extends SpanQuery implements Clonea
       this.matchWeight = matchWeight;
     }
 
-    public SpanPositionCheckWeight(SpanWeight matchWeight, IndexSearcher searcher, Map<Term, TermContext> terms) throws IOException {
-      this(matchWeight, searcher, terms, SpanCollectorFactory.NO_OP_FACTORY);
-    }
-
     @Override
     public void extractTerms(Set<Term> terms) {
       matchWeight.extractTerms(terms);

lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java

@@ -56,7 +56,7 @@ public abstract class SpanWeight extends Weight {
   public SpanWeight(SpanQuery query, IndexSearcher searcher, Map<Term, TermContext> termContexts, SpanCollectorFactory collectorFactory) throws IOException {
     super(query);
     this.field = query.getField();
-    this.similarity = searcher.getSimilarity();
+    this.similarity = searcher.getSimilarity(termContexts != null);
     this.collectorFactory = collectorFactory;
     this.simWeight = buildSimWeight(query, searcher, termContexts);
   }
@@ -71,7 +71,7 @@ public abstract class SpanWeight extends Weight {
       i++;
     }
     CollectionStatistics collectionStats = searcher.collectionStatistics(query.getField());
-    return searcher.getSimilarity().computeWeight(query.getBoost(), collectionStats, termStats);
+    return searcher.getSimilarity(true).computeWeight(query.getBoost(), collectionStats, termStats);
   }
 
   /**

lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java

@@ -240,7 +240,7 @@ public class TestBoolean2 extends LuceneTestCase {
     query.add(new TermQuery(new Term(field, "zz")), BooleanClause.Occur.SHOULD);
 
     int[] expDocNrs = {2, 3};
-    Similarity oldSimilarity = searcher.getSimilarity();
+    Similarity oldSimilarity = searcher.getSimilarity(true);
     try {
       searcher.setSimilarity(new DefaultSimilarity(){
         @Override
@@ -276,7 +276,7 @@ public class TestBoolean2 extends LuceneTestCase {
         QueryUtils.check(random(), q1,searcher); // baseline sim
         try {
           // a little hackish, QueryUtils.check is too costly to do on bigSearcher in this loop.
-          searcher.setSimilarity(bigSearcher.getSimilarity()); // random sim
+          searcher.setSimilarity(bigSearcher.getSimilarity(true)); // random sim
           QueryUtils.check(random(), q1, searcher);
         } finally {
           searcher.setSimilarity(new DefaultSimilarity()); // restore

lucene/core/src/test/org/apache/lucene/search/TestBooleanMinShouldMatch.java

@@ -397,7 +397,7 @@ public class TestBooleanMinShouldMatch extends LuceneTestCase {
     }
 
     public void testRewriteCoord1() throws Exception {
-      final Similarity oldSimilarity = s.getSimilarity();
+      final Similarity oldSimilarity = s.getSimilarity(true);
       try {
         s.setSimilarity(new DefaultSimilarity() {
           @Override
@@ -419,7 +419,7 @@ public class TestBooleanMinShouldMatch extends LuceneTestCase {
     }
     
     public void testRewriteNegate() throws Exception {
-      final Similarity oldSimilarity = s.getSimilarity();
+      final Similarity oldSimilarity = s.getSimilarity(true);
       try {
         s.setSimilarity(new DefaultSimilarity() {
           @Override

lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java

@@ -68,7 +68,7 @@ public class TestDocValuesScoring extends LuceneTestCase {
     
     // no boosting
     IndexSearcher searcher1 = newSearcher(ir, false);
-    final Similarity base = searcher1.getSimilarity();
+    final Similarity base = searcher1.getSimilarity(true);
     // boosting
     IndexSearcher searcher2 = newSearcher(ir, false);
     searcher2.setSimilarity(new PerFieldSimilarityWrapper() {

lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java

@@ -183,4 +183,30 @@ public class TestTermScorer extends LuceneTestCase {
     }
   }
 
+  public void testDoesNotLoadNorms() throws IOException {
+    Term allTerm = new Term(FIELD, "all");
+    TermQuery termQuery = new TermQuery(allTerm);
+    
+    LeafReader forbiddenNorms = new FilterLeafReader(indexReader) {
+      @Override
+      public NumericDocValues getNormValues(String field) throws IOException {
+        fail("Norms should not be loaded");
+        // unreachable
+        return null;
+      }
+    };
+    IndexSearcher indexSearcher = newSearcher(forbiddenNorms);
+    
+    Weight weight = indexSearcher.createNormalizedWeight(termQuery, true);
+    try {
+      weight.scorer(forbiddenNorms.getContext(), null).nextDoc();
+      fail("Should load norms");
+    } catch (AssertionError e) {
+      // ok
+    }
+    
+    weight = indexSearcher.createNormalizedWeight(termQuery, false);
+    // should not fail this time since norms are not necessary
+    weight.scorer(forbiddenNorms.getContext(), null).nextDoc();
+  }
 }

lucene/core/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java

@@ -244,7 +244,7 @@ public class TestFieldMaskingSpanQuery extends LuceneTestCase {
   
   public void testSimple2() throws Exception {
     assumeTrue("Broken scoring: LUCENE-3723", 
-        searcher.getSimilarity() instanceof TFIDFSimilarity);
+        searcher.getSimilarity(true) instanceof TFIDFSimilarity);
     SpanQuery q1 = new SpanTermQuery(new Term("gender", "female"));
     SpanQuery q2 = new SpanTermQuery(new Term("last", "smith"));
     SpanQuery q = new SpanNearQuery(new SpanQuery[]
@@ -300,7 +300,7 @@ public class TestFieldMaskingSpanQuery extends LuceneTestCase {
   
   public void testSpans2() throws Exception {
     assumeTrue("Broken scoring: LUCENE-3723",
-        searcher.getSimilarity() instanceof TFIDFSimilarity);
+        searcher.getSimilarity(true) instanceof TFIDFSimilarity);
     SpanQuery qA1 = new SpanTermQuery(new Term("gender", "female"));
     SpanQuery qA2 = new SpanTermQuery(new Term("first",  "james"));
     SpanQuery qA  = new SpanOrQuery(qA1, new FieldMaskingSpanQuery(qA2, "gender"));

lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java

@@ -293,7 +293,7 @@ public class TestSpans extends LuceneTestCase {
         }
       };
   
-      final Similarity oldSim = searcher.getSimilarity();
+      final Similarity oldSim = searcher.getSimilarity(true);
       Scorer spanScorer;
       try {
         searcher.setSimilarity(sim);

lucene/misc/src/test/org/apache/lucene/search/TestDiversifiedTopDocsCollector.java

@@ -373,7 +373,7 @@ public class TestDiversifiedTopDocsCollector extends LuceneTestCase {
     artistDocValues = ar.getSortedDocValues("artist");
 
     // All searches sort by song popularity 
-    final Similarity base = searcher.getSimilarity();
+    final Similarity base = searcher.getSimilarity(true);
     searcher.setSimilarity(new DocValueSimilarity(base, "weeksAtNumberOne"));
   }
 

lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java

@@ -48,7 +48,7 @@ public class IDFValueSource extends DocFreqValueSource {
   @Override
   public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
     IndexSearcher searcher = (IndexSearcher)context.get("searcher");
-    TFIDFSimilarity sim = asTFIDF(searcher.getSimilarity(), field);
+    TFIDFSimilarity sim = asTFIDF(searcher.getSimilarity(true), field);
     if (sim == null) {
       throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
     }

lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java

@@ -58,7 +58,7 @@ public class NormValueSource extends ValueSource {
   @Override
   public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
     IndexSearcher searcher = (IndexSearcher)context.get("searcher");
-    final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(), field);
+    final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(true), field);
     if (similarity == null) {
       throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
     }

lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java

@@ -54,7 +54,7 @@ public class TFValueSource extends TermFreqValueSource {
     Fields fields = readerContext.reader().fields();
     final Terms terms = fields.terms(indexedField);
     IndexSearcher searcher = (IndexSearcher)context.get("searcher");
-    final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(), indexedField);
+    final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(true), indexedField);
     if (similarity == null) {
       throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
     }

lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java

@@ -85,7 +85,7 @@ public class TestLongNormValueSource extends LuceneTestCase {
   }
 
   public void testNorm() throws Exception {
-    Similarity saved = searcher.getSimilarity();
+    Similarity saved = searcher.getSimilarity(true);
     try {
       // no norm field (so agnostic to indexed similarity)
       searcher.setSimilarity(sim);

lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java

@@ -221,7 +221,7 @@ public class TestValueSources extends LuceneTestCase {
   }
   
   public void testIDF() throws Exception {
-    Similarity saved = searcher.getSimilarity();
+    Similarity saved = searcher.getSimilarity(true);
     try {
       searcher.setSimilarity(new DefaultSimilarity());
       ValueSource vs = new IDFValueSource("bogus", "bogus", "text", new BytesRef("test"));
@@ -339,7 +339,7 @@ public class TestValueSources extends LuceneTestCase {
   }
   
   public void testNorm() throws Exception {
-    Similarity saved = searcher.getSimilarity();
+    Similarity saved = searcher.getSimilarity(true);
     try {
       // no norm field (so agnostic to indexed similarity)
       searcher.setSimilarity(new DefaultSimilarity());
@@ -391,7 +391,7 @@ public class TestValueSources extends LuceneTestCase {
   }
 
   public void testQuery() throws Exception {
-    Similarity saved = searcher.getSimilarity();
+    Similarity saved = searcher.getSimilarity(true);
 
     try {
       searcher.setSimilarity(new DefaultSimilarity());
@@ -498,7 +498,7 @@ public class TestValueSources extends LuceneTestCase {
   }
   
   public void testTF() throws Exception {
-    Similarity saved = searcher.getSimilarity();
+    Similarity saved = searcher.getSimilarity(true);
     try {
       // no norm field (so agnostic to indexed similarity)
       searcher.setSimilarity(new DefaultSimilarity());

lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java

@@ -342,7 +342,7 @@ public class TermAutomatonQuery extends Query {
       this.automaton = automaton;
       this.searcher = searcher;
       this.termStates = termStates;
-      this.similarity = searcher.getSimilarity();
+      this.similarity = searcher.getSimilarity(true);
       List<TermStatistics> allTermStats = new ArrayList<>();
       for(Map.Entry<Integer,BytesRef> ent : idToTerm.entrySet()) {
         Integer termID = ent.getKey();

lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java

@@ -195,7 +195,7 @@ public class QueryUtils {
     };
 
     IndexSearcher out = LuceneTestCase.newSearcher(new FCInvisibleMultiReader(readers));
-    out.setSimilarity(s.getSimilarity());
+    out.setSimilarity(s.getSimilarity(true));
     return out;
   }
   
@@ -411,7 +411,7 @@ public class QueryUtils {
             if (lastReader[0] != null) {
               final LeafReader previousReader = lastReader[0];
               IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader);
-              indexSearcher.setSimilarity(s.getSimilarity());
+              indexSearcher.setSimilarity(s.getSimilarity(true));
               Weight w = indexSearcher.createNormalizedWeight(q, true);
               LeafReaderContext ctx = (LeafReaderContext)indexSearcher.getTopReaderContext();
               Scorer scorer = w.scorer(ctx, ctx.reader().getLiveDocs());
@@ -433,7 +433,7 @@ public class QueryUtils {
           // previous reader, hits NO_MORE_DOCS
           final LeafReader previousReader = lastReader[0];
           IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader, false);
-          indexSearcher.setSimilarity(s.getSimilarity());
+          indexSearcher.setSimilarity(s.getSimilarity(true));
           Weight w = indexSearcher.createNormalizedWeight(q, true);
           LeafReaderContext ctx = previousReader.getContext();
           Scorer scorer = w.scorer(ctx, ctx.reader().getLiveDocs());
@@ -498,7 +498,7 @@ public class QueryUtils {
         if (lastReader[0] != null) {
           final LeafReader previousReader = lastReader[0];
           IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader);
-          indexSearcher.setSimilarity(s.getSimilarity());
+          indexSearcher.setSimilarity(s.getSimilarity(true));
           Weight w = indexSearcher.createNormalizedWeight(q, true);
           Scorer scorer = w.scorer((LeafReaderContext)indexSearcher.getTopReaderContext(), previousReader.getLiveDocs());
           if (scorer != null) {
@@ -519,7 +519,7 @@ public class QueryUtils {
       // previous reader, hits NO_MORE_DOCS
       final LeafReader previousReader = lastReader[0];
       IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader);
-      indexSearcher.setSimilarity(s.getSimilarity());
+      indexSearcher.setSimilarity(s.getSimilarity(true));
       Weight w = indexSearcher.createNormalizedWeight(q, true);
       Scorer scorer = w.scorer((LeafReaderContext)indexSearcher.getTopReaderContext(), previousReader.getLiveDocs());
       if (scorer != null) {

solr/core/src/test/org/apache/solr/search/similarities/BaseSimilarityTestCase.java

@@ -30,7 +30,7 @@ public abstract class BaseSimilarityTestCase extends SolrTestCaseJ4 {
   protected Similarity getSimilarity(String field) {
     SolrCore core = h.getCore();
     RefCounted<SolrIndexSearcher> searcher = core.getSearcher();
-    Similarity sim = searcher.get().getSimilarity();
+    Similarity sim = searcher.get().getSimilarity(true);
     searcher.decref();
     while (sim instanceof PerFieldSimilarityWrapper) {
       sim = ((PerFieldSimilarityWrapper)sim).get(field);

solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java

@@ -354,9 +354,9 @@ public class DocumentBuilderTest extends SolrTestCaseJ4 {
 
       assertTrue("similarity doesn't extend DefaultSimilarity, " + 
                  "config or defaults have changed since test was written",
-                 searcher.getSimilarity() instanceof DefaultSimilarity);
+                 searcher.getSimilarity(true) instanceof DefaultSimilarity);
 
-      DefaultSimilarity sim = (DefaultSimilarity) searcher.getSimilarity();
+      DefaultSimilarity sim = (DefaultSimilarity) searcher.getSimilarity(true);
       
       NumericDocValues titleNorms = reader.getNormValues("title");
       NumericDocValues fooNorms = reader.getNormValues("foo_t");