SOLR-15038: Add elevateOnlyDocsMatchingQuery and collectElevatedDocsWhenCollapsing parameters to query elevation.

Closes #2134

gradle/testing/randomization/policies/solr-tests.policy

@@ -22,7 +22,7 @@
 grant {
   // 3rd party jar resources (where symlinks are not supported), test-files/ resources
   permission java.io.FilePermission "${common.dir}${/}-", "read";
-  permission java.io.FilePermission "${common.dir}${/}..${/}solr${/}-", "read";
+  permission java.io.FilePermission "${common.dir}${/}..${/}solr${/}-", "read,write";
 
   // system jar resources
   permission java.io.FilePermission "${java.home}${/}-", "read";

solr/CHANGES.txt

@@ -239,6 +239,9 @@ Improvements
 
 * SOLR-15101: Add "list" and "delete" APIs for managing incremental backups (Jason Gerlowski, shalin, Cao Manh Dat)
 
+* SOLR-15038: Add elevateOnlyDocsMatchingQuery and collectElevatedDocsWhenCollapsing parameters to query elevation.
+  (Dennis Berger, Tobias Kässmann via Bruno Roustant)
+
 Optimizations
 ---------------------
 * SOLR-15079: Block Collapse - Faster collapse code when groups are co-located via Block Join style nested doc indexing.

solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java

@@ -509,7 +509,10 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
       rb.setQuery(new BoostQuery(elevation.includeQuery, 0f));
     } else {
       BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
-      queryBuilder.add(rb.getQuery(), BooleanClause.Occur.SHOULD);
+      BooleanClause.Occur queryOccurrence =
+              params.getBool(QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, false) ?
+                      BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
+      queryBuilder.add(rb.getQuery(), queryOccurrence);
       queryBuilder.add(new BoostQuery(elevation.includeQuery, 0f), BooleanClause.Occur.SHOULD);
       if (elevation.excludeQueries != null) {
         if (params.getBool(QueryElevationParams.MARK_EXCLUDES, false)) {

solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java

@@ -146,6 +146,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
    */
   public static final String HINT_BLOCK = "block";
 
+  /**
+   * If elevation is used in combination with the collapse query parser, we can define that we only want to return the
+   * representative and not all elevated docs by setting this parameter to false (true by default).
+   */
+  public static String COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING = "collectElevatedDocsWhenCollapsing";
+
   /**
    * @deprecated use {@link NullPolicy} instead.
    */
@@ -585,6 +591,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
     private int nullPolicy;
     private float nullScore = -Float.MAX_VALUE;
     private int nullDoc = -1;
+    private boolean collectElevatedDocsWhenCollapsing;
     private FloatArrayList nullScores;
 
     private final BoostedDocsCollector boostedDocsCollector;
@@ -594,9 +601,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
                              DocValuesProducer collapseValuesProducer,
                              int nullPolicy,
                              IntIntHashMap boostDocsMap,
-                             IndexSearcher searcher) throws IOException {
+                             IndexSearcher searcher,
+                             boolean collectElevatedDocsWhenCollapsing) throws IOException {
       this.maxDoc = maxDoc;
       this.contexts = new LeafReaderContext[segments];
+      this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;
       List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
       for(int i=0; i<con.size(); i++) {
         contexts[i] = con.get(i);
@@ -653,12 +662,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
           ord = -1;
         }
       }
-      
+
-      // Check to see if we have documents boosted by the QueryElevationComponent
+      if (collectElevatedDocsWhenCollapsing) {
-      if (0 <= ord) {
+        // Check to see if we have documents boosted by the QueryElevationComponent
-        if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;
+        if (0 <= ord) {
-      } else {
+          if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;
-        if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
+        } else {
+          if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
+        }
       }
 
       if(ord > -1) {
@@ -785,6 +796,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
     private int nullDoc = -1;
     private FloatArrayList nullScores;
     private String field;
+    private boolean collectElevatedDocsWhenCollapsing;
 
     private final BoostedDocsCollector boostedDocsCollector;
     
@@ -794,9 +806,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
                              int size,
                              String field,
                              IntIntHashMap boostDocsMap,
-                             IndexSearcher searcher) {
+                             IndexSearcher searcher,
+                             boolean collectElevatedDocsWhenCollapsing) {
       this.maxDoc = maxDoc;
       this.contexts = new LeafReaderContext[segments];
+      this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;
       List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
       for(int i=0; i<con.size(); i++) {
         contexts[i] = con.get(i);
@@ -827,8 +841,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
       final int globalDoc = docBase+contextDoc;
       if (collapseValues.advanceExact(contextDoc)) {
         final int collapseValue = (int) collapseValues.longValue();
-        // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
+
-        if (boostedDocsCollector.collectIfBoosted(collapseValue, globalDoc)) return;
+        if (collectElevatedDocsWhenCollapsing) {
+          // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
+          if (boostedDocsCollector.collectIfBoosted(collapseValue, globalDoc)) return;
+        }
 
         float score = scorer.score();
         final int idx;
@@ -847,9 +864,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
         }
 
       } else { // Null Group...
-        
+
-        // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
+        if (collectElevatedDocsWhenCollapsing){
-        if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
+          // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
+          if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
+        }
 
         if(nullPolicy == NullPolicy.COLLAPSE.getCode()) {
           float score = scorer.score();
@@ -958,7 +977,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
     private OrdFieldValueStrategy collapseStrategy;
     private boolean needsScores4Collapsing;
     private boolean needsScores;
-    
+
+    private boolean collectElevatedDocsWhenCollapsing;
+
     private final BoostedDocsCollector boostedDocsCollector;
 
     public OrdFieldValueCollector(int maxDoc,
@@ -971,10 +992,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
                                   boolean needsScores,
                                   FieldType fieldType,
                                   IntIntHashMap boostDocsMap,
-                                  FunctionQuery funcQuery, IndexSearcher searcher) throws IOException{
+                                  FunctionQuery funcQuery, IndexSearcher searcher,
+                                  boolean collectElevatedDocsWhenCollapsing) throws IOException{
 
       assert ! GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type);
 
+      this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;
       this.maxDoc = maxDoc;
       this.contexts = new LeafReaderContext[segments];
       List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
@@ -1053,12 +1076,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
           ord = segmentValues.ordValue();
         }
       }
-      
+
-      // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
+      if (collectElevatedDocsWhenCollapsing){
-      if (-1 == ord) {
+        // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
-        if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
+        if (-1 == ord) {
-      } else {
+          if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
-        if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;
+        } else {
+          if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;
+        }
       }
       
       collapseStrategy.collapse(ord, contextDoc, globalDoc);
@@ -1166,6 +1191,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
     private String collapseField;
     
     private final BoostedDocsCollector boostedDocsCollector;
+    private boolean collectElevatedDocsWhenCollapsing;
 
     public IntFieldValueCollector(int maxDoc,
                                   int size,
@@ -1179,7 +1205,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
                                   FieldType fieldType,
                                   IntIntHashMap boostDocsMap,
                                   FunctionQuery funcQuery,
-                                  IndexSearcher searcher) throws IOException{
+                                  IndexSearcher searcher,
+                                  boolean collectElevatedDocsWhenCollapsing) throws IOException{
+      this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;
 
       assert ! GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type);
 
@@ -1243,10 +1271,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
         collapseStrategy.collapse(collapseKey, contextDoc, globalDoc);
         
       } else { // Null Group...
-        
-        // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
-        if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
 
+        if (collectElevatedDocsWhenCollapsing){
+          // Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
+          if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
+        }
         if (NullPolicy.IGNORE.getCode() != nullPolicy) {
           collapseStrategy.collapseNullGroup(contextDoc, globalDoc);
         }
@@ -1894,20 +1923,23 @@ public class CollapsingQParserPlugin extends QParserPlugin {
       int maxDoc = searcher.maxDoc();
       int leafCount = searcher.getTopReaderContext().leaves().size();
 
+      SolrRequestInfo req = SolrRequestInfo.getRequestInfo();
+      boolean collectElevatedDocsWhenCollapsing = req != null && req.getReq().getParams().getBool(COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, true);
+
       if (GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type)) {
 
         if (collapseFieldType instanceof StrField) {
           if (blockCollapse) {
             return new BlockOrdScoreCollector(collapseField, nullPolicy, boostDocs);
           }
-          return new OrdScoreCollector(maxDoc, leafCount, docValuesProducer, nullPolicy, boostDocs, searcher);
+          return new OrdScoreCollector(maxDoc, leafCount, docValuesProducer, nullPolicy, boostDocs, searcher, collectElevatedDocsWhenCollapsing);
 
         } else if (isNumericCollapsible(collapseFieldType)) {
           if (blockCollapse) {
             return new BlockIntScoreCollector(collapseField, nullPolicy, boostDocs);
           }
 
-          return new IntScoreCollector(maxDoc, leafCount, nullPolicy, size, collapseField, boostDocs, searcher);
+          return new IntScoreCollector(maxDoc, leafCount, nullPolicy, size, collapseField, boostDocs, searcher, collectElevatedDocsWhenCollapsing);
 
         } else {
           throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
@@ -1937,7 +1969,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
                                             minMaxFieldType,
                                             boostDocs,
                                             funcQuery,
-                                            searcher);
+                                            searcher,
+                                            collectElevatedDocsWhenCollapsing);
 
         } else if (isNumericCollapsible(collapseFieldType)) {
 
@@ -1962,7 +1995,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
                                             minMaxFieldType,
                                             boostDocs,
                                             funcQuery,
-                                            searcher);
+                                            searcher,
+                                            collectElevatedDocsWhenCollapsing);
         } else {
           throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
               "Collapsing field should be of either String, Int or Float type");

solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java

@@ -37,6 +37,7 @@ import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.CollapsingQParserPlugin;
 import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.util.FileUtils;
 import org.junit.Before;
@@ -932,6 +933,106 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
     }
   }
 
+  @Test
+  public void testOnlyDocsInSearchResultsWillBeElevated() throws Exception {
+    try {
+      init("schema12.xml");
+      assertU(adoc("id", "1", "title", "XXXX", "str_s1", "a"));
+      assertU(adoc("id", "2", "title", "YYYY", "str_s1", "b"));
+      assertU(adoc("id", "3", "title", "ZZZZ", "str_s1", "c"));
+
+      assertU(adoc("id", "4", "title", "XXXX XXXX", "str_s1", "x"));
+      assertU(adoc("id", "5", "title", "YYYY YYYY", "str_s1", "y"));
+      assertU(adoc("id", "6", "title", "XXXX XXXX", "str_s1", "z"));
+      assertU(adoc("id", "7", "title", "AAAA", "str_s1", "a"));
+
+      assertU(commit());
+
+      // default behaviour
+      assertQ("", req(
+              CommonParams.Q, "YYYY",
+              CommonParams.QT, "/elevate",
+              QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, "false",
+              CommonParams.FL, "id, score, [elevated]"),
+              "//*[@numFound='3']",
+              "//result/doc[1]/str[@name='id'][.='1']",
+              "//result/doc[2]/str[@name='id'][.='2']",
+              "//result/doc[3]/str[@name='id'][.='5']",
+              "//result/doc[1]/bool[@name='[elevated]'][.='true']",
+              "//result/doc[2]/bool[@name='[elevated]'][.='true']",
+              "//result/doc[3]/bool[@name='[elevated]'][.='false']"
+      );
+
+      // only docs that matches q
+      assertQ("", req(
+              CommonParams.Q, "YYYY",
+              CommonParams.QT, "/elevate",
+              QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, "true",
+              CommonParams.FL, "id, score, [elevated]"),
+              "//*[@numFound='2']",
+              "//result/doc[1]/str[@name='id'][.='2']",
+              "//result/doc[2]/str[@name='id'][.='5']",
+              "//result/doc[1]/bool[@name='[elevated]'][.='true']",
+              "//result/doc[2]/bool[@name='[elevated]'][.='false']"
+      );
+
+
+
+    } finally {
+      delete();
+    }
+  }
+
+  @Test
+  public void testOnlyRepresentativeIsVisibleWhenCollapsing() throws Exception {
+    try {
+      init("schema12.xml");
+      assertU(adoc("id", "1", "title", "ZZZZ", "str_s1", "a"));
+      assertU(adoc("id", "2", "title", "ZZZZ", "str_s1", "b"));
+      assertU(adoc("id", "3", "title", "ZZZZ ZZZZ", "str_s1", "a"));
+      assertU(adoc("id", "4", "title", "ZZZZ ZZZZ", "str_s1", "c"));
+
+      assertU(commit());
+
+      // default behaviour - all elevated docs are visible
+      assertQ("", req(
+              CommonParams.Q, "ZZZZ",
+              CommonParams.QT, "/elevate",
+              CollapsingQParserPlugin.COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, "true",
+              CommonParams.FQ, "{!collapse field=str_s1 sort='score desc'}",
+              CommonParams.FL, "id, score, [elevated]"),
+              "//*[@numFound='4']",
+              "//result/doc[1]/str[@name='id'][.='1']",
+              "//result/doc[2]/str[@name='id'][.='2']",
+              "//result/doc[3]/str[@name='id'][.='3']",
+              "//result/doc[4]/str[@name='id'][.='4']",
+              "//result/doc[1]/bool[@name='[elevated]'][.='true']",
+              "//result/doc[2]/bool[@name='[elevated]'][.='true']",
+              "//result/doc[3]/bool[@name='[elevated]'][.='true']",
+              "//result/doc[4]/bool[@name='[elevated]'][.='false']"
+      );
+
+      // only representative elevated doc visible
+      assertQ("", req(
+              CommonParams.Q, "ZZZZ",
+              CommonParams.QT, "/elevate",
+              CollapsingQParserPlugin.COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, "false",
+              CommonParams.FQ, "{!collapse field=str_s1 sort='score desc'}",
+              CommonParams.FL, "id, score, [elevated]"),
+              "//*[@numFound='3']",
+              "//result/doc[1]/str[@name='id'][.='2']",
+              "//result/doc[2]/str[@name='id'][.='3']",
+              "//result/doc[3]/str[@name='id'][.='4']",
+              "//result/doc[1]/bool[@name='[elevated]'][.='true']",
+              "//result/doc[2]/bool[@name='[elevated]'][.='true']",
+              "//result/doc[3]/bool[@name='[elevated]'][.='false']"
+      );
+
+    } finally {
+      delete();
+    }
+  }
+
   private static Set<BytesRef> toIdSet(String... ids) {
     return Arrays.stream(ids).map(BytesRef::new).collect(Collectors.toSet());
   }

solr/solr-ref-guide/src/collapse-and-expand-results.adoc

@@ -81,6 +81,10 @@ The data structures used for collapsing grow dynamically when collapsing on nume
 +
 The default is 100,000.
 
+`collectElevatedDocsWhenCollapsing`::
+In combination with the <<collapse-and-expand-results.adoc#collapsing-query-parser,Collapse Query Parser>> all elevated docs are visible at the beginning of the result set.
+If this parameter is `false`, only the representative is visible if the elevated docs has the same collapse key (default is `true`).
+
 
 === Sample Usage Syntax
 

solr/solr-ref-guide/src/the-query-elevation-component.adoc

@@ -93,6 +93,10 @@ they be subject to whatever the sort criteria is?  True by default.
 This is also a request parameter, which will override the config.
 The effect is most apparent when forceElevation is true and there is sorting on fields.
 
+`elevateOnlyDocsMatchingQuery`::
+By default, the component will also elevate docs that aren't part of the search result (matching the query).
+If you only want to elevate the docs that are part of the search result, set this to `true` (default is `false`).
+
 === The elevate.xml File
 
 Elevated query results can be configured in an external XML file specified in the `config-file` argument. An `elevate.xml` file might look like this:

solr/solrj/src/java/org/apache/solr/common/params/QueryElevationParams.java

@@ -55,4 +55,11 @@ public interface QueryElevationParams {
    * they be subject to whatever the sort criteria is?  True by default.
    */
   String USE_CONFIGURED_ELEVATED_ORDER = "useConfiguredElevatedOrder";
+
+  /**
+   * By default, the component will also elevate docs that aren't part of the search result (matching the query).
+   * If you only want to elevate the docs that are part of the search result, set this to true. False by default.
+   */
+  String ELEVATE_ONLY_DOCS_MATCHING_QUERY = "elevateOnlyDocsMatchingQuery";
+
 }