Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-15038: Add elevateOnlyDocsMatchingQuery and collectElevatedDocsWhenCollapsing parameters to query elevation. 

Closes #2134
This commit is contained in:
Tobias Kaessmann 2021-02-17 09:47:03 +01:00 committed by Bruno Roustant
parent 2ae45cc985
commit f142bf9c54
No known key found for this signature in database
GPG Key ID: CD28DABB95360525
8 changed files with 187 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gradle/testing/randomization/policies/solr-tests.policy
View File

@ -22,7 +22,7 @@
grant {
// 3rd party jar resources (where symlinks are not supported), test-files/ resources
permission java.io.FilePermission "${common.dir}${/}-", "read";
permission java.io.FilePermission "${common.dir}${/}..${/}solr${/}-", "read";
permission java.io.FilePermission "${common.dir}${/}..${/}solr${/}-", "read,write";
// system jar resources
permission java.io.FilePermission "${java.home}${/}-", "read";

3
solr/CHANGES.txt
View File

@ -239,6 +239,9 @@ Improvements
* SOLR-15101: Add "list" and "delete" APIs for managing incremental backups (Jason Gerlowski, shalin, Cao Manh Dat)
* SOLR-15038: Add elevateOnlyDocsMatchingQuery and collectElevatedDocsWhenCollapsing parameters to query elevation.
(Dennis Berger, Tobias Kässmann via Bruno Roustant)
Optimizations
---------------------
* SOLR-15079: Block Collapse - Faster collapse code when groups are co-located via Block Join style nested doc indexing.

5
solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
View File

@ -509,7 +509,10 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore
rb.setQuery(new BoostQuery(elevation.includeQuery, 0f));
} else {
BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
queryBuilder.add(rb.getQuery(), BooleanClause.Occur.SHOULD);
BooleanClause.Occur queryOccurrence =
params.getBool(QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, false) ?
BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
queryBuilder.add(rb.getQuery(), queryOccurrence);
queryBuilder.add(new BoostQuery(elevation.includeQuery, 0f), BooleanClause.Occur.SHOULD);
if (elevation.excludeQueries != null) {
if (params.getBool(QueryElevationParams.MARK_EXCLUDES, false)) {

92
solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
View File

@ -146,6 +146,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
*/
public static final String HINT_BLOCK = "block";
/**
* If elevation is used in combination with the collapse query parser, we can define that we only want to return the
* representative and not all elevated docs by setting this parameter to false (true by default).
*/
public static String COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING = "collectElevatedDocsWhenCollapsing";
/**
* @deprecated use {@link NullPolicy} instead.
*/
@ -585,6 +591,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private int nullPolicy;
private float nullScore = -Float.MAX_VALUE;
private int nullDoc = -1;
private boolean collectElevatedDocsWhenCollapsing;
private FloatArrayList nullScores;
private final BoostedDocsCollector boostedDocsCollector;
@ -594,9 +601,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
DocValuesProducer collapseValuesProducer,
int nullPolicy,
IntIntHashMap boostDocsMap,
IndexSearcher searcher) throws IOException {
IndexSearcher searcher,
boolean collectElevatedDocsWhenCollapsing) throws IOException {
this.maxDoc = maxDoc;
this.contexts = new LeafReaderContext[segments];
this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;
List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
for(int i=0; i<con.size(); i++) {
contexts[i] = con.get(i);
@ -653,12 +662,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
ord = -1;
}
}
// Check to see if we have documents boosted by the QueryElevationComponent
if (0 <= ord) {
if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;
} else {
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
if (collectElevatedDocsWhenCollapsing) {
// Check to see if we have documents boosted by the QueryElevationComponent
if (0 <= ord) {
if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;
} else {
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
}
}
if(ord > -1) {
@ -785,6 +796,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private int nullDoc = -1;
private FloatArrayList nullScores;
private String field;
private boolean collectElevatedDocsWhenCollapsing;
private final BoostedDocsCollector boostedDocsCollector;
@ -794,9 +806,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int size,
String field,
IntIntHashMap boostDocsMap,
IndexSearcher searcher) {
IndexSearcher searcher,
boolean collectElevatedDocsWhenCollapsing) {
this.maxDoc = maxDoc;
this.contexts = new LeafReaderContext[segments];
this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;
List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
for(int i=0; i<con.size(); i++) {
contexts[i] = con.get(i);
@ -827,8 +841,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
final int globalDoc = docBase+contextDoc;
if (collapseValues.advanceExact(contextDoc)) {
final int collapseValue = (int) collapseValues.longValue();
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectIfBoosted(collapseValue, globalDoc)) return;
if (collectElevatedDocsWhenCollapsing) {
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectIfBoosted(collapseValue, globalDoc)) return;
}
float score = scorer.score();
final int idx;
@ -847,9 +864,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
} else { // Null Group...
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
if (collectElevatedDocsWhenCollapsing){
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
}
if(nullPolicy == NullPolicy.COLLAPSE.getCode()) {
float score = scorer.score();
@ -958,7 +977,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private OrdFieldValueStrategy collapseStrategy;
private boolean needsScores4Collapsing;
private boolean needsScores;
private boolean collectElevatedDocsWhenCollapsing;
private final BoostedDocsCollector boostedDocsCollector;
public OrdFieldValueCollector(int maxDoc,
@ -971,10 +992,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
boolean needsScores,
FieldType fieldType,
IntIntHashMap boostDocsMap,
FunctionQuery funcQuery, IndexSearcher searcher) throws IOException{
FunctionQuery funcQuery, IndexSearcher searcher,
boolean collectElevatedDocsWhenCollapsing) throws IOException{
assert ! GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type);
this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;
this.maxDoc = maxDoc;
this.contexts = new LeafReaderContext[segments];
List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
@ -1053,12 +1076,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
ord = segmentValues.ordValue();
}
}
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (-1 == ord) {
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
} else {
if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;
if (collectElevatedDocsWhenCollapsing){
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (-1 == ord) {
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
} else {
if (boostedDocsCollector.collectIfBoosted(ord, globalDoc)) return;
}
}
collapseStrategy.collapse(ord, contextDoc, globalDoc);
@ -1166,6 +1191,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private String collapseField;
private final BoostedDocsCollector boostedDocsCollector;
private boolean collectElevatedDocsWhenCollapsing;
public IntFieldValueCollector(int maxDoc,
int size,
@ -1179,7 +1205,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
FieldType fieldType,
IntIntHashMap boostDocsMap,
FunctionQuery funcQuery,
IndexSearcher searcher) throws IOException{
IndexSearcher searcher,
boolean collectElevatedDocsWhenCollapsing) throws IOException{
this.collectElevatedDocsWhenCollapsing = collectElevatedDocsWhenCollapsing;
assert ! GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type);
@ -1243,10 +1271,11 @@ public class CollapsingQParserPlugin extends QParserPlugin {
collapseStrategy.collapse(collapseKey, contextDoc, globalDoc);
} else { // Null Group...
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
if (collectElevatedDocsWhenCollapsing){
// Check to see if we have documents boosted by the QueryElevationComponent (skip normal strategy based collection)
if (boostedDocsCollector.collectInNullGroupIfBoosted(globalDoc)) return;
}
if (NullPolicy.IGNORE.getCode() != nullPolicy) {
collapseStrategy.collapseNullGroup(contextDoc, globalDoc);
}
@ -1894,20 +1923,23 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int maxDoc = searcher.maxDoc();
int leafCount = searcher.getTopReaderContext().leaves().size();
SolrRequestInfo req = SolrRequestInfo.getRequestInfo();
boolean collectElevatedDocsWhenCollapsing = req != null && req.getReq().getParams().getBool(COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, true);
if (GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type)) {
if (collapseFieldType instanceof StrField) {
if (blockCollapse) {
return new BlockOrdScoreCollector(collapseField, nullPolicy, boostDocs);
}
return new OrdScoreCollector(maxDoc, leafCount, docValuesProducer, nullPolicy, boostDocs, searcher);
return new OrdScoreCollector(maxDoc, leafCount, docValuesProducer, nullPolicy, boostDocs, searcher, collectElevatedDocsWhenCollapsing);
} else if (isNumericCollapsible(collapseFieldType)) {
if (blockCollapse) {
return new BlockIntScoreCollector(collapseField, nullPolicy, boostDocs);
}
return new IntScoreCollector(maxDoc, leafCount, nullPolicy, size, collapseField, boostDocs, searcher);
return new IntScoreCollector(maxDoc, leafCount, nullPolicy, size, collapseField, boostDocs, searcher, collectElevatedDocsWhenCollapsing);
} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
@ -1937,7 +1969,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
minMaxFieldType,
boostDocs,
funcQuery,
searcher);
searcher,
collectElevatedDocsWhenCollapsing);
} else if (isNumericCollapsible(collapseFieldType)) {
@ -1962,7 +1995,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
minMaxFieldType,
boostDocs,
funcQuery,
searcher);
searcher,
collectElevatedDocsWhenCollapsing);
} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Collapsing field should be of either String, Int or Float type");

101
solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
View File

@ -37,6 +37,7 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.CollapsingQParserPlugin;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.FileUtils;
import org.junit.Before;
@ -932,6 +933,106 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
}
}
@Test
public void testOnlyDocsInSearchResultsWillBeElevated() throws Exception {
try {
init("schema12.xml");
assertU(adoc("id", "1", "title", "XXXX", "str_s1", "a"));
assertU(adoc("id", "2", "title", "YYYY", "str_s1", "b"));
assertU(adoc("id", "3", "title", "ZZZZ", "str_s1", "c"));
assertU(adoc("id", "4", "title", "XXXX XXXX", "str_s1", "x"));
assertU(adoc("id", "5", "title", "YYYY YYYY", "str_s1", "y"));
assertU(adoc("id", "6", "title", "XXXX XXXX", "str_s1", "z"));
assertU(adoc("id", "7", "title", "AAAA", "str_s1", "a"));
assertU(commit());
// default behaviour
assertQ("", req(
CommonParams.Q, "YYYY",
CommonParams.QT, "/elevate",
QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, "false",
CommonParams.FL, "id, score, [elevated]"),
"//*[@numFound='3']",
"//result/doc[1]/str[@name='id'][.='1']",
"//result/doc[2]/str[@name='id'][.='2']",
"//result/doc[3]/str[@name='id'][.='5']",
"//result/doc[1]/bool[@name='[elevated]'][.='true']",
"//result/doc[2]/bool[@name='[elevated]'][.='true']",
"//result/doc[3]/bool[@name='[elevated]'][.='false']"
);
// only docs that matches q
assertQ("", req(
CommonParams.Q, "YYYY",
CommonParams.QT, "/elevate",
QueryElevationParams.ELEVATE_ONLY_DOCS_MATCHING_QUERY, "true",
CommonParams.FL, "id, score, [elevated]"),
"//*[@numFound='2']",
"//result/doc[1]/str[@name='id'][.='2']",
"//result/doc[2]/str[@name='id'][.='5']",
"//result/doc[1]/bool[@name='[elevated]'][.='true']",
"//result/doc[2]/bool[@name='[elevated]'][.='false']"
);
} finally {
delete();
}
}
@Test
public void testOnlyRepresentativeIsVisibleWhenCollapsing() throws Exception {
try {
init("schema12.xml");
assertU(adoc("id", "1", "title", "ZZZZ", "str_s1", "a"));
assertU(adoc("id", "2", "title", "ZZZZ", "str_s1", "b"));
assertU(adoc("id", "3", "title", "ZZZZ ZZZZ", "str_s1", "a"));
assertU(adoc("id", "4", "title", "ZZZZ ZZZZ", "str_s1", "c"));
assertU(commit());
// default behaviour - all elevated docs are visible
assertQ("", req(
CommonParams.Q, "ZZZZ",
CommonParams.QT, "/elevate",
CollapsingQParserPlugin.COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, "true",
CommonParams.FQ, "{!collapse field=str_s1 sort='score desc'}",
CommonParams.FL, "id, score, [elevated]"),
"//*[@numFound='4']",
"//result/doc[1]/str[@name='id'][.='1']",
"//result/doc[2]/str[@name='id'][.='2']",
"//result/doc[3]/str[@name='id'][.='3']",
"//result/doc[4]/str[@name='id'][.='4']",
"//result/doc[1]/bool[@name='[elevated]'][.='true']",
"//result/doc[2]/bool[@name='[elevated]'][.='true']",
"//result/doc[3]/bool[@name='[elevated]'][.='true']",
"//result/doc[4]/bool[@name='[elevated]'][.='false']"
);
// only representative elevated doc visible
assertQ("", req(
CommonParams.Q, "ZZZZ",
CommonParams.QT, "/elevate",
CollapsingQParserPlugin.COLLECT_ELEVATED_DOCS_WHEN_COLLAPSING, "false",
CommonParams.FQ, "{!collapse field=str_s1 sort='score desc'}",
CommonParams.FL, "id, score, [elevated]"),
"//*[@numFound='3']",
"//result/doc[1]/str[@name='id'][.='2']",
"//result/doc[2]/str[@name='id'][.='3']",
"//result/doc[3]/str[@name='id'][.='4']",
"//result/doc[1]/bool[@name='[elevated]'][.='true']",
"//result/doc[2]/bool[@name='[elevated]'][.='true']",
"//result/doc[3]/bool[@name='[elevated]'][.='false']"
);
} finally {
delete();
}
}
private static Set<BytesRef> toIdSet(String... ids) {
return Arrays.stream(ids).map(BytesRef::new).collect(Collectors.toSet());
}

4
solr/solr-ref-guide/src/collapse-and-expand-results.adoc
View File

@ -81,6 +81,10 @@ The data structures used for collapsing grow dynamically when collapsing on nume
+
The default is 100,000.
`collectElevatedDocsWhenCollapsing`::
In combination with the <<collapse-and-expand-results.adoc#collapsing-query-parser,Collapse Query Parser>> all elevated docs are visible at the beginning of the result set.
If this parameter is `false`, only the representative is visible if the elevated docs has the same collapse key (default is `true`).
=== Sample Usage Syntax

4
solr/solr-ref-guide/src/the-query-elevation-component.adoc
View File

@ -93,6 +93,10 @@ they be subject to whatever the sort criteria is? True by default.
This is also a request parameter, which will override the config.
The effect is most apparent when forceElevation is true and there is sorting on fields.
`elevateOnlyDocsMatchingQuery`::
By default, the component will also elevate docs that aren't part of the search result (matching the query).
If you only want to elevate the docs that are part of the search result, set this to `true` (default is `false`).
=== The elevate.xml File
Elevated query results can be configured in an external XML file specified in the `config-file` argument. An `elevate.xml` file might look like this:

7
solr/solrj/src/java/org/apache/solr/common/params/QueryElevationParams.java
View File

@ -55,4 +55,11 @@ public interface QueryElevationParams {
* they be subject to whatever the sort criteria is? True by default.
*/
String USE_CONFIGURED_ELEVATED_ORDER = "useConfiguredElevatedOrder";
/**
* By default, the component will also elevate docs that aren't part of the search result (matching the query).
* If you only want to elevate the docs that are part of the search result, set this to true. False by default.
*/
String ELEVATE_ONLY_DOCS_MATCHING_QUERY = "elevateOnlyDocsMatchingQuery";
}