mirror of https://github.com/apache/lucene.git
SOLR-13348: Speed up collapsing by avoiding scoring of ineligible documents
This commit is contained in:
Ishan Chattopadhyaya
parent
ceeac409a5
commit
29a98825d0
|
|
@ -285,6 +285,8 @@ Improvements
|
|||
* SOLR-12833: Avoid unnecessary memory cost when DistributedUpdateProcessor timed-out lock is not used.
|
||||
(jefferyyuan, ab)
|
||||
|
||||
* SOLR-13348: Speed up collapsing by avoiding scoring of ineligible documents (Andrzej Wislowski via
|
||||
Ishan Chattopadhyaya)
|
||||
|
||||
Other Changes
|
||||
----------------------
|
||||
|
|
|
|||
|
|
@ -216,6 +216,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
private final SortSpec sortSpec; // may be null, parsed at most once from groupHeadSelector
|
||||
public String hint;
|
||||
private boolean needsScores = true;
|
||||
private boolean needsScores4Collapsing = false;
|
||||
private int nullPolicy;
|
||||
private Set<BytesRef> boosted; // ordered by "priority"
|
||||
public static final int NULL_POLICY_IGNORE = 0;
|
||||
|
|
@ -319,15 +320,15 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
// may be null in some esoteric corner usages
|
||||
final ResponseBuilder rb = info.getResponseBuilder();
|
||||
final SortSpec topSort = null == rb ? null : rb.getSortSpec();
|
||||
|
||||
this.needsScores =
|
||||
|
||||
this.needsScores4Collapsing = GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type) ||
|
||||
(GroupHeadSelectorType.SORT.equals(groupHeadSelector.type)
|
||||
&& this.sortSpec.includesScore()) ||
|
||||
(GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type)
|
||||
&& CollapseScore.wantsCScore(groupHeadSelector.selectorText));
|
||||
this.needsScores = needsScores4Collapsing ||
|
||||
(info.getRsp().getReturnFields().wantsScore() ||
|
||||
(null != topSort && topSort.includesScore()) ||
|
||||
GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type) ||
|
||||
(GroupHeadSelectorType.SORT.equals(groupHeadSelector.type)
|
||||
&& this.sortSpec.includesScore()) ||
|
||||
(GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type)
|
||||
&& CollapseScore.wantsCScore(groupHeadSelector.selectorText)) ||
|
||||
(this.boosted != null));
|
||||
|
||||
if (this.needsScores && null != rb) {
|
||||
|
|
@ -378,6 +379,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
this.sortSpec,
|
||||
this.nullPolicy,
|
||||
this.hint,
|
||||
this.needsScores4Collapsing,
|
||||
this.needsScores,
|
||||
this.size,
|
||||
boostDocsMap,
|
||||
|
|
@ -930,6 +932,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
private int nullPolicy;
|
||||
|
||||
private OrdFieldValueStrategy collapseStrategy;
|
||||
private boolean needsScores4Collapsing;
|
||||
private boolean needsScores;
|
||||
|
||||
public OrdFieldValueCollector(int maxDoc,
|
||||
|
|
@ -938,6 +941,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
int nullPolicy,
|
||||
GroupHeadSelector groupHeadSelector,
|
||||
SortSpec sortSpec,
|
||||
boolean needsScores4Collapsing,
|
||||
boolean needsScores,
|
||||
FieldType fieldType,
|
||||
IntIntHashMap boostDocs,
|
||||
|
|
@ -956,11 +960,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
|
||||
int valueCount = collapseValues.getValueCount();
|
||||
this.nullPolicy = nullPolicy;
|
||||
this.needsScores4Collapsing = needsScores4Collapsing;
|
||||
this.needsScores = needsScores;
|
||||
if (null != sortSpec) {
|
||||
this.collapseStrategy = new OrdSortSpecStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores, boostDocs, sortSpec, searcher, collapseValues);
|
||||
this.collapseStrategy = new OrdSortSpecStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocs, sortSpec, searcher, collapseValues);
|
||||
} else if (funcQuery != null) {
|
||||
this.collapseStrategy = new OrdValueSourceStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores, boostDocs, funcQuery, searcher, collapseValues);
|
||||
this.collapseStrategy = new OrdValueSourceStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocs, funcQuery, searcher, collapseValues);
|
||||
} else {
|
||||
NumberType numType = fieldType.getNumberType();
|
||||
if (null == numType) {
|
||||
|
|
@ -1116,6 +1121,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
private int nullPolicy;
|
||||
|
||||
private IntFieldValueStrategy collapseStrategy;
|
||||
private boolean needsScores4Collapsing;
|
||||
private boolean needsScores;
|
||||
private String collapseField;
|
||||
|
||||
|
|
@ -1127,6 +1133,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
String collapseField,
|
||||
GroupHeadSelector groupHeadSelector,
|
||||
SortSpec sortSpec,
|
||||
boolean needsScores4Collapsing,
|
||||
boolean needsScores,
|
||||
FieldType fieldType,
|
||||
IntIntHashMap boostDocsMap,
|
||||
|
|
@ -1140,11 +1147,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
this.collapseField = collapseField;
|
||||
this.nullValue = nullValue;
|
||||
this.nullPolicy = nullPolicy;
|
||||
this.needsScores4Collapsing = needsScores4Collapsing;
|
||||
this.needsScores = needsScores;
|
||||
if (null != sortSpec) {
|
||||
this.collapseStrategy = new IntSortSpecStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores, boostDocsMap, sortSpec, searcher);
|
||||
this.collapseStrategy = new IntSortSpecStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocsMap, sortSpec, searcher);
|
||||
} else if (funcQuery != null) {
|
||||
this.collapseStrategy = new IntValueSourceStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores, boostDocsMap, funcQuery, searcher);
|
||||
this.collapseStrategy = new IntValueSourceStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocsMap, funcQuery, searcher);
|
||||
} else {
|
||||
NumberType numType = fieldType.getNumberType();
|
||||
assert null != numType; // shouldn't make it here for non-numeric types
|
||||
|
|
@ -1269,6 +1277,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
SortSpec sortSpec,
|
||||
int nullPolicy,
|
||||
String hint,
|
||||
boolean needsScores4Collapsing,
|
||||
boolean needsScores,
|
||||
int size,
|
||||
IntIntHashMap boostDocs,
|
||||
|
|
@ -1367,6 +1376,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
nullPolicy,
|
||||
groupHeadSelector,
|
||||
sortSpec,
|
||||
needsScores4Collapsing,
|
||||
needsScores,
|
||||
minMaxFieldType,
|
||||
boostDocs,
|
||||
|
|
@ -1398,6 +1408,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
collapseField,
|
||||
groupHeadSelector,
|
||||
sortSpec,
|
||||
needsScores4Collapsing,
|
||||
needsScores,
|
||||
minMaxFieldType,
|
||||
boostDocs,
|
||||
|
|
@ -1823,18 +1834,20 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
private float[] ordVals;
|
||||
private Map rcontext;
|
||||
private final CollapseScore collapseScore = new CollapseScore();
|
||||
private float score;
|
||||
private boolean needsScores4Collapsing;
|
||||
|
||||
public OrdValueSourceStrategy(int maxDoc,
|
||||
int nullPolicy,
|
||||
int[] ords,
|
||||
GroupHeadSelector groupHeadSelector,
|
||||
boolean needsScores4Collapsing,
|
||||
boolean needsScores,
|
||||
IntIntHashMap boostDocs,
|
||||
FunctionQuery funcQuery,
|
||||
IndexSearcher searcher,
|
||||
SortedDocValues values) throws IOException {
|
||||
super(maxDoc, ords, nullPolicy, needsScores, boostDocs, values);
|
||||
this.needsScores4Collapsing = needsScores4Collapsing;
|
||||
this.valueSource = funcQuery.getValueSource();
|
||||
this.rcontext = ValueSource.newContext(searcher);
|
||||
this.ordVals = new float[ords.length];
|
||||
|
|
@ -1865,14 +1878,15 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
}
|
||||
|
||||
public void collapse(int ord, int contextDoc, int globalDoc) throws IOException {
|
||||
float score = 0;
|
||||
|
||||
if(boosted && mergeBoost.boost(globalDoc)) {
|
||||
this.boostOrds.add(ord);
|
||||
this.boostDocs.add(globalDoc);
|
||||
}
|
||||
|
||||
if (needsScores) {
|
||||
this.score = scorer.score();
|
||||
if (needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
this.collapseScore.score = score;
|
||||
}
|
||||
|
||||
|
|
@ -1883,6 +1897,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
ords[ord] = globalDoc;
|
||||
ordVals[ord] = currentVal;
|
||||
if(needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
scores[ord] = score;
|
||||
}
|
||||
}
|
||||
|
|
@ -1891,12 +1908,18 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
nullVal = currentVal;
|
||||
nullDoc = globalDoc;
|
||||
if(needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
nullScore = score;
|
||||
}
|
||||
}
|
||||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
|
||||
this.collapsedSet.set(globalDoc);
|
||||
if(needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
nullScores.add(score);
|
||||
}
|
||||
}
|
||||
|
|
@ -1910,25 +1933,26 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
private static class OrdSortSpecStrategy extends OrdFieldValueStrategy {
|
||||
|
||||
private final SortFieldsCompare compareState;
|
||||
private final SortSpec sortSpec;
|
||||
private final Sort sort;
|
||||
|
||||
private float score;
|
||||
private boolean needsScores4Collapsing;
|
||||
|
||||
public OrdSortSpecStrategy(int maxDoc,
|
||||
int nullPolicy,
|
||||
int[] ords,
|
||||
GroupHeadSelector groupHeadSelector,
|
||||
boolean needsScores4Collapsing,
|
||||
boolean needsScores,
|
||||
IntIntHashMap boostDocs,
|
||||
SortSpec sortSpec,
|
||||
IndexSearcher searcher,
|
||||
SortedDocValues values) throws IOException {
|
||||
super(maxDoc, ords, nullPolicy, needsScores, boostDocs, values);
|
||||
this.needsScores4Collapsing = needsScores4Collapsing;
|
||||
|
||||
assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type);
|
||||
|
||||
this.sortSpec = sortSpec;
|
||||
this.sort = rewriteSort(sortSpec, searcher);
|
||||
|
||||
this.compareState = new SortFieldsCompare(sort.getSort(), ords.length);
|
||||
|
|
@ -1960,7 +1984,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
this.boostDocs.add(globalDoc);
|
||||
}
|
||||
|
||||
if (needsScores) {
|
||||
if (needsScores4Collapsing) {
|
||||
this.score = scorer.score();
|
||||
}
|
||||
|
||||
|
|
@ -1970,13 +1994,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
compareState.setGroupValues(ord, contextDoc);
|
||||
ords[ord] = globalDoc;
|
||||
if (needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
this.score = scorer.score();
|
||||
}
|
||||
scores[ord] = score;
|
||||
}
|
||||
} else {
|
||||
// test this ord to see if it's a new group leader
|
||||
if (compareState.testAndSetGroupValues(ord, contextDoc)) {
|
||||
if (compareState.testAndSetGroupValues(ord, contextDoc)) {//TODO X
|
||||
ords[ord] = globalDoc;
|
||||
if (needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
this.score = scorer.score();
|
||||
}
|
||||
scores[ord] = score;
|
||||
}
|
||||
}
|
||||
|
|
@ -1987,6 +2017,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
compareState.setNullGroupValues(contextDoc);
|
||||
nullDoc = globalDoc;
|
||||
if (needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
this.score = scorer.score();
|
||||
}
|
||||
nullScore = score;
|
||||
}
|
||||
} else {
|
||||
|
|
@ -1994,6 +2027,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
if (compareState.testAndSetNullGroupValues(contextDoc)) {
|
||||
nullDoc = globalDoc;
|
||||
if (needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
this.score = scorer.score();
|
||||
}
|
||||
nullScore = score;
|
||||
}
|
||||
}
|
||||
|
|
@ -2001,6 +2037,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
|
||||
this.collapsedSet.set(globalDoc);
|
||||
if (needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
this.score = scorer.score();
|
||||
}
|
||||
nullScores.add(score);
|
||||
}
|
||||
}
|
||||
|
|
@ -2355,8 +2394,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
private FunctionValues functionValues;
|
||||
private Map rcontext;
|
||||
private final CollapseScore collapseScore = new CollapseScore();
|
||||
private float score;
|
||||
private int index=-1;
|
||||
private boolean needsScores4Collapsing;
|
||||
|
||||
public IntValueSourceStrategy(int maxDoc,
|
||||
int size,
|
||||
|
|
@ -2364,6 +2403,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
int nullValue,
|
||||
int nullPolicy,
|
||||
GroupHeadSelector groupHeadSelector,
|
||||
boolean needsScores4Collapsing,
|
||||
boolean needsScores,
|
||||
IntIntHashMap boostDocs,
|
||||
FunctionQuery funcQuery,
|
||||
|
|
@ -2371,6 +2411,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
|
||||
super(maxDoc, size, collapseField, nullValue, nullPolicy, needsScores, boostDocs);
|
||||
|
||||
this.needsScores4Collapsing = needsScores4Collapsing;
|
||||
this.testValues = new float[size];
|
||||
|
||||
this.valueSource = funcQuery.getValueSource();
|
||||
|
|
@ -2401,6 +2442,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
}
|
||||
|
||||
public void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException {
|
||||
float score = 0;
|
||||
|
||||
// Check to see if we have documents boosted by the QueryElevationComponent
|
||||
if(boosts && mergeBoost.boost(globalDoc)) {
|
||||
|
|
@ -2409,8 +2451,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
return;
|
||||
}
|
||||
|
||||
if (needsScores) {
|
||||
this.score = scorer.score();
|
||||
if (needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
this.collapseScore.score = score;
|
||||
}
|
||||
|
||||
|
|
@ -2424,6 +2466,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
testValues[pointer] = currentVal;
|
||||
docs[pointer] = globalDoc;
|
||||
if(needsScores){
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
scores[pointer] = score;
|
||||
}
|
||||
}
|
||||
|
|
@ -2440,6 +2485,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
docs[index] = globalDoc;
|
||||
testValues[index] = currentVal;
|
||||
if(needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
scores[index] = score;
|
||||
}
|
||||
}
|
||||
|
|
@ -2448,13 +2496,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
nullCompVal = currentVal;
|
||||
nullDoc = globalDoc;
|
||||
if(needsScores) {
|
||||
nullScore = scorer.score();
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
nullScore = score;
|
||||
}
|
||||
}
|
||||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
|
||||
this.collapsedSet.set(globalDoc);
|
||||
if(needsScores) {
|
||||
nullScores.add(scorer.score());
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
nullScores.add(score);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2471,8 +2525,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
private final SortSpec sortSpec;
|
||||
private final Sort sort;
|
||||
|
||||
private float score;
|
||||
private int index=-1;
|
||||
private boolean needsScores4Collapsing;
|
||||
|
||||
public IntSortSpecStrategy(int maxDoc,
|
||||
int size,
|
||||
|
|
@ -2480,12 +2534,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
int nullValue,
|
||||
int nullPolicy,
|
||||
GroupHeadSelector groupHeadSelector,
|
||||
boolean needsScores4Collapsing,
|
||||
boolean needsScores,
|
||||
IntIntHashMap boostDocs,
|
||||
SortSpec sortSpec,
|
||||
IndexSearcher searcher) throws IOException {
|
||||
|
||||
super(maxDoc, size, collapseField, nullValue, nullPolicy, needsScores, boostDocs);
|
||||
this.needsScores4Collapsing = needsScores4Collapsing;
|
||||
|
||||
assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type);
|
||||
|
||||
|
|
@ -2513,6 +2569,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
}
|
||||
|
||||
public void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException {
|
||||
float score = 0;
|
||||
|
||||
// Check to see if we have documents boosted by the QueryElevationComponent
|
||||
if(boosts && mergeBoost.boost(globalDoc)) {
|
||||
|
|
@ -2521,8 +2578,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
return;
|
||||
}
|
||||
|
||||
if (needsScores) {
|
||||
this.score = scorer.score();
|
||||
if (needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
|
||||
if (collapseKey != nullValue) {
|
||||
|
|
@ -2533,6 +2590,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
if (compareState.testAndSetGroupValues(pointer, contextDoc)) {
|
||||
docs[pointer] = globalDoc;
|
||||
if (needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
scores[pointer] = score;
|
||||
}
|
||||
}
|
||||
|
|
@ -2550,6 +2610,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
docs[index] = globalDoc;
|
||||
compareState.setGroupValues(index, contextDoc);
|
||||
if(needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
scores[index] = score;
|
||||
}
|
||||
}
|
||||
|
|
@ -2559,6 +2622,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
compareState.setNullGroupValues(contextDoc);
|
||||
nullDoc = globalDoc;
|
||||
if (needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
nullScore = score;
|
||||
}
|
||||
} else {
|
||||
|
|
@ -2566,6 +2632,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
if (compareState.testAndSetNullGroupValues(contextDoc)) {
|
||||
nullDoc = globalDoc;
|
||||
if (needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
nullScore = score;
|
||||
}
|
||||
}
|
||||
|
|
@ -2573,6 +2642,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
|
|||
} else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
|
||||
this.collapsedSet.set(globalDoc);
|
||||
if (needsScores) {
|
||||
if (!needsScores4Collapsing) {
|
||||
score = scorer.score();
|
||||
}
|
||||
nullScores.add(score);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue