From 29a98825d0b70166c6c759d022f2c1426b145fe1 Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Fri, 3 May 2019 02:45:15 +0530 Subject: [PATCH] SOLR-13348: Speed up collapsing by avoiding scoring of ineligible documents --- solr/CHANGES.txt | 2 + .../solr/search/CollapsingQParserPlugin.java | 124 ++++++++++++++---- 2 files changed, 100 insertions(+), 26 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 3abcfa1bfdf..1c5e89b9a0e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -285,6 +285,8 @@ Improvements * SOLR-12833: Avoid unnecessary memory cost when DistributedUpdateProcessor timed-out lock is not used. (jefferyyuan, ab) +* SOLR-13348: Speed up collapsing by avoiding scoring of ineligible documents (Andrzej Wislowski via + Ishan Chattopadhyaya) Other Changes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index ccbb71fed18..c0e9223c7d3 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -216,6 +216,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { private final SortSpec sortSpec; // may be null, parsed at most once from groupHeadSelector public String hint; private boolean needsScores = true; + private boolean needsScores4Collapsing = false; private int nullPolicy; private Set boosted; // ordered by "priority" public static final int NULL_POLICY_IGNORE = 0; @@ -319,15 +320,15 @@ public class CollapsingQParserPlugin extends QParserPlugin { // may be null in some esoteric corner usages final ResponseBuilder rb = info.getResponseBuilder(); final SortSpec topSort = null == rb ? null : rb.getSortSpec(); - - this.needsScores = + + this.needsScores4Collapsing = GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type) || + (GroupHeadSelectorType.SORT.equals(groupHeadSelector.type) + && this.sortSpec.includesScore()) || + (GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type) + && CollapseScore.wantsCScore(groupHeadSelector.selectorText)); + this.needsScores = needsScores4Collapsing || (info.getRsp().getReturnFields().wantsScore() || (null != topSort && topSort.includesScore()) || - GroupHeadSelectorType.SCORE.equals(groupHeadSelector.type) || - (GroupHeadSelectorType.SORT.equals(groupHeadSelector.type) - && this.sortSpec.includesScore()) || - (GroupHeadSelectorType.MIN_MAX.contains(groupHeadSelector.type) - && CollapseScore.wantsCScore(groupHeadSelector.selectorText)) || (this.boosted != null)); if (this.needsScores && null != rb) { @@ -378,6 +379,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { this.sortSpec, this.nullPolicy, this.hint, + this.needsScores4Collapsing, this.needsScores, this.size, boostDocsMap, @@ -930,6 +932,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { private int nullPolicy; private OrdFieldValueStrategy collapseStrategy; + private boolean needsScores4Collapsing; private boolean needsScores; public OrdFieldValueCollector(int maxDoc, @@ -938,6 +941,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { int nullPolicy, GroupHeadSelector groupHeadSelector, SortSpec sortSpec, + boolean needsScores4Collapsing, boolean needsScores, FieldType fieldType, IntIntHashMap boostDocs, @@ -956,11 +960,12 @@ public class CollapsingQParserPlugin extends QParserPlugin { int valueCount = collapseValues.getValueCount(); this.nullPolicy = nullPolicy; + this.needsScores4Collapsing = needsScores4Collapsing; this.needsScores = needsScores; if (null != sortSpec) { - this.collapseStrategy = new OrdSortSpecStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores, boostDocs, sortSpec, searcher, collapseValues); + this.collapseStrategy = new OrdSortSpecStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocs, sortSpec, searcher, collapseValues); } else if (funcQuery != null) { - this.collapseStrategy = new OrdValueSourceStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores, boostDocs, funcQuery, searcher, collapseValues); + this.collapseStrategy = new OrdValueSourceStrategy(maxDoc, nullPolicy, new int[valueCount], groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocs, funcQuery, searcher, collapseValues); } else { NumberType numType = fieldType.getNumberType(); if (null == numType) { @@ -1116,6 +1121,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { private int nullPolicy; private IntFieldValueStrategy collapseStrategy; + private boolean needsScores4Collapsing; private boolean needsScores; private String collapseField; @@ -1127,6 +1133,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { String collapseField, GroupHeadSelector groupHeadSelector, SortSpec sortSpec, + boolean needsScores4Collapsing, boolean needsScores, FieldType fieldType, IntIntHashMap boostDocsMap, @@ -1140,11 +1147,12 @@ public class CollapsingQParserPlugin extends QParserPlugin { this.collapseField = collapseField; this.nullValue = nullValue; this.nullPolicy = nullPolicy; + this.needsScores4Collapsing = needsScores4Collapsing; this.needsScores = needsScores; if (null != sortSpec) { - this.collapseStrategy = new IntSortSpecStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores, boostDocsMap, sortSpec, searcher); + this.collapseStrategy = new IntSortSpecStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocsMap, sortSpec, searcher); } else if (funcQuery != null) { - this.collapseStrategy = new IntValueSourceStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores, boostDocsMap, funcQuery, searcher); + this.collapseStrategy = new IntValueSourceStrategy(maxDoc, size, collapseField, nullValue, nullPolicy, groupHeadSelector, this.needsScores4Collapsing, this.needsScores, boostDocsMap, funcQuery, searcher); } else { NumberType numType = fieldType.getNumberType(); assert null != numType; // shouldn't make it here for non-numeric types @@ -1269,6 +1277,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { SortSpec sortSpec, int nullPolicy, String hint, + boolean needsScores4Collapsing, boolean needsScores, int size, IntIntHashMap boostDocs, @@ -1367,6 +1376,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { nullPolicy, groupHeadSelector, sortSpec, + needsScores4Collapsing, needsScores, minMaxFieldType, boostDocs, @@ -1398,6 +1408,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { collapseField, groupHeadSelector, sortSpec, + needsScores4Collapsing, needsScores, minMaxFieldType, boostDocs, @@ -1823,18 +1834,20 @@ public class CollapsingQParserPlugin extends QParserPlugin { private float[] ordVals; private Map rcontext; private final CollapseScore collapseScore = new CollapseScore(); - private float score; + private boolean needsScores4Collapsing; public OrdValueSourceStrategy(int maxDoc, int nullPolicy, int[] ords, GroupHeadSelector groupHeadSelector, + boolean needsScores4Collapsing, boolean needsScores, IntIntHashMap boostDocs, FunctionQuery funcQuery, IndexSearcher searcher, SortedDocValues values) throws IOException { super(maxDoc, ords, nullPolicy, needsScores, boostDocs, values); + this.needsScores4Collapsing = needsScores4Collapsing; this.valueSource = funcQuery.getValueSource(); this.rcontext = ValueSource.newContext(searcher); this.ordVals = new float[ords.length]; @@ -1865,14 +1878,15 @@ public class CollapsingQParserPlugin extends QParserPlugin { } public void collapse(int ord, int contextDoc, int globalDoc) throws IOException { + float score = 0; if(boosted && mergeBoost.boost(globalDoc)) { this.boostOrds.add(ord); this.boostDocs.add(globalDoc); } - if (needsScores) { - this.score = scorer.score(); + if (needsScores4Collapsing) { + score = scorer.score(); this.collapseScore.score = score; } @@ -1883,6 +1897,9 @@ public class CollapsingQParserPlugin extends QParserPlugin { ords[ord] = globalDoc; ordVals[ord] = currentVal; if(needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } scores[ord] = score; } } @@ -1891,12 +1908,18 @@ public class CollapsingQParserPlugin extends QParserPlugin { nullVal = currentVal; nullDoc = globalDoc; if(needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } nullScore = score; } } } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) { this.collapsedSet.set(globalDoc); if(needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } nullScores.add(score); } } @@ -1910,25 +1933,26 @@ public class CollapsingQParserPlugin extends QParserPlugin { private static class OrdSortSpecStrategy extends OrdFieldValueStrategy { private final SortFieldsCompare compareState; - private final SortSpec sortSpec; private final Sort sort; private float score; + private boolean needsScores4Collapsing; public OrdSortSpecStrategy(int maxDoc, int nullPolicy, int[] ords, GroupHeadSelector groupHeadSelector, + boolean needsScores4Collapsing, boolean needsScores, IntIntHashMap boostDocs, SortSpec sortSpec, IndexSearcher searcher, SortedDocValues values) throws IOException { super(maxDoc, ords, nullPolicy, needsScores, boostDocs, values); + this.needsScores4Collapsing = needsScores4Collapsing; assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type); - this.sortSpec = sortSpec; this.sort = rewriteSort(sortSpec, searcher); this.compareState = new SortFieldsCompare(sort.getSort(), ords.length); @@ -1960,7 +1984,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { this.boostDocs.add(globalDoc); } - if (needsScores) { + if (needsScores4Collapsing) { this.score = scorer.score(); } @@ -1970,13 +1994,19 @@ public class CollapsingQParserPlugin extends QParserPlugin { compareState.setGroupValues(ord, contextDoc); ords[ord] = globalDoc; if (needsScores) { + if (!needsScores4Collapsing) { + this.score = scorer.score(); + } scores[ord] = score; } } else { // test this ord to see if it's a new group leader - if (compareState.testAndSetGroupValues(ord, contextDoc)) { + if (compareState.testAndSetGroupValues(ord, contextDoc)) {//TODO X ords[ord] = globalDoc; if (needsScores) { + if (!needsScores4Collapsing) { + this.score = scorer.score(); + } scores[ord] = score; } } @@ -1987,6 +2017,9 @@ public class CollapsingQParserPlugin extends QParserPlugin { compareState.setNullGroupValues(contextDoc); nullDoc = globalDoc; if (needsScores) { + if (!needsScores4Collapsing) { + this.score = scorer.score(); + } nullScore = score; } } else { @@ -1994,6 +2027,9 @@ public class CollapsingQParserPlugin extends QParserPlugin { if (compareState.testAndSetNullGroupValues(contextDoc)) { nullDoc = globalDoc; if (needsScores) { + if (!needsScores4Collapsing) { + this.score = scorer.score(); + } nullScore = score; } } @@ -2001,6 +2037,9 @@ public class CollapsingQParserPlugin extends QParserPlugin { } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) { this.collapsedSet.set(globalDoc); if (needsScores) { + if (!needsScores4Collapsing) { + this.score = scorer.score(); + } nullScores.add(score); } } @@ -2355,8 +2394,8 @@ public class CollapsingQParserPlugin extends QParserPlugin { private FunctionValues functionValues; private Map rcontext; private final CollapseScore collapseScore = new CollapseScore(); - private float score; private int index=-1; + private boolean needsScores4Collapsing; public IntValueSourceStrategy(int maxDoc, int size, @@ -2364,6 +2403,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { int nullValue, int nullPolicy, GroupHeadSelector groupHeadSelector, + boolean needsScores4Collapsing, boolean needsScores, IntIntHashMap boostDocs, FunctionQuery funcQuery, @@ -2371,6 +2411,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { super(maxDoc, size, collapseField, nullValue, nullPolicy, needsScores, boostDocs); + this.needsScores4Collapsing = needsScores4Collapsing; this.testValues = new float[size]; this.valueSource = funcQuery.getValueSource(); @@ -2401,6 +2442,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { } public void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException { + float score = 0; // Check to see if we have documents boosted by the QueryElevationComponent if(boosts && mergeBoost.boost(globalDoc)) { @@ -2409,8 +2451,8 @@ public class CollapsingQParserPlugin extends QParserPlugin { return; } - if (needsScores) { - this.score = scorer.score(); + if (needsScores4Collapsing) { + score = scorer.score(); this.collapseScore.score = score; } @@ -2424,6 +2466,9 @@ public class CollapsingQParserPlugin extends QParserPlugin { testValues[pointer] = currentVal; docs[pointer] = globalDoc; if(needsScores){ + if (!needsScores4Collapsing) { + score = scorer.score(); + } scores[pointer] = score; } } @@ -2440,6 +2485,9 @@ public class CollapsingQParserPlugin extends QParserPlugin { docs[index] = globalDoc; testValues[index] = currentVal; if(needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } scores[index] = score; } } @@ -2448,13 +2496,19 @@ public class CollapsingQParserPlugin extends QParserPlugin { nullCompVal = currentVal; nullDoc = globalDoc; if(needsScores) { - nullScore = scorer.score(); + if (!needsScores4Collapsing) { + score = scorer.score(); + } + nullScore = score; } } } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) { this.collapsedSet.set(globalDoc); if(needsScores) { - nullScores.add(scorer.score()); + if (!needsScores4Collapsing) { + score = scorer.score(); + } + nullScores.add(score); } } } @@ -2471,8 +2525,8 @@ public class CollapsingQParserPlugin extends QParserPlugin { private final SortSpec sortSpec; private final Sort sort; - private float score; private int index=-1; + private boolean needsScores4Collapsing; public IntSortSpecStrategy(int maxDoc, int size, @@ -2480,12 +2534,14 @@ public class CollapsingQParserPlugin extends QParserPlugin { int nullValue, int nullPolicy, GroupHeadSelector groupHeadSelector, + boolean needsScores4Collapsing, boolean needsScores, IntIntHashMap boostDocs, SortSpec sortSpec, IndexSearcher searcher) throws IOException { super(maxDoc, size, collapseField, nullValue, nullPolicy, needsScores, boostDocs); + this.needsScores4Collapsing = needsScores4Collapsing; assert GroupHeadSelectorType.SORT.equals(groupHeadSelector.type); @@ -2513,6 +2569,7 @@ public class CollapsingQParserPlugin extends QParserPlugin { } public void collapse(int collapseKey, int contextDoc, int globalDoc) throws IOException { + float score = 0; // Check to see if we have documents boosted by the QueryElevationComponent if(boosts && mergeBoost.boost(globalDoc)) { @@ -2521,8 +2578,8 @@ public class CollapsingQParserPlugin extends QParserPlugin { return; } - if (needsScores) { - this.score = scorer.score(); + if (needsScores4Collapsing) { + score = scorer.score(); } if (collapseKey != nullValue) { @@ -2533,6 +2590,9 @@ public class CollapsingQParserPlugin extends QParserPlugin { if (compareState.testAndSetGroupValues(pointer, contextDoc)) { docs[pointer] = globalDoc; if (needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } scores[pointer] = score; } } @@ -2550,6 +2610,9 @@ public class CollapsingQParserPlugin extends QParserPlugin { docs[index] = globalDoc; compareState.setGroupValues(index, contextDoc); if(needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } scores[index] = score; } } @@ -2559,6 +2622,9 @@ public class CollapsingQParserPlugin extends QParserPlugin { compareState.setNullGroupValues(contextDoc); nullDoc = globalDoc; if (needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } nullScore = score; } } else { @@ -2566,6 +2632,9 @@ public class CollapsingQParserPlugin extends QParserPlugin { if (compareState.testAndSetNullGroupValues(contextDoc)) { nullDoc = globalDoc; if (needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } nullScore = score; } } @@ -2573,6 +2642,9 @@ public class CollapsingQParserPlugin extends QParserPlugin { } else if(this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) { this.collapsedSet.set(globalDoc); if (needsScores) { + if (!needsScores4Collapsing) { + score = scorer.score(); + } nullScores.add(score); } }