LUCENE-8411: Remove fillFields from TopFieldCollector factory methods.

This commit is contained in:
Adrien Grand 2018-07-18 12:49:31 +02:00
parent a2f113c5c6
commit 7d8fc543f0
30 changed files with 108 additions and 158 deletions

View File

@ -60,6 +60,9 @@ API Changes
no longer have an option to compute the maximum score when sorting by field. no longer have an option to compute the maximum score when sorting by field.
(Adrien Grand) (Adrien Grand)
* LUCENE-8411: TopFieldCollector no longer takes a fillFields options, it now
always fills fields. (Adrien Grand)
Changes in Runtime Behavior Changes in Runtime Behavior
* LUCENE-8333: Switch MoreLikeThis.setMaxDocFreqPct to use maxDoc instead of * LUCENE-8333: Switch MoreLikeThis.setMaxDocFreqPct to use maxDoc instead of

View File

@ -76,3 +76,9 @@ separate query:
Thanks to other optimizations that were added to Lucene 8, this query will be Thanks to other optimizations that were added to Lucene 8, this query will be
able to efficiently select the top-scoring document without having to visit able to efficiently select the top-scoring document without having to visit
all matches. all matches.
## TopFieldCollector always assumes fillFields=true ##
Because filling sort values doesn't have a significant overhead, the fillFields
option has been removed from TopFieldCollector factory methods. Everything
behaves as if it was set to true.

View File

@ -112,7 +112,7 @@ public abstract class ReadTask extends PerfTask {
// Weight public again, we can go back to // Weight public again, we can go back to
// pulling the Weight ourselves: // pulling the Weight ourselves:
TopFieldCollector collector = TopFieldCollector.create(sort, numHits, TopFieldCollector collector = TopFieldCollector.create(sort, numHits,
true, withScore(), withScore(),
withTotalHits()); withTotalHits());
searcher.search(q, collector); searcher.search(q, collector);
hits = collector.topDocs(); hits = collector.topDocs();

View File

@ -512,9 +512,8 @@ public class IndexSearcher {
@Override @Override
public TopFieldCollector newCollector() throws IOException { public TopFieldCollector newCollector() throws IOException {
final boolean fillFields = true;
// TODO: don't pay the price for accurate hit counts by default // TODO: don't pay the price for accurate hit counts by default
return TopFieldCollector.create(rewrittenSort, cappedNumHits, after, fillFields, doDocScores, true); return TopFieldCollector.create(rewrittenSort, cappedNumHits, after, doDocScores, true);
} }
@Override @Override

View File

@ -54,7 +54,7 @@ public class SortRescorer extends Rescorer {
List<LeafReaderContext> leaves = searcher.getIndexReader().leaves(); List<LeafReaderContext> leaves = searcher.getIndexReader().leaves();
TopFieldCollector collector = TopFieldCollector.create(sort, topN, true, true, true); TopFieldCollector collector = TopFieldCollector.create(sort, topN, true, true);
// Now merge sort docIDs from hits, with reader's leaves: // Now merge sort docIDs from hits, with reader's leaves:
int hitUpto = 0; int hitUpto = 0;

View File

@ -140,7 +140,7 @@ public class TopDocs {
} }
final FieldDoc fd = (FieldDoc) sd; final FieldDoc fd = (FieldDoc) sd;
if (fd.fields == null) { if (fd.fields == null) {
throw new IllegalArgumentException("shard " + shardIDX + " did not set sort field values (FieldDoc.fields is null); you must pass fillFields=true to IndexSearcher.search on each shard"); throw new IllegalArgumentException("shard " + shardIDX + " did not set sort field values (FieldDoc.fields is null)");
} }
} }
} }

View File

@ -28,7 +28,7 @@ import org.apache.lucene.util.PriorityQueue;
* A {@link Collector} that sorts by {@link SortField} using * A {@link Collector} that sorts by {@link SortField} using
* {@link FieldComparator}s. * {@link FieldComparator}s.
* <p> * <p>
* See the {@link #create(org.apache.lucene.search.Sort, int, boolean, boolean, boolean)} method * See the {@link #create(org.apache.lucene.search.Sort, int, boolean, boolean)} method
* for instantiating a TopFieldCollector. * for instantiating a TopFieldCollector.
* *
* @lucene.experimental * @lucene.experimental
@ -97,9 +97,9 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
final boolean mayNeedScoresTwice; final boolean mayNeedScoresTwice;
final boolean trackTotalHits; final boolean trackTotalHits;
public SimpleFieldCollector(Sort sort, FieldValueHitQueue<Entry> queue, int numHits, boolean fillFields, public SimpleFieldCollector(Sort sort, FieldValueHitQueue<Entry> queue, int numHits,
boolean trackDocScores, boolean trackTotalHits) { boolean trackDocScores, boolean trackTotalHits) {
super(queue, numHits, fillFields, sort.needsScores() || trackDocScores); super(queue, numHits, sort.needsScores() || trackDocScores);
this.sort = sort; this.sort = sort;
this.queue = queue; this.queue = queue;
this.trackDocScores = trackDocScores; this.trackDocScores = trackDocScores;
@ -189,9 +189,9 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
final boolean mayNeedScoresTwice; final boolean mayNeedScoresTwice;
final boolean trackTotalHits; final boolean trackTotalHits;
public PagingFieldCollector(Sort sort, FieldValueHitQueue<Entry> queue, FieldDoc after, int numHits, boolean fillFields, public PagingFieldCollector(Sort sort, FieldValueHitQueue<Entry> queue, FieldDoc after, int numHits,
boolean trackDocScores, boolean trackTotalHits) { boolean trackDocScores, boolean trackTotalHits) {
super(queue, numHits, fillFields, trackDocScores || sort.needsScores()); super(queue, numHits, trackDocScores || sort.needsScores());
this.sort = sort; this.sort = sort;
this.queue = queue; this.queue = queue;
this.trackDocScores = trackDocScores; this.trackDocScores = trackDocScores;
@ -290,8 +290,6 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
private static final ScoreDoc[] EMPTY_SCOREDOCS = new ScoreDoc[0]; private static final ScoreDoc[] EMPTY_SCOREDOCS = new ScoreDoc[0];
private final boolean fillFields;
final int numHits; final int numHits;
FieldValueHitQueue.Entry bottom = null; FieldValueHitQueue.Entry bottom = null;
boolean queueFull; boolean queueFull;
@ -304,11 +302,10 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
// internal versions. If someone will define a constructor with any other // internal versions. If someone will define a constructor with any other
// visibility, then anyone will be able to extend the class, which is not what // visibility, then anyone will be able to extend the class, which is not what
// we want. // we want.
private TopFieldCollector(PriorityQueue<Entry> pq, int numHits, boolean fillFields, boolean needsScores) { private TopFieldCollector(PriorityQueue<Entry> pq, int numHits, boolean needsScores) {
super(pq); super(pq);
this.needsScores = needsScores; this.needsScores = needsScores;
this.numHits = numHits; this.numHits = numHits;
this.fillFields = fillFields;
} }
@Override @Override
@ -328,9 +325,6 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
* the sort criteria (SortFields). * the sort criteria (SortFields).
* @param numHits * @param numHits
* the number of results to collect. * the number of results to collect.
* @param fillFields
* specifies whether the actual field values should be returned on
* the results (FieldDoc).
* @param trackDocScores * @param trackDocScores
* specifies whether document scores should be tracked and set on the * specifies whether document scores should be tracked and set on the
* results. Note that if set to false, then the results' scores will * results. Note that if set to false, then the results' scores will
@ -346,8 +340,8 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
* the sort criteria. * the sort criteria.
*/ */
public static TopFieldCollector create(Sort sort, int numHits, public static TopFieldCollector create(Sort sort, int numHits,
boolean fillFields, boolean trackDocScores, boolean trackTotalHits) { boolean trackDocScores, boolean trackTotalHits) {
return create(sort, numHits, null, fillFields, trackDocScores, trackTotalHits); return create(sort, numHits, null, trackDocScores, trackTotalHits);
} }
/** /**
@ -364,9 +358,6 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
* the number of results to collect. * the number of results to collect.
* @param after * @param after
* only hits after this FieldDoc will be collected * only hits after this FieldDoc will be collected
* @param fillFields
* specifies whether the actual field values should be returned on
* the results (FieldDoc).
* @param trackDocScores * @param trackDocScores
* specifies whether document scores should be tracked and set on the * specifies whether document scores should be tracked and set on the
* results. Note that if set to false, then the results' scores will * results. Note that if set to false, then the results' scores will
@ -383,7 +374,7 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
* the sort criteria. * the sort criteria.
*/ */
public static TopFieldCollector create(Sort sort, int numHits, FieldDoc after, public static TopFieldCollector create(Sort sort, int numHits, FieldDoc after,
boolean fillFields, boolean trackDocScores, boolean trackTotalHits) { boolean trackDocScores, boolean trackTotalHits) {
if (sort.fields.length == 0) { if (sort.fields.length == 0) {
throw new IllegalArgumentException("Sort must contain at least one field"); throw new IllegalArgumentException("Sort must contain at least one field");
@ -396,7 +387,7 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
FieldValueHitQueue<Entry> queue = FieldValueHitQueue.create(sort.fields, numHits); FieldValueHitQueue<Entry> queue = FieldValueHitQueue.create(sort.fields, numHits);
if (after == null) { if (after == null) {
return new SimpleFieldCollector(sort, queue, numHits, fillFields, trackDocScores, trackTotalHits); return new SimpleFieldCollector(sort, queue, numHits, trackDocScores, trackTotalHits);
} else { } else {
if (after.fields == null) { if (after.fields == null) {
throw new IllegalArgumentException("after.fields wasn't set; you must pass fillFields=true for the previous search"); throw new IllegalArgumentException("after.fields wasn't set; you must pass fillFields=true for the previous search");
@ -406,7 +397,7 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
throw new IllegalArgumentException("after.fields has " + after.fields.length + " values but sort has " + sort.getSort().length); throw new IllegalArgumentException("after.fields has " + after.fields.length + " values but sort has " + sort.getSort().length);
} }
return new PagingFieldCollector(sort, queue, after, numHits, fillFields, trackDocScores, trackTotalHits); return new PagingFieldCollector(sort, queue, after, numHits, trackDocScores, trackTotalHits);
} }
} }
@ -434,18 +425,11 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
@Override @Override
protected void populateResults(ScoreDoc[] results, int howMany) { protected void populateResults(ScoreDoc[] results, int howMany) {
if (fillFields) {
// avoid casting if unnecessary. // avoid casting if unnecessary.
FieldValueHitQueue<Entry> queue = (FieldValueHitQueue<Entry>) pq; FieldValueHitQueue<Entry> queue = (FieldValueHitQueue<Entry>) pq;
for (int i = howMany - 1; i >= 0; i--) { for (int i = howMany - 1; i >= 0; i--) {
results[i] = queue.fillFields(queue.pop()); results[i] = queue.fillFields(queue.pop());
} }
} else {
for (int i = howMany - 1; i >= 0; i--) {
Entry entry = pq.pop();
results[i] = new FieldDoc(entry.doc, entry.score);
}
}
} }
@Override @Override

View File

@ -2324,11 +2324,11 @@ public class TestIndexSorting extends LuceneTestCase {
System.out.println("TEST: iter=" + iter + " numHits=" + numHits); System.out.println("TEST: iter=" + iter + " numHits=" + numHits);
} }
TopFieldCollector c1 = TopFieldCollector.create(sort, numHits, true, true, true); TopFieldCollector c1 = TopFieldCollector.create(sort, numHits, true, true);
s1.search(new MatchAllDocsQuery(), c1); s1.search(new MatchAllDocsQuery(), c1);
TopDocs hits1 = c1.topDocs(); TopDocs hits1 = c1.topDocs();
TopFieldCollector c2 = TopFieldCollector.create(sort, numHits, true, true, false); TopFieldCollector c2 = TopFieldCollector.create(sort, numHits, true, false);
s2.search(new MatchAllDocsQuery(), c2); s2.search(new MatchAllDocsQuery(), c2);
TopDocs hits2 = c2.topDocs(); TopDocs hits2 = c2.topDocs();

View File

@ -386,10 +386,10 @@ public class TestBoolean2 extends LuceneTestCase {
} }
// check diff (randomized) scorers (from AssertingSearcher) produce the same results // check diff (randomized) scorers (from AssertingSearcher) produce the same results
TopFieldCollector collector = TopFieldCollector.create(sort, 1000, false, true, false); TopFieldCollector collector = TopFieldCollector.create(sort, 1000, true, false);
searcher.search(q1, collector); searcher.search(q1, collector);
ScoreDoc[] hits1 = collector.topDocs().scoreDocs; ScoreDoc[] hits1 = collector.topDocs().scoreDocs;
collector = TopFieldCollector.create(sort, 1000, false, true, false); collector = TopFieldCollector.create(sort, 1000, true, false);
searcher.search(q1, collector); searcher.search(q1, collector);
ScoreDoc[] hits2 = collector.topDocs().scoreDocs; ScoreDoc[] hits2 = collector.topDocs().scoreDocs;
tot+=hits2.length; tot+=hits2.length;
@ -402,10 +402,10 @@ public class TestBoolean2 extends LuceneTestCase {
assertEquals(mulFactor*collector.totalHits + NUM_EXTRA_DOCS/2, hits4.totalHits); assertEquals(mulFactor*collector.totalHits + NUM_EXTRA_DOCS/2, hits4.totalHits);
// test diff (randomized) scorers produce the same results on bigSearcher as well // test diff (randomized) scorers produce the same results on bigSearcher as well
collector = TopFieldCollector.create(sort, 1000 * mulFactor, false, true, false); collector = TopFieldCollector.create(sort, 1000 * mulFactor, true, false);
bigSearcher.search(q1, collector); bigSearcher.search(q1, collector);
hits1 = collector.topDocs().scoreDocs; hits1 = collector.topDocs().scoreDocs;
collector = TopFieldCollector.create(sort, 1000 * mulFactor, false, true, false); collector = TopFieldCollector.create(sort, 1000 * mulFactor, true, false);
bigSearcher.search(q1, collector); bigSearcher.search(q1, collector);
hits2 = collector.topDocs().scoreDocs; hits2 = collector.topDocs().scoreDocs;
CheckHits.checkEqual(q1, hits1, hits2); CheckHits.checkEqual(q1, hits1, hits2);

View File

@ -86,7 +86,7 @@ public class TestElevationComparator extends LuceneTestCase {
new SortField(null, SortField.Type.SCORE, reversed) new SortField(null, SortField.Type.SCORE, reversed)
); );
TopDocsCollector<Entry> topCollector = TopFieldCollector.create(sort, 50, false, true, true); TopDocsCollector<Entry> topCollector = TopFieldCollector.create(sort, 50, true, true);
searcher.search(newq.build(), topCollector); searcher.search(newq.build(), topCollector);
TopDocs topDocs = topCollector.topDocs(0, 10); TopDocs topDocs = topCollector.topDocs(0, 10);

View File

@ -281,7 +281,7 @@ public class TestTopDocsMerge extends LuceneTestCase {
topHits = searcher.search(query, numHits); topHits = searcher.search(query, numHits);
} }
} else { } else {
final TopFieldCollector c = TopFieldCollector.create(sort, numHits, true, true, true); final TopFieldCollector c = TopFieldCollector.create(sort, numHits, true, true);
searcher.search(query, c); searcher.search(query, c);
if (useFrom) { if (useFrom) {
from = TestUtil.nextInt(random(), 0, numHits - 1); from = TestUtil.nextInt(random(), 0, numHits - 1);
@ -330,7 +330,7 @@ public class TestTopDocsMerge extends LuceneTestCase {
if (sort == null) { if (sort == null) {
subHits = subSearcher.search(w, numHits); subHits = subSearcher.search(w, numHits);
} else { } else {
final TopFieldCollector c = TopFieldCollector.create(sort, numHits, true, true, true); final TopFieldCollector c = TopFieldCollector.create(sort, numHits, true, true);
subSearcher.search(w, c); subSearcher.search(w, c);
subHits = c.topDocs(0, numHits); subHits = c.topDocs(0, numHits);
} }

View File

@ -70,7 +70,7 @@ public class TestTopFieldCollector extends LuceneTestCase {
Sort[] sort = new Sort[] { new Sort(SortField.FIELD_DOC), new Sort() }; Sort[] sort = new Sort[] { new Sort(SortField.FIELD_DOC), new Sort() };
for(int i = 0; i < sort.length; i++) { for(int i = 0; i < sort.length; i++) {
Query q = new MatchAllDocsQuery(); Query q = new MatchAllDocsQuery();
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, false, TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10,
false, true); false, true);
is.search(q, tdc); is.search(q, tdc);
@ -89,7 +89,7 @@ public class TestTopFieldCollector extends LuceneTestCase {
Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() }; Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
for(int i = 0; i < sort.length; i++) { for(int i = 0; i < sort.length; i++) {
Query q = new MatchAllDocsQuery(); Query q = new MatchAllDocsQuery();
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, false, TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, false,
true); true);
is.search(q, tdc); is.search(q, tdc);
@ -110,10 +110,10 @@ public class TestTopFieldCollector extends LuceneTestCase {
// the index is not sorted // the index is not sorted
TopDocsCollector<Entry> tdc; TopDocsCollector<Entry> tdc;
if (i % 2 == 0) { if (i % 2 == 0) {
tdc = TopFieldCollector.create(sort, 10, true, false, false); tdc = TopFieldCollector.create(sort, 10, false, false);
} else { } else {
FieldDoc fieldDoc = new FieldDoc(1, Float.NaN, new Object[] { 1 }); FieldDoc fieldDoc = new FieldDoc(1, Float.NaN, new Object[] { 1 });
tdc = TopFieldCollector.create(sort, 10, fieldDoc, true, false, false); tdc = TopFieldCollector.create(sort, 10, fieldDoc, false, false);
} }
is.search(q, tdc); is.search(q, tdc);
@ -132,7 +132,7 @@ public class TestTopFieldCollector extends LuceneTestCase {
Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() }; Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
for(int i = 0; i < sort.length; i++) { for(int i = 0; i < sort.length; i++) {
Query q = new MatchAllDocsQuery(); Query q = new MatchAllDocsQuery();
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true, TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true,
true); true);
is.search(q, tdc); is.search(q, tdc);
@ -150,7 +150,7 @@ public class TestTopFieldCollector extends LuceneTestCase {
// Two Sort criteria to instantiate the multi/single comparators. // Two Sort criteria to instantiate the multi/single comparators.
Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() }; Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
for(int i = 0; i < sort.length; i++) { for(int i = 0; i < sort.length; i++) {
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true, true); TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true);
TopDocs td = tdc.topDocs(); TopDocs td = tdc.topDocs();
assertEquals(0, td.totalHits); assertEquals(0, td.totalHits);
} }
@ -183,7 +183,7 @@ public class TestTopFieldCollector extends LuceneTestCase {
final IndexSearcher searcher = new IndexSearcher(reader); final IndexSearcher searcher = new IndexSearcher(reader);
for (Sort sort : new Sort[] {new Sort(SortField.FIELD_SCORE), new Sort(new SortField("f", SortField.Type.SCORE))}) { for (Sort sort : new Sort[] {new Sort(SortField.FIELD_SCORE), new Sort(new SortField("f", SortField.Type.SCORE))}) {
for (boolean doDocScores : new boolean[] {false, true}) { for (boolean doDocScores : new boolean[] {false, true}) {
final TopFieldCollector topCollector = TopFieldCollector.create(sort, TestUtil.nextInt(random(), 1, 2), true, doDocScores, true); final TopFieldCollector topCollector = TopFieldCollector.create(sort, TestUtil.nextInt(random(), 1, 2), doDocScores, true);
final Collector assertingCollector = new Collector() { final Collector assertingCollector = new Collector() {
@Override @Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {

View File

@ -136,10 +136,9 @@ public class TestTopFieldCollectorEarlyTermination extends LuceneTestCase {
} else { } else {
after = null; after = null;
} }
final boolean fillFields = random().nextBoolean();
final boolean trackDocScores = random().nextBoolean(); final boolean trackDocScores = random().nextBoolean();
final TopFieldCollector collector1 = TopFieldCollector.create(sort, numHits, after, fillFields, trackDocScores, true); final TopFieldCollector collector1 = TopFieldCollector.create(sort, numHits, after, trackDocScores, true);
final TopFieldCollector collector2 = TopFieldCollector.create(sort, numHits, after, fillFields, trackDocScores, false); final TopFieldCollector collector2 = TopFieldCollector.create(sort, numHits, after, trackDocScores, false);
final Query query; final Query query;
if (random().nextBoolean()) { if (random().nextBoolean()) {

View File

@ -241,7 +241,7 @@ public class DrillSideways {
@Override @Override
public TopFieldCollector newCollector() throws IOException { public TopFieldCollector newCollector() throws IOException {
return TopFieldCollector.create(sort, fTopN, after, true, doDocScores, true); return TopFieldCollector.create(sort, fTopN, after, doDocScores, true);
} }
@Override @Override
@ -260,7 +260,7 @@ public class DrillSideways {
} else { } else {
final TopFieldCollector hitCollector = final TopFieldCollector hitCollector =
TopFieldCollector.create(sort, fTopN, after, true, doDocScores, true); TopFieldCollector.create(sort, fTopN, after, doDocScores, true);
DrillSidewaysResult r = search(query, hitCollector); DrillSidewaysResult r = search(query, hitCollector);
return new DrillSidewaysResult(r.facets, hitCollector.topDocs()); return new DrillSidewaysResult(r.facets, hitCollector.topDocs());
} }

View File

@ -228,10 +228,8 @@ public class FacetsCollector extends SimpleCollector implements Collector {
// remove this // remove this
throw new IllegalArgumentException("after must be a FieldDoc; got " + after); throw new IllegalArgumentException("after must be a FieldDoc; got " + after);
} }
boolean fillFields = true;
hitsCollector = TopFieldCollector.create(sort, n, hitsCollector = TopFieldCollector.create(sort, n,
(FieldDoc) after, (FieldDoc) after,
fillFields,
doDocScores, doDocScores,
true); // TODO: can we disable exact hit counts true); // TODO: can we disable exact hit counts
} else { } else {

View File

@ -270,10 +270,8 @@ public class BlockGroupingCollector extends SimpleCollector {
* within each group * within each group
* @param maxDocsPerGroup How many top documents to keep * @param maxDocsPerGroup How many top documents to keep
* within each group. * within each group.
* @param fillSortFields If true then the Comparable
* values for the sort fields will be set
*/ */
public TopGroups<?> getTopGroups(Sort withinGroupSort, int groupOffset, int withinGroupOffset, int maxDocsPerGroup, boolean fillSortFields) throws IOException { public TopGroups<?> getTopGroups(Sort withinGroupSort, int groupOffset, int withinGroupOffset, int maxDocsPerGroup) throws IOException {
//if (queueFull) { //if (queueFull) {
//System.out.println("getTopGroups groupOffset=" + groupOffset + " topNGroups=" + topNGroups); //System.out.println("getTopGroups groupOffset=" + groupOffset + " topNGroups=" + topNGroups);
@ -306,7 +304,7 @@ public class BlockGroupingCollector extends SimpleCollector {
collector = TopScoreDocCollector.create(maxDocsPerGroup); collector = TopScoreDocCollector.create(maxDocsPerGroup);
} else { } else {
// Sort by fields // Sort by fields
collector = TopFieldCollector.create(withinGroupSort, maxDocsPerGroup, fillSortFields, needsScores, true); // TODO: disable exact counts? collector = TopFieldCollector.create(withinGroupSort, maxDocsPerGroup, needsScores, true); // TODO: disable exact counts?
} }
float groupMaxScore = needsScores ? Float.NEGATIVE_INFINITY : Float.NaN; float groupMaxScore = needsScores ? Float.NEGATIVE_INFINITY : Float.NaN;
@ -325,14 +323,10 @@ public class BlockGroupingCollector extends SimpleCollector {
final Object[] groupSortValues; final Object[] groupSortValues;
if (fillSortFields) {
groupSortValues = new Comparable<?>[comparators.length]; groupSortValues = new Comparable<?>[comparators.length];
for(int sortFieldIDX=0;sortFieldIDX<comparators.length;sortFieldIDX++) { for(int sortFieldIDX=0;sortFieldIDX<comparators.length;sortFieldIDX++) {
groupSortValues[sortFieldIDX] = comparators[sortFieldIDX].value(og.comparatorSlot); groupSortValues[sortFieldIDX] = comparators[sortFieldIDX].value(og.comparatorSlot);
} }
} else {
groupSortValues = null;
}
final TopDocs topDocs = collector.topDocs(withinGroupOffset, maxDocsPerGroup); final TopDocs topDocs = collector.topDocs(withinGroupOffset, maxDocsPerGroup);

View File

@ -111,10 +111,9 @@ public class FirstPassGroupingCollector<T> extends SimpleCollector {
* number of unique groups collected is &lt;= offset. * number of unique groups collected is &lt;= offset.
* *
* @param groupOffset The offset in the collected groups * @param groupOffset The offset in the collected groups
* @param fillFields Whether to fill to {@link SearchGroup#sortValues}
* @return top groups, starting from offset * @return top groups, starting from offset
*/ */
public Collection<SearchGroup<T>> getTopGroups(int groupOffset, boolean fillFields) throws IOException { public Collection<SearchGroup<T>> getTopGroups(int groupOffset) throws IOException {
//System.out.println("FP.getTopGroups groupOffset=" + groupOffset + " fillFields=" + fillFields + " groupMap.size()=" + groupMap.size()); //System.out.println("FP.getTopGroups groupOffset=" + groupOffset + " fillFields=" + fillFields + " groupMap.size()=" + groupMap.size());
@ -140,12 +139,10 @@ public class FirstPassGroupingCollector<T> extends SimpleCollector {
// System.out.println(" group=" + (group.groupValue == null ? "null" : group.groupValue.toString())); // System.out.println(" group=" + (group.groupValue == null ? "null" : group.groupValue.toString()));
SearchGroup<T> searchGroup = new SearchGroup<>(); SearchGroup<T> searchGroup = new SearchGroup<>();
searchGroup.groupValue = group.groupValue; searchGroup.groupValue = group.groupValue;
if (fillFields) {
searchGroup.sortValues = new Object[sortFieldCount]; searchGroup.sortValues = new Object[sortFieldCount];
for(int sortFieldIDX=0;sortFieldIDX<sortFieldCount;sortFieldIDX++) { for(int sortFieldIDX=0;sortFieldIDX<sortFieldCount;sortFieldIDX++) {
searchGroup.sortValues[sortFieldIDX] = comparators[sortFieldIDX].value(group.comparatorSlot); searchGroup.sortValues[sortFieldIDX] = comparators[sortFieldIDX].value(group.comparatorSlot);
} }
}
result.add(searchGroup); result.add(searchGroup);
} }
//System.out.println(" return " + result.size() + " groups"); //System.out.println(" return " + result.size() + " groups");

View File

@ -50,7 +50,6 @@ public class GroupingSearch {
private int groupDocsOffset; private int groupDocsOffset;
private int groupDocsLimit = 1; private int groupDocsLimit = 1;
private boolean fillSortFields;
private boolean includeScores = true; private boolean includeScores = true;
private boolean includeMaxScore = true; private boolean includeMaxScore = true;
@ -147,7 +146,7 @@ public class GroupingSearch {
matchingGroupHeads = allGroupHeads ? allGroupHeadsCollector.retrieveGroupHeads(searcher.getIndexReader().maxDoc()) matchingGroupHeads = allGroupHeads ? allGroupHeadsCollector.retrieveGroupHeads(searcher.getIndexReader().maxDoc())
: new Bits.MatchNoBits(searcher.getIndexReader().maxDoc()); : new Bits.MatchNoBits(searcher.getIndexReader().maxDoc());
Collection<SearchGroup> topSearchGroups = firstPassCollector.getTopGroups(groupOffset, fillSortFields); Collection<SearchGroup> topSearchGroups = firstPassCollector.getTopGroups(groupOffset);
if (topSearchGroups == null) { if (topSearchGroups == null) {
return new TopGroups(new SortField[0], new SortField[0], 0, 0, new GroupDocs[0], Float.NaN); return new TopGroups(new SortField[0], new SortField[0], 0, 0, new GroupDocs[0], Float.NaN);
} }
@ -155,7 +154,7 @@ public class GroupingSearch {
int topNInsideGroup = groupDocsOffset + groupDocsLimit; int topNInsideGroup = groupDocsOffset + groupDocsLimit;
TopGroupsCollector secondPassCollector TopGroupsCollector secondPassCollector
= new TopGroupsCollector(grouper, topSearchGroups, groupSort, sortWithinGroup, topNInsideGroup, = new TopGroupsCollector(grouper, topSearchGroups, groupSort, sortWithinGroup, topNInsideGroup,
includeScores, includeMaxScore, fillSortFields); includeScores, includeMaxScore);
if (cachedCollector != null && cachedCollector.isCached()) { if (cachedCollector != null && cachedCollector.isCached()) {
cachedCollector.replay(secondPassCollector); cachedCollector.replay(secondPassCollector);
@ -177,7 +176,7 @@ public class GroupingSearch {
BlockGroupingCollector c = new BlockGroupingCollector(groupSort, topN, includeScores, groupEndDocs); BlockGroupingCollector c = new BlockGroupingCollector(groupSort, topN, includeScores, groupEndDocs);
searcher.search(query, c); searcher.search(query, c);
int topNInsideGroup = groupDocsOffset + groupDocsLimit; int topNInsideGroup = groupDocsOffset + groupDocsLimit;
return c.getTopGroups(sortWithinGroup, groupOffset, groupDocsOffset, topNInsideGroup, fillSortFields); return c.getTopGroups(sortWithinGroup, groupOffset, groupDocsOffset, topNInsideGroup);
} }
/** /**
@ -269,17 +268,6 @@ public class GroupingSearch {
return this; return this;
} }
/**
* Whether to also fill the sort fields per returned group and groups docs.
*
* @param fillSortFields Whether to also fill the sort fields per returned group and groups docs
* @return <code>this</code>
*/
public GroupingSearch setFillSortFields(boolean fillSortFields) {
this.fillSortFields = fillSortFields;
return this;
}
/** /**
* Whether to include the scores per doc inside a group. * Whether to include the scores per doc inside a group.
* *

View File

@ -56,12 +56,11 @@ public class TopGroupsCollector<T> extends SecondPassGroupingCollector<T> {
* @param maxDocsPerGroup the maximum number of docs to collect for each group * @param maxDocsPerGroup the maximum number of docs to collect for each group
* @param getScores if true, record the scores of all docs in each group * @param getScores if true, record the scores of all docs in each group
* @param getMaxScores if true, record the maximum score for each group * @param getMaxScores if true, record the maximum score for each group
* @param fillSortFields if true, record the sort field values for all docs
*/ */
public TopGroupsCollector(GroupSelector<T> groupSelector, Collection<SearchGroup<T>> groups, Sort groupSort, Sort withinGroupSort, public TopGroupsCollector(GroupSelector<T> groupSelector, Collection<SearchGroup<T>> groups, Sort groupSort, Sort withinGroupSort,
int maxDocsPerGroup, boolean getScores, boolean getMaxScores, boolean fillSortFields) { int maxDocsPerGroup, boolean getScores, boolean getMaxScores) {
super(groupSelector, groups, super(groupSelector, groups,
new TopDocsReducer<>(withinGroupSort, maxDocsPerGroup, getScores, getMaxScores, fillSortFields)); new TopDocsReducer<>(withinGroupSort, maxDocsPerGroup, getScores, getMaxScores));
this.groupSort = Objects.requireNonNull(groupSort); this.groupSort = Objects.requireNonNull(groupSort);
this.withinGroupSort = Objects.requireNonNull(withinGroupSort); this.withinGroupSort = Objects.requireNonNull(withinGroupSort);
this.maxDocsPerGroup = maxDocsPerGroup; this.maxDocsPerGroup = maxDocsPerGroup;
@ -115,13 +114,13 @@ public class TopGroupsCollector<T> extends SecondPassGroupingCollector<T> {
private final boolean needsScores; private final boolean needsScores;
TopDocsReducer(Sort withinGroupSort, TopDocsReducer(Sort withinGroupSort,
int maxDocsPerGroup, boolean getScores, boolean getMaxScores, boolean fillSortFields) { int maxDocsPerGroup, boolean getScores, boolean getMaxScores) {
this.needsScores = getScores || getMaxScores || withinGroupSort.needsScores(); this.needsScores = getScores || getMaxScores || withinGroupSort.needsScores();
if (withinGroupSort == Sort.RELEVANCE) { if (withinGroupSort == Sort.RELEVANCE) {
supplier = () -> new TopDocsAndMaxScoreCollector(true, TopScoreDocCollector.create(maxDocsPerGroup), null); supplier = () -> new TopDocsAndMaxScoreCollector(true, TopScoreDocCollector.create(maxDocsPerGroup), null);
} else { } else {
supplier = () -> { supplier = () -> {
TopFieldCollector topDocsCollector = TopFieldCollector.create(withinGroupSort, maxDocsPerGroup, fillSortFields, getScores, true); // TODO: disable exact counts? TopFieldCollector topDocsCollector = TopFieldCollector.create(withinGroupSort, maxDocsPerGroup, getScores, true); // TODO: disable exact counts?
MaxScoreCollector maxScoreCollector = getMaxScores ? new MaxScoreCollector() : null; MaxScoreCollector maxScoreCollector = getMaxScores ? new MaxScoreCollector() : null;
return new TopDocsAndMaxScoreCollector(false, topDocsCollector, maxScoreCollector); return new TopDocsAndMaxScoreCollector(false, topDocsCollector, maxScoreCollector);
}; };

View File

@ -246,7 +246,7 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase {
System.out.println("1st pass collector class name=" + firstCollector.getClass().getName()); System.out.println("1st pass collector class name=" + firstCollector.getClass().getName());
System.out.println("2nd pass collector class name=" + distinctValuesCollector.getClass().getName()); System.out.println("2nd pass collector class name=" + distinctValuesCollector.getClass().getName());
System.out.println("Search term=" + term); System.out.println("Search term=" + term);
System.out.println("1st pass groups=" + firstCollector.getTopGroups(0, false)); System.out.println("1st pass groups=" + firstCollector.getTopGroups(0));
System.out.println("Expected:"); System.out.println("Expected:");
printGroups(expectedResult); printGroups(expectedResult);
System.out.println("Actual:"); System.out.println("Actual:");
@ -342,7 +342,7 @@ public class DistinctValuesCollectorTest extends AbstractGroupingTestCase {
@SuppressWarnings({"unchecked","rawtypes"}) @SuppressWarnings({"unchecked","rawtypes"})
private <T extends Comparable<Object>, R extends Comparable<Object>> DistinctValuesCollector<T, R> createDistinctCountCollector(FirstPassGroupingCollector<T> firstPassGroupingCollector, private <T extends Comparable<Object>, R extends Comparable<Object>> DistinctValuesCollector<T, R> createDistinctCountCollector(FirstPassGroupingCollector<T> firstPassGroupingCollector,
String countField) throws IOException { String countField) throws IOException {
Collection<SearchGroup<T>> searchGroups = firstPassGroupingCollector.getTopGroups(0, false); Collection<SearchGroup<T>> searchGroups = firstPassGroupingCollector.getTopGroups(0);
GroupSelector<T> selector = firstPassGroupingCollector.getGroupSelector(); GroupSelector<T> selector = firstPassGroupingCollector.getGroupSelector();
if (ValueSourceGroupSelector.class.isAssignableFrom(selector.getClass())) { if (ValueSourceGroupSelector.class.isAssignableFrom(selector.getClass())) {
GroupSelector gs = new ValueSourceGroupSelector(new BytesRefFieldSource(countField), new HashMap<>()); GroupSelector gs = new ValueSourceGroupSelector(new BytesRefFieldSource(countField), new HashMap<>());

View File

@ -145,7 +145,7 @@ public class TestGrouping extends LuceneTestCase {
final FirstPassGroupingCollector<?> c1 = createRandomFirstPassCollector(groupField, groupSort, 10); final FirstPassGroupingCollector<?> c1 = createRandomFirstPassCollector(groupField, groupSort, 10);
indexSearcher.search(new TermQuery(new Term("content", "random")), c1); indexSearcher.search(new TermQuery(new Term("content", "random")), c1);
final TopGroupsCollector<?> c2 = createSecondPassCollector(c1, groupSort, Sort.RELEVANCE, 0, 5, true, true, true); final TopGroupsCollector<?> c2 = createSecondPassCollector(c1, groupSort, Sort.RELEVANCE, 0, 5, true, true);
indexSearcher.search(new TermQuery(new Term("content", "random")), c2); indexSearcher.search(new TermQuery(new Term("content", "random")), c2);
final TopGroups<?> groups = c2.getTopGroups(0); final TopGroups<?> groups = c2.getTopGroups(0);
@ -219,11 +219,10 @@ public class TestGrouping extends LuceneTestCase {
int groupOffset, int groupOffset,
int maxDocsPerGroup, int maxDocsPerGroup,
boolean getScores, boolean getScores,
boolean getMaxScores, boolean getMaxScores) throws IOException {
boolean fillSortFields) throws IOException {
Collection<SearchGroup<T>> searchGroups = firstPassGroupingCollector.getTopGroups(groupOffset, fillSortFields); Collection<SearchGroup<T>> searchGroups = firstPassGroupingCollector.getTopGroups(groupOffset);
return new TopGroupsCollector<>(firstPassGroupingCollector.getGroupSelector(), searchGroups, groupSort, sortWithinGroup, maxDocsPerGroup, getScores, getMaxScores, fillSortFields); return new TopGroupsCollector<>(firstPassGroupingCollector.getGroupSelector(), searchGroups, groupSort, sortWithinGroup, maxDocsPerGroup, getScores, getMaxScores);
} }
// Basically converts searchGroups from MutableValue to BytesRef if grouping by ValueSource // Basically converts searchGroups from MutableValue to BytesRef if grouping by ValueSource
@ -235,11 +234,10 @@ public class TestGrouping extends LuceneTestCase {
Sort sortWithinGroup, Sort sortWithinGroup,
int maxDocsPerGroup, int maxDocsPerGroup,
boolean getScores, boolean getScores,
boolean getMaxScores, boolean getMaxScores) throws IOException {
boolean fillSortFields) throws IOException {
if (firstPassGroupingCollector.getGroupSelector().getClass().isAssignableFrom(TermGroupSelector.class)) { if (firstPassGroupingCollector.getGroupSelector().getClass().isAssignableFrom(TermGroupSelector.class)) {
GroupSelector<BytesRef> selector = (GroupSelector<BytesRef>) firstPassGroupingCollector.getGroupSelector(); GroupSelector<BytesRef> selector = (GroupSelector<BytesRef>) firstPassGroupingCollector.getGroupSelector();
return new TopGroupsCollector<>(selector, searchGroups, groupSort, sortWithinGroup, maxDocsPerGroup , getScores, getMaxScores, fillSortFields); return new TopGroupsCollector<>(selector, searchGroups, groupSort, sortWithinGroup, maxDocsPerGroup , getScores, getMaxScores);
} else { } else {
ValueSource vs = new BytesRefFieldSource(groupField); ValueSource vs = new BytesRefFieldSource(groupField);
List<SearchGroup<MutableValue>> mvalSearchGroups = new ArrayList<>(searchGroups.size()); List<SearchGroup<MutableValue>> mvalSearchGroups = new ArrayList<>(searchGroups.size());
@ -256,7 +254,7 @@ public class TestGrouping extends LuceneTestCase {
mvalSearchGroups.add(sg); mvalSearchGroups.add(sg);
} }
ValueSourceGroupSelector selector = new ValueSourceGroupSelector(vs, new HashMap<>()); ValueSourceGroupSelector selector = new ValueSourceGroupSelector(vs, new HashMap<>());
return new TopGroupsCollector<>(selector, mvalSearchGroups, groupSort, sortWithinGroup, maxDocsPerGroup, getScores, getMaxScores, fillSortFields); return new TopGroupsCollector<>(selector, mvalSearchGroups, groupSort, sortWithinGroup, maxDocsPerGroup, getScores, getMaxScores);
} }
} }
@ -288,13 +286,13 @@ public class TestGrouping extends LuceneTestCase {
} }
} }
private Collection<SearchGroup<BytesRef>> getSearchGroups(FirstPassGroupingCollector<?> c, int groupOffset, boolean fillFields) throws IOException { private Collection<SearchGroup<BytesRef>> getSearchGroups(FirstPassGroupingCollector<?> c, int groupOffset) throws IOException {
if (TermGroupSelector.class.isAssignableFrom(c.getGroupSelector().getClass())) { if (TermGroupSelector.class.isAssignableFrom(c.getGroupSelector().getClass())) {
FirstPassGroupingCollector<BytesRef> collector = (FirstPassGroupingCollector<BytesRef>) c; FirstPassGroupingCollector<BytesRef> collector = (FirstPassGroupingCollector<BytesRef>) c;
return collector.getTopGroups(groupOffset, fillFields); return collector.getTopGroups(groupOffset);
} else if (ValueSourceGroupSelector.class.isAssignableFrom(c.getGroupSelector().getClass())) { } else if (ValueSourceGroupSelector.class.isAssignableFrom(c.getGroupSelector().getClass())) {
FirstPassGroupingCollector<MutableValue> collector = (FirstPassGroupingCollector<MutableValue>) c; FirstPassGroupingCollector<MutableValue> collector = (FirstPassGroupingCollector<MutableValue>) c;
Collection<SearchGroup<MutableValue>> mutableValueGroups = collector.getTopGroups(groupOffset, fillFields); Collection<SearchGroup<MutableValue>> mutableValueGroups = collector.getTopGroups(groupOffset);
if (mutableValueGroups == null) { if (mutableValueGroups == null) {
return null; return null;
} }
@ -439,7 +437,6 @@ public class TestGrouping extends LuceneTestCase {
private TopGroups<BytesRef> slowGrouping(GroupDoc[] groupDocs, private TopGroups<BytesRef> slowGrouping(GroupDoc[] groupDocs,
String searchTerm, String searchTerm,
boolean fillFields,
boolean getScores, boolean getScores,
boolean getMaxScores, boolean getMaxScores,
boolean doAllGroups, boolean doAllGroups,
@ -480,9 +477,7 @@ public class TestGrouping extends LuceneTestCase {
if (l == null) { if (l == null) {
//System.out.println(" add sortedGroup=" + groupToString(d.group)); //System.out.println(" add sortedGroup=" + groupToString(d.group));
sortedGroups.add(d.group); sortedGroups.add(d.group);
if (fillFields) {
sortedGroupFields.add(fillFields(d, groupSort)); sortedGroupFields.add(fillFields(d, groupSort));
}
l = new ArrayList<>(); l = new ArrayList<>();
groups.put(d.group, l); groups.put(d.group, l);
} }
@ -512,11 +507,7 @@ public class TestGrouping extends LuceneTestCase {
for(int docIDX=docOffset; docIDX < docIDXLimit; docIDX++) { for(int docIDX=docOffset; docIDX < docIDXLimit; docIDX++) {
final GroupDoc d = docs.get(docIDX); final GroupDoc d = docs.get(docIDX);
final FieldDoc fd; final FieldDoc fd;
if (fillFields) {
fd = new FieldDoc(d.id, getScores ? d.score : Float.NaN, fillFields(d, docSort)); fd = new FieldDoc(d.id, getScores ? d.score : Float.NaN, fillFields(d, docSort));
} else {
fd = new FieldDoc(d.id, getScores ? d.score : Float.NaN);
}
hits[docIDX-docOffset] = fd; hits[docIDX-docOffset] = fd;
} }
} else { } else {
@ -528,7 +519,7 @@ public class TestGrouping extends LuceneTestCase {
docs.size(), docs.size(),
hits, hits,
group, group,
fillFields ? sortedGroupFields.get(idx) : null); sortedGroupFields.get(idx));
} }
if (doAllGroups) { if (doAllGroups) {
@ -838,7 +829,6 @@ public class TestGrouping extends LuceneTestCase {
} }
final String searchTerm = "real" + random().nextInt(3); final String searchTerm = "real" + random().nextInt(3);
final boolean fillFields = random().nextBoolean();
boolean getScores = random().nextBoolean(); boolean getScores = random().nextBoolean();
final boolean getMaxScores = random().nextBoolean(); final boolean getMaxScores = random().nextBoolean();
final Sort groupSort = getRandomSort(); final Sort groupSort = getRandomSort();
@ -929,7 +919,7 @@ public class TestGrouping extends LuceneTestCase {
} }
// Get 1st pass top groups // Get 1st pass top groups
final Collection<SearchGroup<BytesRef>> topGroups = getSearchGroups(c1, groupOffset, fillFields); final Collection<SearchGroup<BytesRef>> topGroups = getSearchGroups(c1, groupOffset);
final TopGroups<BytesRef> groupsResult; final TopGroups<BytesRef> groupsResult;
if (VERBOSE) { if (VERBOSE) {
System.out.println("TEST: first pass topGroups"); System.out.println("TEST: first pass topGroups");
@ -956,7 +946,7 @@ public class TestGrouping extends LuceneTestCase {
} }
} }
c2 = createSecondPassCollector(c1, groupSort, docSort, groupOffset, docOffset + docsPerGroup, getScores, getMaxScores, fillFields); c2 = createSecondPassCollector(c1, groupSort, docSort, groupOffset, docOffset + docsPerGroup, getScores, getMaxScores);
if (doCache) { if (doCache) {
if (cCache.isCached()) { if (cCache.isCached()) {
if (VERBOSE) { if (VERBOSE) {
@ -987,7 +977,7 @@ public class TestGrouping extends LuceneTestCase {
} }
} }
final TopGroups<BytesRef> expectedGroups = slowGrouping(groupDocs, searchTerm, fillFields, getScores, getMaxScores, doAllGroups, groupSort, docSort, topNGroups, docsPerGroup, groupOffset, docOffset); final TopGroups<BytesRef> expectedGroups = slowGrouping(groupDocs, searchTerm, getScores, getMaxScores, doAllGroups, groupSort, docSort, topNGroups, docsPerGroup, groupOffset, docOffset);
if (VERBOSE) { if (VERBOSE) {
if (expectedGroups == null) { if (expectedGroups == null) {
@ -1033,10 +1023,10 @@ public class TestGrouping extends LuceneTestCase {
} }
} }
assertEquals(docIDToID, expectedGroups, groupsResult, true, true, true, getScores, true); assertEquals(docIDToID, expectedGroups, groupsResult, true, true, getScores, true);
// Confirm merged shards match: // Confirm merged shards match:
assertEquals(docIDToID, expectedGroups, topGroupsShards, true, false, fillFields, getScores, true); assertEquals(docIDToID, expectedGroups, topGroupsShards, true, false, getScores, true);
if (topGroupsShards != null) { if (topGroupsShards != null) {
verifyShards(shards.docStarts, topGroupsShards); verifyShards(shards.docStarts, topGroupsShards);
} }
@ -1059,7 +1049,7 @@ public class TestGrouping extends LuceneTestCase {
// Get block grouping result: // Get block grouping result:
sBlocks.search(query, c4); sBlocks.search(query, c4);
@SuppressWarnings({"unchecked","rawtypes"}) @SuppressWarnings({"unchecked","rawtypes"})
final TopGroups<BytesRef> tempTopGroupsBlocks = (TopGroups<BytesRef>) c3.getTopGroups(docSort, groupOffset, docOffset, docOffset+docsPerGroup, fillFields); final TopGroups<BytesRef> tempTopGroupsBlocks = (TopGroups<BytesRef>) c3.getTopGroups(docSort, groupOffset, docOffset, docOffset+docsPerGroup);
final TopGroups<BytesRef> groupsResultBlocks; final TopGroups<BytesRef> groupsResultBlocks;
if (doAllGroups && tempTopGroupsBlocks != null) { if (doAllGroups && tempTopGroupsBlocks != null) {
assertEquals((int) tempTopGroupsBlocks.totalGroupCount, allGroupsCollector2.getGroupCount()); assertEquals((int) tempTopGroupsBlocks.totalGroupCount, allGroupsCollector2.getGroupCount());
@ -1132,8 +1122,8 @@ public class TestGrouping extends LuceneTestCase {
} }
} }
assertEquals(docIDToIDBlocks, expectedGroups, groupsResultBlocks, false, true, true, getScores, false); assertEquals(docIDToIDBlocks, expectedGroups, groupsResultBlocks, false, true, getScores, false);
assertEquals(docIDToIDBlocks, expectedGroups, topGroupsBlockShards, false, false, fillFields, getScores, false); assertEquals(docIDToIDBlocks, expectedGroups, topGroupsBlockShards, false, false, getScores, false);
} }
r.close(); r.close();
@ -1187,7 +1177,7 @@ public class TestGrouping extends LuceneTestCase {
} }
firstPassGroupingCollectors.add(firstPassCollector); firstPassGroupingCollectors.add(firstPassCollector);
subSearchers[shardIDX].search(w, firstPassCollector); subSearchers[shardIDX].search(w, firstPassCollector);
final Collection<SearchGroup<BytesRef>> topGroups = getSearchGroups(firstPassCollector, 0, true); final Collection<SearchGroup<BytesRef>> topGroups = getSearchGroups(firstPassCollector, 0);
if (topGroups != null) { if (topGroups != null) {
if (VERBOSE) { if (VERBOSE) {
System.out.println(" shard " + shardIDX + " s=" + subSearchers[shardIDX] + " totalGroupedHitCount=?" + " " + topGroups.size() + " groups:"); System.out.println(" shard " + shardIDX + " s=" + subSearchers[shardIDX] + " totalGroupedHitCount=?" + " " + topGroups.size() + " groups:");
@ -1218,7 +1208,7 @@ public class TestGrouping extends LuceneTestCase {
final TopGroups<BytesRef>[] shardTopGroups = new TopGroups[subSearchers.length]; final TopGroups<BytesRef>[] shardTopGroups = new TopGroups[subSearchers.length];
for(int shardIDX=0;shardIDX<subSearchers.length;shardIDX++) { for(int shardIDX=0;shardIDX<subSearchers.length;shardIDX++) {
final TopGroupsCollector<?> secondPassCollector = createSecondPassCollector(firstPassGroupingCollectors.get(shardIDX), final TopGroupsCollector<?> secondPassCollector = createSecondPassCollector(firstPassGroupingCollectors.get(shardIDX),
groupField, mergedTopGroups, groupSort, docSort, docOffset + topNDocs, getScores, getMaxScores, true); groupField, mergedTopGroups, groupSort, docSort, docOffset + topNDocs, getScores, getMaxScores);
subSearchers[shardIDX].search(w, secondPassCollector); subSearchers[shardIDX].search(w, secondPassCollector);
shardTopGroups[shardIDX] = getTopGroups(secondPassCollector, 0); shardTopGroups[shardIDX] = getTopGroups(secondPassCollector, 0);
if (VERBOSE) { if (VERBOSE) {
@ -1242,7 +1232,7 @@ public class TestGrouping extends LuceneTestCase {
} }
} }
private void assertEquals(int[] docIDtoID, TopGroups<BytesRef> expected, TopGroups<BytesRef> actual, boolean verifyGroupValues, boolean verifyTotalGroupCount, boolean verifySortValues, boolean testScores, boolean idvBasedImplsUsed) { private void assertEquals(int[] docIDtoID, TopGroups<BytesRef> expected, TopGroups<BytesRef> actual, boolean verifyGroupValues, boolean verifyTotalGroupCount, boolean testScores, boolean idvBasedImplsUsed) {
if (expected == null) { if (expected == null) {
assertNull(actual); assertNull(actual);
return; return;
@ -1274,9 +1264,7 @@ public class TestGrouping extends LuceneTestCase {
} }
} }
if (verifySortValues) {
assertArrayEquals(expectedGroup.groupSortValues, actualGroup.groupSortValues); assertArrayEquals(expectedGroup.groupSortValues, actualGroup.groupSortValues);
}
// TODO // TODO
// assertEquals(expectedGroup.maxScore, actualGroup.maxScore); // assertEquals(expectedGroup.maxScore, actualGroup.maxScore);
@ -1297,12 +1285,10 @@ public class TestGrouping extends LuceneTestCase {
// TODO: too anal for now // TODO: too anal for now
//assertEquals(Float.NaN, actualFD.score); //assertEquals(Float.NaN, actualFD.score);
} }
if (verifySortValues) {
assertArrayEquals(expectedFD.fields, actualFD.fields); assertArrayEquals(expectedFD.fields, actualFD.fields);
} }
} }
} }
}
private static class ShardSearcher extends IndexSearcher { private static class ShardSearcher extends IndexSearcher {
private final List<LeafReaderContext> ctx; private final List<LeafReaderContext> ctx;

View File

@ -647,7 +647,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
//System.out.println("finalQuery=" + finalQuery); //System.out.println("finalQuery=" + finalQuery);
// Sort by weight, descending: // Sort by weight, descending:
TopFieldCollector c = TopFieldCollector.create(SORT, num, true, false, false); TopFieldCollector c = TopFieldCollector.create(SORT, num, false, false);
List<LookupResult> results = null; List<LookupResult> results = null;
SearcherManager mgr; SearcherManager mgr;
IndexSearcher searcher; IndexSearcher searcher;

View File

@ -535,7 +535,7 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
DocIdSetIterator iterator = new BitSetIterator(groupBits, 0); // cost is not useful here DocIdSetIterator iterator = new BitSetIterator(groupBits, 0); // cost is not useful here
int group; int group;
while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
Collector collector = (sort == null) ? TopScoreDocCollector.create(limit) : TopFieldCollector.create(sort, limit, false, false, true); Collector collector = (sort == null) ? TopScoreDocCollector.create(limit) : TopFieldCollector.create(sort, limit, false, true);
groups.put(group, collector); groups.put(group, collector);
} }
@ -619,7 +619,7 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
Iterator<LongCursor> iterator = groupSet.iterator(); Iterator<LongCursor> iterator = groupSet.iterator();
while (iterator.hasNext()) { while (iterator.hasNext()) {
LongCursor cursor = iterator.next(); LongCursor cursor = iterator.next();
Collector collector = (sort == null) ? TopScoreDocCollector.create(limit) : TopFieldCollector.create(sort, limit, false, false, true); Collector collector = (sort == null) ? TopScoreDocCollector.create(limit) : TopFieldCollector.create(sort, limit, false, true);
groups.put(cursor.value, collector); groups.put(cursor.value, collector);
} }

View File

@ -731,7 +731,7 @@ public class Grouping {
return totalCount == TotalCount.grouped ? allGroupsCollector : null; return totalCount == TotalCount.grouped ? allGroupsCollector : null;
} }
topGroups = format == Format.grouped ? firstPass.getTopGroups(offset, false) : firstPass.getTopGroups(0, false); topGroups = format == Format.grouped ? firstPass.getTopGroups(offset) : firstPass.getTopGroups(0);
if (topGroups == null) { if (topGroups == null) {
if (totalCount == TotalCount.grouped) { if (totalCount == TotalCount.grouped) {
allGroupsCollector = new AllGroupsCollector<>(new TermGroupSelector(groupBy)); allGroupsCollector = new AllGroupsCollector<>(new TermGroupSelector(groupBy));
@ -747,7 +747,7 @@ public class Grouping {
groupedDocsToCollect = Math.max(groupedDocsToCollect, 1); groupedDocsToCollect = Math.max(groupedDocsToCollect, 1);
Sort withinGroupSort = this.withinGroupSort != null ? this.withinGroupSort : Sort.RELEVANCE; Sort withinGroupSort = this.withinGroupSort != null ? this.withinGroupSort : Sort.RELEVANCE;
secondPass = new TopGroupsCollector<>(new TermGroupSelector(groupBy), secondPass = new TopGroupsCollector<>(new TermGroupSelector(groupBy),
topGroups, groupSort, withinGroupSort, groupedDocsToCollect, needScores, needScores, false topGroups, groupSort, withinGroupSort, groupedDocsToCollect, needScores, needScores
); );
if (totalCount == TotalCount.grouped) { if (totalCount == TotalCount.grouped) {
@ -850,7 +850,7 @@ public class Grouping {
if (withinGroupSort == null || withinGroupSort.equals(Sort.RELEVANCE)) { if (withinGroupSort == null || withinGroupSort.equals(Sort.RELEVANCE)) {
subCollector = topCollector = TopScoreDocCollector.create(groupDocsToCollect); subCollector = topCollector = TopScoreDocCollector.create(groupDocsToCollect);
} else { } else {
topCollector = TopFieldCollector.create(searcher.weightSort(withinGroupSort), groupDocsToCollect, false, needScores, true); topCollector = TopFieldCollector.create(searcher.weightSort(withinGroupSort), groupDocsToCollect, needScores, true);
if (needScores) { if (needScores) {
maxScoreCollector = new MaxScoreCollector(); maxScoreCollector = new MaxScoreCollector();
subCollector = MultiCollector.wrap(topCollector, maxScoreCollector); subCollector = MultiCollector.wrap(topCollector, maxScoreCollector);
@ -936,7 +936,7 @@ public class Grouping {
return totalCount == TotalCount.grouped ? allGroupsCollector : null; return totalCount == TotalCount.grouped ? allGroupsCollector : null;
} }
topGroups = format == Format.grouped ? firstPass.getTopGroups(offset, false) : firstPass.getTopGroups(0, false); topGroups = format == Format.grouped ? firstPass.getTopGroups(offset) : firstPass.getTopGroups(0);
if (topGroups == null) { if (topGroups == null) {
if (totalCount == TotalCount.grouped) { if (totalCount == TotalCount.grouped) {
allGroupsCollector = new AllGroupsCollector<>(newSelector()); allGroupsCollector = new AllGroupsCollector<>(newSelector());
@ -952,7 +952,7 @@ public class Grouping {
groupdDocsToCollect = Math.max(groupdDocsToCollect, 1); groupdDocsToCollect = Math.max(groupdDocsToCollect, 1);
Sort withinGroupSort = this.withinGroupSort != null ? this.withinGroupSort : Sort.RELEVANCE; Sort withinGroupSort = this.withinGroupSort != null ? this.withinGroupSort : Sort.RELEVANCE;
secondPass = new TopGroupsCollector<>(newSelector(), secondPass = new TopGroupsCollector<>(newSelector(),
topGroups, groupSort, withinGroupSort, groupdDocsToCollect, needScores, needScores, false topGroups, groupSort, withinGroupSort, groupdDocsToCollect, needScores, needScores
); );
if (totalCount == TotalCount.grouped) { if (totalCount == TotalCount.grouped) {

View File

@ -67,7 +67,7 @@ public class ReRankCollector extends TopDocsCollector {
} else { } else {
sort = sort.rewrite(searcher); sort = sort.rewrite(searcher);
//scores are needed for Rescorer (regardless of whether sort needs it) //scores are needed for Rescorer (regardless of whether sort needs it)
this.mainCollector = TopFieldCollector.create(sort, Math.max(this.reRankDocs, length), false, true, true); this.mainCollector = TopFieldCollector.create(sort, Math.max(this.reRankDocs, length), true, true);
} }
this.searcher = searcher; this.searcher = searcher;
this.reRankQueryRescorer = reRankQueryRescorer; this.reRankQueryRescorer = reRankQueryRescorer;

View File

@ -1536,11 +1536,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
final Sort weightedSort = weightSort(cmd.getSort()); final Sort weightedSort = weightSort(cmd.getSort());
final CursorMark cursor = cmd.getCursorMark(); final CursorMark cursor = cmd.getCursorMark();
// :TODO: make fillFields its own QueryCommand flag? ...
// ... see comments in populateNextCursorMarkFromTopDocs for cache issues (SOLR-5595)
final boolean fillFields = (null != cursor);
final FieldDoc searchAfter = (null != cursor ? cursor.getSearchAfterFieldDoc() : null); final FieldDoc searchAfter = (null != cursor ? cursor.getSearchAfterFieldDoc() : null);
return TopFieldCollector.create(weightedSort, len, searchAfter, fillFields, needScores, true); return TopFieldCollector.create(weightedSort, len, searchAfter, needScores, true);
} }
} }

View File

@ -130,7 +130,7 @@ public class QueryCommand implements Command<QueryCommandResult> {
if (sort == null || sort.equals(Sort.RELEVANCE)) { if (sort == null || sort.equals(Sort.RELEVANCE)) {
subCollector = topDocsCollector = TopScoreDocCollector.create(docsToCollect); subCollector = topDocsCollector = TopScoreDocCollector.create(docsToCollect);
} else { } else {
topDocsCollector = TopFieldCollector.create(sort, docsToCollect, true, needScores, true); topDocsCollector = TopFieldCollector.create(sort, docsToCollect, needScores, true);
if (needScores) { if (needScores) {
maxScoreCollector = new MaxScoreCollector(); maxScoreCollector = new MaxScoreCollector();
subCollector = MultiCollector.wrap(topDocsCollector, maxScoreCollector); subCollector = MultiCollector.wrap(topDocsCollector, maxScoreCollector);

View File

@ -125,9 +125,9 @@ public class SearchGroupsFieldCommand implements Command<SearchGroupsFieldComman
final Collection<SearchGroup<BytesRef>> topGroups; final Collection<SearchGroup<BytesRef>> topGroups;
if (firstPassGroupingCollector != null) { if (firstPassGroupingCollector != null) {
if (field.getType().getNumberType() != null) { if (field.getType().getNumberType() != null) {
topGroups = GroupConverter.fromMutable(field, firstPassGroupingCollector.getTopGroups(0, true)); topGroups = GroupConverter.fromMutable(field, firstPassGroupingCollector.getTopGroups(0));
} else { } else {
topGroups = firstPassGroupingCollector.getTopGroups(0, true); topGroups = firstPassGroupingCollector.getTopGroups(0);
} }
} else { } else {
topGroups = Collections.emptyList(); topGroups = Collections.emptyList();

View File

@ -136,11 +136,11 @@ public class TopGroupsFieldCommand implements Command<TopGroups<BytesRef>> {
ValueSource vs = fieldType.getValueSource(field, null); ValueSource vs = fieldType.getValueSource(field, null);
Collection<SearchGroup<MutableValue>> v = GroupConverter.toMutable(field, firstPhaseGroups); Collection<SearchGroup<MutableValue>> v = GroupConverter.toMutable(field, firstPhaseGroups);
secondPassCollector = new TopGroupsCollector<>(new ValueSourceGroupSelector(vs, new HashMap<>()), secondPassCollector = new TopGroupsCollector<>(new ValueSourceGroupSelector(vs, new HashMap<>()),
v, groupSort, withinGroupSort, maxDocPerGroup, needScores, needMaxScore, true v, groupSort, withinGroupSort, maxDocPerGroup, needScores, needMaxScore
); );
} else { } else {
secondPassCollector = new TopGroupsCollector<>(new TermGroupSelector(field.getName()), secondPassCollector = new TopGroupsCollector<>(new TermGroupSelector(field.getName()),
firstPhaseGroups, groupSort, withinGroupSort, maxDocPerGroup, needScores, needMaxScore, true firstPhaseGroups, groupSort, withinGroupSort, maxDocPerGroup, needScores, needMaxScore
); );
} }
collectors.add(secondPassCollector); collectors.add(secondPassCollector);

View File

@ -284,7 +284,7 @@ public class TestSort extends SolrTestCaseJ4 {
boolean trackScores = r.nextBoolean(); boolean trackScores = r.nextBoolean();
boolean scoreInOrder = r.nextBoolean(); boolean scoreInOrder = r.nextBoolean();
final TopFieldCollector topCollector = TopFieldCollector.create(sort, top, true, trackScores, true); final TopFieldCollector topCollector = TopFieldCollector.create(sort, top, trackScores, true);
final List<MyDoc> collectedDocs = new ArrayList<>(); final List<MyDoc> collectedDocs = new ArrayList<>();
// delegate and collect docs ourselves // delegate and collect docs ourselves