From cdb651976e8213d6e7e140c5e8325af09f5c3948 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 6 Jan 2015 15:03:19 +0000 Subject: [PATCH] LUCENE-5702: Move comparators to a per-leaf API. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1649818 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/CHANGES.txt | 3 + .../apache/lucene/search/FieldComparator.java | 181 +- .../lucene/search/FieldValueHitQueue.java | 44 +- .../lucene/search/LeafFieldComparator.java | 122 ++ .../lucene/search/SimpleFieldComparator.java | 42 + .../apache/lucene/search/SortRescorer.java | 7 +- .../lucene/search/TopDocsCollector.java | 2 +- .../lucene/search/TopFieldCollector.java | 1534 +++++++---------- .../lucene/search/TopScoreDocCollector.java | 348 ++-- .../lucene/search/JustCompileSearch.java | 51 +- .../search/TestElevationComparator.java | 81 +- .../org/apache/lucene/search/TestSort.java | 59 + .../lucene/search/TestTopDocsCollector.java | 37 +- .../lucene/search/TestTopFieldCollector.java | 32 +- .../search/TestTopScoreDocCollector.java | 7 +- .../expressions/ExpressionComparator.java | 6 +- .../AbstractFirstPassGroupingCollector.java | 26 +- .../AbstractSecondPassGroupingCollector.java | 7 +- .../grouping/BlockGroupingCollector.java | 26 +- .../lucene/search/grouping/SearchGroup.java | 3 +- .../FunctionAllGroupHeadsCollector.java | 20 +- .../term/TermAllGroupHeadsCollector.java | 22 +- .../join/ToParentBlockJoinCollector.java | 328 ++-- .../ToParentBlockJoinFieldComparator.java | 54 +- .../join/ToParentBlockJoinSortField.java | 1 + .../lucene/search/join/TestJoinUtil.java | 40 +- .../java/org/apache/lucene/index/Sorter.java | 9 +- .../search/BlockJoinComparatorSource.java | 137 +- .../lucene/queries/function/ValueSource.java | 14 +- .../queries/SlowCollatedStringComparator.java | 7 +- .../handler/component/QueryComponent.java | 10 +- .../component/QueryElevationComponent.java | 10 +- .../apache/solr/schema/RandomSortField.java | 5 +- .../solr/search/ExportQParserPlugin.java | 38 +- .../solr/search/ReRankQParserPlugin.java | 26 +- .../apache/solr/search/SolrIndexSearcher.java | 5 +- .../solr/search/TestRankQueryPlugin.java | 80 +- 37 files changed, 1663 insertions(+), 1761 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/search/LeafFieldComparator.java create mode 100644 lucene/core/src/java/org/apache/lucene/search/SimpleFieldComparator.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index be7e505d280..cc722e74828 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -225,6 +225,9 @@ API Changes * LUCENE-5527: The Collector API has been refactored to use a dedicated Collector per leaf. (Shikhar Bhushan, Adrien Grand) +* LUCENE-5702: The FieldComparator API has been refactor to a per-leaf API, just + like Collectors. (Adrien Grand) + * LUCENE-4246: IndexWriter.close now always closes, even if it throws an exception. The new IndexWriterConfig.setCommitOnClose (default true) determines whether close() should commit before closing. diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java b/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java index 9951d195133..142b2cd56d5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java @@ -17,11 +17,12 @@ package org.apache.lucene.search; * limitations under the License. */ + import java.io.IOException; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.util.Bits; @@ -38,46 +39,24 @@ import org.apache.lucene.util.BytesRefBuilder; * sorting, by exposing a tight interaction with {@link * FieldValueHitQueue} as it visits hits. Whenever a hit is * competitive, it's enrolled into a virtual slot, which is - * an int ranging from 0 to numHits-1. The {@link - * FieldComparator} is made aware of segment transitions - * during searching in case any internal state it's tracking - * needs to be recomputed during these transitions.

- * - *

A comparator must define these functions:

- * + * an int ranging from 0 to numHits-1. Segment transitions are + * handled by creating a dedicated per-segment + * {@link LeafFieldComparator} which also needs to interact + * with the {@link FieldValueHitQueue} but can optimize based + * on the segment to collect.

+ * + *

The following functions need to be implemented

* * + * @see LeafFieldComparator * @lucene.experimental */ public abstract class FieldComparator { @@ -104,93 +84,14 @@ public abstract class FieldComparator { */ public abstract int compare(int slot1, int slot2); - /** - * Set the bottom slot, ie the "weakest" (sorted last) - * entry in the queue. When {@link #compareBottom} is - * called, you should compare against this slot. This - * will always be called before {@link #compareBottom}. - * - * @param slot the currently weakest (sorted last) slot in the queue - */ - public abstract void setBottom(final int slot); - /** * Record the top value, for future calls to {@link - * #compareTop}. This is only called for searches that + * LeafFieldComparator#compareTop}. This is only called for searches that * use searchAfter (deep paging), and is called before any - * calls to {@link #setNextReader}. + * calls to {@link #getLeafComparator(LeafReaderContext)}. */ public abstract void setTopValue(T value); - /** - * Compare the bottom of the queue with this doc. This will - * only invoked after setBottom has been called. This - * should return the same result as {@link - * #compare(int,int)}} as if bottom were slot1 and the new - * document were slot 2. - * - *

For a search that hits many results, this method - * will be the hotspot (invoked by far the most - * frequently).

- * - * @param doc that was hit - * @return any {@code N < 0} if the doc's value is sorted after - * the bottom entry (not competitive), any {@code N > 0} if the - * doc's value is sorted before the bottom entry and {@code 0} if - * they are equal. - */ - public abstract int compareBottom(int doc) throws IOException; - - /** - * Compare the top value with this doc. This will - * only invoked after setTopValue has been called. This - * should return the same result as {@link - * #compare(int,int)}} as if topValue were slot1 and the new - * document were slot 2. This is only called for searches that - * use searchAfter (deep paging). - * - * @param doc that was hit - * @return any {@code N < 0} if the doc's value is sorted after - * the bottom entry (not competitive), any {@code N > 0} if the - * doc's value is sorted before the bottom entry and {@code 0} if - * they are equal. - */ - public abstract int compareTop(int doc) throws IOException; - - /** - * This method is called when a new hit is competitive. - * You should copy any state associated with this document - * that will be required for future comparisons, into the - * specified slot. - * - * @param slot which slot to copy the hit to - * @param doc docID relative to current reader - */ - public abstract void copy(int slot, int doc) throws IOException; - - /** - * Set a new {@link org.apache.lucene.index.LeafReaderContext}. All subsequent docIDs are relative to - * the current reader (you must add docBase if you need to - * map it to a top-level docID). - * - * @param context current reader context - * @return the comparator to use for this segment; most - * comparators can just return "this" to reuse the same - * comparator across segments - * @throws IOException if there is a low-level IO error - */ - public abstract FieldComparator setNextReader(LeafReaderContext context) throws IOException; - - /** Sets the Scorer to use in case a document's score is - * needed. - * - * @param scorer Scorer instance that you should use to - * obtain the current hit's score, if necessary. */ - public void setScorer(Scorer scorer) { - // Empty implementation since most comparators don't need the score. This - // can be overridden by those that need it. - } - /** * Return the actual value in the slot. * @@ -199,7 +100,20 @@ public abstract class FieldComparator { */ public abstract T value(int slot); - /** Returns -1 if first is less than second. Default + /** + * Get a per-segment {@link LeafFieldComparator} to collect the given + * {@link org.apache.lucene.index.LeafReaderContext}. All docIDs supplied to + * this {@link LeafFieldComparator} are relative to the current reader (you + * must add docBase if you need to map it to a top-level docID). + * + * @param context current reader context + * @return the comparator to use for this segment + * @throws IOException if there is a low-level IO error + */ + public abstract LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException; + + /** Returns a negative integer if first is less than second, + * 0 if they are equal and a positive integer otherwise. Default * impl to assume the type implements Comparable and * invoke .compareTo; be sure to override this method if * your FieldComparator's type isn't a Comparable or @@ -219,10 +133,11 @@ public abstract class FieldComparator { } } + /** * Base FieldComparator class for numeric types */ - public static abstract class NumericComparator extends FieldComparator { + public static abstract class NumericComparator extends SimpleFieldComparator { protected final T missingValue; protected final String field; protected Bits docsWithField; @@ -234,7 +149,7 @@ public abstract class FieldComparator { } @Override - public FieldComparator setNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { currentReaderValues = getNumericDocValues(context, field); if (missingValue != null) { docsWithField = DocValues.getDocsWithField(context.reader(), field); @@ -245,7 +160,6 @@ public abstract class FieldComparator { } else { docsWithField = null; } - return this; } /** Retrieves the NumericDocValues for the field in this segment */ @@ -551,7 +465,7 @@ public abstract class FieldComparator { * using {@link TopScoreDocCollector} directly (which {@link * IndexSearcher#search} uses when no {@link Sort} is * specified). */ - public static final class RelevanceComparator extends FieldComparator { + public static final class RelevanceComparator extends FieldComparator implements LeafFieldComparator { private final float[] scores; private float bottom; private Scorer scorer; @@ -581,7 +495,7 @@ public abstract class FieldComparator { } @Override - public FieldComparator setNextReader(LeafReaderContext context) { + public LeafFieldComparator getLeafComparator(LeafReaderContext context) { return this; } @@ -629,7 +543,7 @@ public abstract class FieldComparator { } /** Sorts by ascending docID */ - public static final class DocComparator extends FieldComparator { + public static final class DocComparator extends FieldComparator implements LeafFieldComparator { private final int[] docIDs; private int docBase; private int bottom; @@ -658,7 +572,7 @@ public abstract class FieldComparator { } @Override - public FieldComparator setNextReader(LeafReaderContext context) { + public LeafFieldComparator getLeafComparator(LeafReaderContext context) { // TODO: can we "map" our docIDs to the current // reader? saves having to then subtract on every // compare call @@ -686,6 +600,9 @@ public abstract class FieldComparator { int docValue = docBase + doc; return Integer.compare(topValue, docValue); } + + @Override + public void setScorer(Scorer scorer) {} } /** Sorts by field's natural Term sort order, using @@ -697,7 +614,7 @@ public abstract class FieldComparator { * to large results, this comparator will be much faster * than {@link org.apache.lucene.search.FieldComparator.TermValComparator}. For very small * result sets it may be slower. */ - public static class TermOrdValComparator extends FieldComparator { + public static class TermOrdValComparator extends FieldComparator implements LeafFieldComparator { /* Ords for each slot. @lucene.internal */ final int[] ords; @@ -841,7 +758,7 @@ public abstract class FieldComparator { } @Override - public FieldComparator setNextReader(LeafReaderContext context) throws IOException { + public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { termsIndex = getSortedDocValues(context, field); currentReaderGen++; @@ -859,7 +776,7 @@ public abstract class FieldComparator { topOrd = missingOrd; topSameReader = true; } - //System.out.println(" setNextReader topOrd=" + topOrd + " topSameReader=" + topSameReader); + //System.out.println(" getLeafComparator topOrd=" + topOrd + " topSameReader=" + topSameReader); if (bottomSlot != -1) { // Recompute bottomOrd/SameReader @@ -947,13 +864,16 @@ public abstract class FieldComparator { } return val1.compareTo(val2); } + + @Override + public void setScorer(Scorer scorer) {} } /** Sorts by field's natural Term sort order. All * comparisons are done using BytesRef.compareTo, which is * slow for medium to large result sets but possibly * very fast for very small results sets. */ - public static class TermValComparator extends FieldComparator { + public static class TermValComparator extends FieldComparator implements LeafFieldComparator { private final BytesRef[] values; private final BytesRefBuilder[] tempBRs; @@ -1019,7 +939,7 @@ public abstract class FieldComparator { } @Override - public FieldComparator setNextReader(LeafReaderContext context) throws IOException { + public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { docTerms = getBinaryDocValues(context, field); docsWithField = getDocsWithField(context, field); if (docsWithField instanceof Bits.MatchAllBits) { @@ -1075,5 +995,8 @@ public abstract class FieldComparator { } return term; } + + @Override + public void setScorer(Scorer scorer) {} } } diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldValueHitQueue.java b/lucene/core/src/java/org/apache/lucene/search/FieldValueHitQueue.java index 6dae17b9034..3a6f664225e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldValueHitQueue.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldValueHitQueue.java @@ -19,6 +19,7 @@ package org.apache.lucene.search; import java.io.IOException; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.PriorityQueue; /** @@ -53,17 +54,17 @@ public abstract class FieldValueHitQueue ext * there is just one comparator. */ private static final class OneComparatorFieldValueHitQueue extends FieldValueHitQueue { + private final int oneReverseMul; + private final FieldComparator oneComparator; public OneComparatorFieldValueHitQueue(SortField[] fields, int size) throws IOException { super(fields, size); - SortField field = fields[0]; - setComparator(0,field.getComparator(size, 0)); - oneReverseMul = field.reverse ? -1 : 1; - - reverseMul[0] = oneReverseMul; + assert fields.length == 1; + oneComparator = comparators[0]; + oneReverseMul = reverseMul[0]; } /** @@ -78,7 +79,7 @@ public abstract class FieldValueHitQueue ext assert hitA != hitB; assert hitA.slot != hitB.slot; - final int c = oneReverseMul * firstComparator.compare(hitA.slot, hitB.slot); + final int c = oneReverseMul * oneComparator.compare(hitA.slot, hitB.slot); if (c != 0) { return c > 0; } @@ -98,14 +99,6 @@ public abstract class FieldValueHitQueue ext public MultiComparatorsFieldValueHitQueue(SortField[] fields, int size) throws IOException { super(fields, size); - - int numComparators = comparators.length; - for (int i = 0; i < numComparators; ++i) { - SortField field = fields[i]; - - reverseMul[i] = field.reverse ? -1 : 1; - setComparator(i, field.getComparator(size, i)); - } } @Override @@ -130,8 +123,7 @@ public abstract class FieldValueHitQueue ext } // prevent instantiation and extension. - @SuppressWarnings({"rawtypes","unchecked"}) - private FieldValueHitQueue(SortField[] fields, int size) { + private FieldValueHitQueue(SortField[] fields, int size) throws IOException { super(size); // When we get here, fields.length is guaranteed to be > 0, therefore no // need to check it again. @@ -141,8 +133,14 @@ public abstract class FieldValueHitQueue ext // anyway. this.fields = fields; int numComparators = fields.length; - comparators = new FieldComparator[numComparators]; + comparators = new FieldComparator[numComparators]; reverseMul = new int[numComparators]; + for (int i = 0; i < numComparators; ++i) { + SortField field = fields[i]; + + reverseMul[i] = field.reverse ? -1 : 1; + comparators[i] = field.getComparator(size, i); + } } /** @@ -179,15 +177,17 @@ public abstract class FieldValueHitQueue ext return reverseMul; } - public void setComparator(int pos, FieldComparator comparator) { - if (pos==0) firstComparator = comparator; - comparators[pos] = comparator; + public LeafFieldComparator[] getComparators(LeafReaderContext context) throws IOException { + LeafFieldComparator[] comparators = new LeafFieldComparator[this.comparators.length]; + for (int i = 0; i < comparators.length; ++i) { + comparators[i] = this.comparators[i].getLeafComparator(context); + } + return comparators; } /** Stores the sort criteria being used. */ protected final SortField[] fields; - protected final FieldComparator[] comparators; // use setComparator to change this array - protected FieldComparator firstComparator; // this must always be equal to comparators[0] + protected final FieldComparator[] comparators; protected final int[] reverseMul; @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/LeafFieldComparator.java b/lucene/core/src/java/org/apache/lucene/search/LeafFieldComparator.java new file mode 100644 index 00000000000..3d1e43e88bf --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/LeafFieldComparator.java @@ -0,0 +1,122 @@ +package org.apache.lucene.search; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +/** + * Expert: comparator that gets instantiated on each leaf + * from a top-level {@link FieldComparator} instance. + * + *

A leaf comparator must define these functions:

+ * + *
    + * + *
  • {@link #setBottom} This method is called by + * {@link FieldValueHitQueue} to notify the + * FieldComparator of the current weakest ("bottom") + * slot. Note that this slot may not hold the weakest + * value according to your comparator, in cases where + * your comparator is not the primary one (ie, is only + * used to break ties from the comparators before it). + * + *
  • {@link #compareBottom} Compare a new hit (docID) + * against the "weakest" (bottom) entry in the queue. + * + *
  • {@link #compareBottom} Compare a new hit (docID) + * against the "weakest" (bottom) entry in the queue. + * + *
  • {@link #compareTop} Compare a new hit (docID) + * against the top value previously set by a call to + * {@link FieldComparator#setTopValue}. + * + *
  • {@link #copy} Installs a new hit into the + * priority queue. The {@link FieldValueHitQueue} + * calls this method when a new hit is competitive. + * + *
+ * + * @see FieldComparator + * @lucene.experimental + */ +public interface LeafFieldComparator { + + /** + * Set the bottom slot, ie the "weakest" (sorted last) + * entry in the queue. When {@link #compareBottom} is + * called, you should compare against this slot. This + * will always be called before {@link #compareBottom}. + * + * @param slot the currently weakest (sorted last) slot in the queue + */ + void setBottom(final int slot); + + /** + * Compare the bottom of the queue with this doc. This will + * only invoked after setBottom has been called. This + * should return the same result as {@link + * FieldComparator#compare(int,int)}} as if bottom were slot1 and the new + * document were slot 2. + * + *

For a search that hits many results, this method + * will be the hotspot (invoked by far the most + * frequently).

+ * + * @param doc that was hit + * @return any {@code N < 0} if the doc's value is sorted after + * the bottom entry (not competitive), any {@code N > 0} if the + * doc's value is sorted before the bottom entry and {@code 0} if + * they are equal. + */ + int compareBottom(int doc) throws IOException; + + /** + * Compare the top value with this doc. This will + * only invoked after setTopValue has been called. This + * should return the same result as {@link + * FieldComparator#compare(int,int)}} as if topValue were slot1 and the new + * document were slot 2. This is only called for searches that + * use searchAfter (deep paging). + * + * @param doc that was hit + * @return any {@code N < 0} if the doc's value is sorted after + * the bottom entry (not competitive), any {@code N > 0} if the + * doc's value is sorted before the bottom entry and {@code 0} if + * they are equal. + */ + int compareTop(int doc) throws IOException; + + /** + * This method is called when a new hit is competitive. + * You should copy any state associated with this document + * that will be required for future comparisons, into the + * specified slot. + * + * @param slot which slot to copy the hit to + * @param doc docID relative to current reader + */ + void copy(int slot, int doc) throws IOException; + + /** Sets the Scorer to use in case a document's score is + * needed. + * + * @param scorer Scorer instance that you should use to + * obtain the current hit's score, if necessary. */ + void setScorer(Scorer scorer); + +} diff --git a/lucene/core/src/java/org/apache/lucene/search/SimpleFieldComparator.java b/lucene/core/src/java/org/apache/lucene/search/SimpleFieldComparator.java new file mode 100644 index 00000000000..eb304c19657 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/search/SimpleFieldComparator.java @@ -0,0 +1,42 @@ +package org.apache.lucene.search; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.LeafReaderContext; + +/** + * Base {@link FieldComparator} implementation that is used for all contexts. + * + * @lucene.experimental + */ +public abstract class SimpleFieldComparator extends FieldComparator implements LeafFieldComparator { + + /** This method is called before collecting context. */ + protected abstract void doSetNextReader(LeafReaderContext context) throws IOException; + + @Override + public final LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { + doSetNextReader(context); + return this; + } + + @Override + public void setScorer(Scorer scorer) {} +} diff --git a/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java b/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java index 8d5144023f8..9f166cf3449 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java @@ -61,6 +61,7 @@ public class SortRescorer extends Rescorer { int endDoc = 0; int docBase = 0; + LeafCollector leafCollector = null; FakeScorer fakeScorer = new FakeScorer(); while (hitUpto < hits.length) { @@ -75,15 +76,15 @@ public class SortRescorer extends Rescorer { if (readerContext != null) { // We advanced to another segment: - collector.getLeafCollector(readerContext); - collector.setScorer(fakeScorer); + leafCollector = collector.getLeafCollector(readerContext); + leafCollector.setScorer(fakeScorer); docBase = readerContext.docBase; } fakeScorer.score = hit.score; fakeScorer.doc = docID - docBase; - collector.collect(fakeScorer.doc); + leafCollector.collect(fakeScorer.doc); hitUpto++; } diff --git a/lucene/core/src/java/org/apache/lucene/search/TopDocsCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopDocsCollector.java index cbef3b3484e..d9da8aef762 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopDocsCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopDocsCollector.java @@ -31,7 +31,7 @@ import org.apache.lucene.util.PriorityQueue; * however, you might want to consider overriding all methods, in order to avoid * a NullPointerException. */ -public abstract class TopDocsCollector extends SimpleCollector { +public abstract class TopDocsCollector implements Collector { /** This is used in case topDocs() is called with illegal parameters, or there * simply aren't (enough) results. */ diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java index 092ad3c3da3..261b589e35c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java @@ -29,80 +29,193 @@ import org.apache.lucene.util.PriorityQueue; *

* See the {@link #create(org.apache.lucene.search.Sort, int, boolean, boolean, boolean, boolean)} method * for instantiating a TopFieldCollector. - * + * * @lucene.experimental */ public abstract class TopFieldCollector extends TopDocsCollector { - + // TODO: one optimization we could do is to pre-fill // the queue with sentinel value that guaranteed to // always compare lower than a real hit; this would // save having to check queueFull on each insert + private static abstract class OneComparatorLeafCollector implements LeafCollector { + + final LeafFieldComparator comparator; + final int reverseMul; + Scorer scorer; + + OneComparatorLeafCollector(LeafFieldComparator comparator, int reverseMul) { + this.comparator = comparator; + this.reverseMul = reverseMul; + } + + @Override + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + comparator.setScorer(scorer); + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return false; + } + } + + private static abstract class MultiComparatorLeafCollector implements LeafCollector { + + final LeafFieldComparator[] comparators; + final int[] reverseMul; + final LeafFieldComparator firstComparator; + final int firstReverseMul; + Scorer scorer; + + MultiComparatorLeafCollector(LeafFieldComparator[] comparators, int[] reverseMul) { + this.comparators = comparators; + this.reverseMul = reverseMul; + firstComparator = comparators[0]; + firstReverseMul = reverseMul[0]; + } + + protected final int compareBottom(int doc) throws IOException { + int cmp = firstReverseMul * firstComparator.compareBottom(doc); + if (cmp != 0) { + return cmp; + } + for (int i = 1; i < comparators.length; ++i) { + cmp = reverseMul[i] * comparators[i].compareBottom(doc); + if (cmp != 0) { + return cmp; + } + } + return 0; + } + + protected final void copy(int slot, int doc) throws IOException { + for (LeafFieldComparator comparator : comparators) { + comparator.copy(slot, doc); + } + } + + protected final void setBottom(int slot) { + for (LeafFieldComparator comparator : comparators) { + comparator.setBottom(slot); + } + } + + protected final int compareTop(int doc) throws IOException { + int cmp = firstReverseMul * firstComparator.compareTop(doc); + if (cmp != 0) { + return cmp; + } + for (int i = 1; i < comparators.length; ++i) { + cmp = reverseMul[i] * comparators[i].compareTop(doc); + if (cmp != 0) { + return cmp; + } + } + return 0; + } + + @Override + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + for (LeafFieldComparator comparator : comparators) { + comparator.setScorer(scorer); + } + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return false; + } + } + /* * Implements a TopFieldCollector over one SortField criteria, without * tracking document scores and maxScore. */ - private static class OneComparatorNonScoringCollector extends - TopFieldCollector { + private static class NonScoringCollector extends TopFieldCollector { - FieldComparator comparator; - final int reverseMul; final FieldValueHitQueue queue; - - public OneComparatorNonScoringCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) { + + public NonScoringCollector(FieldValueHitQueue queue, int numHits, boolean fillFields) { super(queue, numHits, fillFields); this.queue = queue; - comparator = queue.getComparators()[0]; - reverseMul = queue.getReverseMul()[0]; - } - - final void updateBottom(int doc) { - // bottom.score is already set to Float.NaN in add(). - bottom.doc = docBase + doc; - bottom = pq.updateTop(); } @Override - public void collect(int doc) throws IOException { - ++totalHits; - if (queueFull) { - if ((reverseMul * comparator.compareBottom(doc)) <= 0) { - // since docs are visited in doc Id order, if compare is 0, it means - // this document is larger than anything else in the queue, and - // therefore not competitive. - return; - } - - // This hit is competitive - replace bottom element in queue & adjustTop - comparator.copy(bottom.slot, doc); - updateBottom(doc); - comparator.setBottom(bottom.slot); + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + docBase = context.docBase; + + final LeafFieldComparator[] comparators = queue.getComparators(context); + final int[] reverseMul = queue.getReverseMul(); + + if (comparators.length == 1) { + return new OneComparatorLeafCollector(comparators[0], reverseMul[0]) { + + @Override + public void collect(int doc) throws IOException { + ++totalHits; + if (queueFull) { + if ((reverseMul * comparator.compareBottom(doc)) <= 0) { + // since docs are visited in doc Id order, if compare is 0, it means + // this document is larger than anything else in the queue, and + // therefore not competitive. + return; + } + + // This hit is competitive - replace bottom element in queue & adjustTop + comparator.copy(bottom.slot, doc); + updateBottom(doc); + comparator.setBottom(bottom.slot); + } else { + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + comparator.copy(slot, doc); + add(slot, doc, Float.NaN); + if (queueFull) { + comparator.setBottom(bottom.slot); + } + } + } + + }; } else { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - comparator.copy(slot, doc); - add(slot, doc, Float.NaN); - if (queueFull) { - comparator.setBottom(bottom.slot); - } + return new MultiComparatorLeafCollector(comparators, reverseMul) { + + @Override + public void collect(int doc) throws IOException { + ++totalHits; + if (queueFull) { + if ((compareBottom(doc)) <= 0) { + // since docs are visited in doc Id order, if compare is 0, it means + // this document is larger than anything else in the queue, and + // therefore not competitive. + return; + } + + // This hit is competitive - replace bottom element in queue & adjustTop + copy(bottom.slot, doc); + updateBottom(doc); + setBottom(bottom.slot); + } else { + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + copy(slot, doc); + add(slot, doc, Float.NaN); + if (queueFull) { + setBottom(bottom.slot); + } + } + } + + }; } } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - this.docBase = context.docBase; - queue.setComparator(0, comparator.setNextReader(context)); - comparator = queue.firstComparator; - } - - @Override - public void setScorer(Scorer scorer) throws IOException { - comparator.setScorer(scorer); - } - + } /* @@ -110,44 +223,94 @@ public abstract class TopFieldCollector extends TopDocsCollector { * tracking document scores and maxScore, and assumes out of orderness in doc * Ids collection. */ - private static class OutOfOrderOneComparatorNonScoringCollector extends - OneComparatorNonScoringCollector { + private static class OutOfOrderNonScoringCollector extends TopFieldCollector { - public OutOfOrderOneComparatorNonScoringCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) { + final FieldValueHitQueue queue; + + public OutOfOrderNonScoringCollector(FieldValueHitQueue queue, int numHits, boolean fillFields) { super(queue, numHits, fillFields); + this.queue = queue; } - + @Override - public void collect(int doc) throws IOException { - ++totalHits; - if (queueFull) { - // Fastmatch: return if this hit is not competitive - final int cmp = reverseMul * comparator.compareBottom(doc); - if (cmp < 0 || (cmp == 0 && doc + docBase > bottom.doc)) { - return; - } - - // This hit is competitive - replace bottom element in queue & adjustTop - comparator.copy(bottom.slot, doc); - updateBottom(doc); - comparator.setBottom(bottom.slot); + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + docBase = context.docBase; + + final LeafFieldComparator[] comparators = queue.getComparators(context); + final int[] reverseMul = queue.getReverseMul(); + + if (comparators.length == 1) { + return new OneComparatorLeafCollector(comparators[0], reverseMul[0]) { + + @Override + public void collect(int doc) throws IOException { + ++totalHits; + if (queueFull) { + // Fastmatch: return if this hit is not competitive + final int cmp = reverseMul * comparator.compareBottom(doc); + if (cmp < 0 || (cmp == 0 && doc + docBase > bottom.doc)) { + return; + } + + // This hit is competitive - replace bottom element in queue & adjustTop + comparator.copy(bottom.slot, doc); + updateBottom(doc); + comparator.setBottom(bottom.slot); + } else { + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + comparator.copy(slot, doc); + add(slot, doc, Float.NaN); + if (queueFull) { + comparator.setBottom(bottom.slot); + } + } + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + + }; } else { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - comparator.copy(slot, doc); - add(slot, doc, Float.NaN); - if (queueFull) { - comparator.setBottom(bottom.slot); - } + return new MultiComparatorLeafCollector(comparators, reverseMul) { + + @Override + public void collect(int doc) throws IOException { + ++totalHits; + if (queueFull) { + // Fastmatch: return if this hit is not competitive + final int cmp = compareBottom(doc); + if (cmp < 0 || (cmp == 0 && doc + docBase > bottom.doc)) { + return; + } + + // This hit is competitive - replace bottom element in queue & adjustTop + copy(bottom.slot, doc); + updateBottom(doc); + setBottom(bottom.slot); + } else { + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + copy(slot, doc); + add(slot, doc, Float.NaN); + if (queueFull) { + setBottom(bottom.slot); + } + } + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + + }; } } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } } @@ -155,61 +318,99 @@ public abstract class TopFieldCollector extends TopDocsCollector { * Implements a TopFieldCollector over one SortField criteria, while tracking * document scores but no maxScore. */ - private static class OneComparatorScoringNoMaxScoreCollector extends - OneComparatorNonScoringCollector { + private static class ScoringNoMaxScoreCollector extends TopFieldCollector { - Scorer scorer; + final FieldValueHitQueue queue; - public OneComparatorScoringNoMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) { + public ScoringNoMaxScoreCollector(FieldValueHitQueue queue, int numHits, boolean fillFields) { super(queue, numHits, fillFields); - } - - final void updateBottom(int doc, float score) { - bottom.doc = docBase + doc; - bottom.score = score; - bottom = pq.updateTop(); + this.queue = queue; } @Override - public void collect(int doc) throws IOException { - ++totalHits; - if (queueFull) { - if ((reverseMul * comparator.compareBottom(doc)) <= 0) { - // since docs are visited in doc Id order, if compare is 0, it means - // this document is largest than anything else in the queue, and - // therefore not competitive. - return; - } - - // Compute the score only if the hit is competitive. - final float score = scorer.score(); + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + docBase = context.docBase; - // This hit is competitive - replace bottom element in queue & adjustTop - comparator.copy(bottom.slot, doc); - updateBottom(doc, score); - comparator.setBottom(bottom.slot); + final LeafFieldComparator[] comparators = queue.getComparators(context); + final int[] reverseMul = queue.getReverseMul(); + + if (comparators.length == 1) { + return new OneComparatorLeafCollector(comparators[0], reverseMul[0]) { + + @Override + public void collect(int doc) throws IOException { + ++totalHits; + if (queueFull) { + if ((reverseMul * comparator.compareBottom(doc)) <= 0) { + // since docs are visited in doc Id order, if compare is 0, it means + // this document is largest than anything else in the queue, and + // therefore not competitive. + return; + } + + // Compute the score only if the hit is competitive. + final float score = scorer.score(); + + // This hit is competitive - replace bottom element in queue & adjustTop + comparator.copy(bottom.slot, doc); + updateBottom(doc, score); + comparator.setBottom(bottom.slot); + } else { + // Compute the score only if the hit is competitive. + final float score = scorer.score(); + + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + comparator.copy(slot, doc); + add(slot, doc, score); + if (queueFull) { + comparator.setBottom(bottom.slot); + } + } + } + + }; } else { - // Compute the score only if the hit is competitive. - final float score = scorer.score(); + return new MultiComparatorLeafCollector(comparators, reverseMul) { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - comparator.copy(slot, doc); - add(slot, doc, score); - if (queueFull) { - comparator.setBottom(bottom.slot); - } + @Override + public void collect(int doc) throws IOException { + ++totalHits; + if (queueFull) { + if ((compareBottom(doc)) <= 0) { + // since docs are visited in doc Id order, if compare is 0, it means + // this document is largest than anything else in the queue, and + // therefore not competitive. + return; + } + + // Compute the score only if the hit is competitive. + final float score = scorer.score(); + + // This hit is competitive - replace bottom element in queue & adjustTop + copy(bottom.slot, doc); + updateBottom(doc, score); + setBottom(bottom.slot); + } else { + // Compute the score only if the hit is competitive. + final float score = scorer.score(); + + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + copy(slot, doc); + add(slot, doc, score); + if (queueFull) { + setBottom(bottom.slot); + } + } + } + + }; } } - - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - comparator.setScorer(scorer); - } - + } /* @@ -217,50 +418,105 @@ public abstract class TopFieldCollector extends TopDocsCollector { * document scores but no maxScore, and assumes out of orderness in doc Ids * collection. */ - private static class OutOfOrderOneComparatorScoringNoMaxScoreCollector extends - OneComparatorScoringNoMaxScoreCollector { + private static class OutOfOrderScoringNoMaxScoreCollector extends TopFieldCollector { - public OutOfOrderOneComparatorScoringNoMaxScoreCollector( - FieldValueHitQueue queue, int numHits, boolean fillFields) { + final FieldValueHitQueue queue; + + public OutOfOrderScoringNoMaxScoreCollector(FieldValueHitQueue queue, int numHits, boolean fillFields) { super(queue, numHits, fillFields); + this.queue = queue; } - - @Override - public void collect(int doc) throws IOException { - ++totalHits; - if (queueFull) { - // Fastmatch: return if this hit is not competitive - final int cmp = reverseMul * comparator.compareBottom(doc); - if (cmp < 0 || (cmp == 0 && doc + docBase > bottom.doc)) { - return; - } - - // Compute the score only if the hit is competitive. - final float score = scorer.score(); - // This hit is competitive - replace bottom element in queue & adjustTop - comparator.copy(bottom.slot, doc); - updateBottom(doc, score); - comparator.setBottom(bottom.slot); + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + docBase = context.docBase; + + final LeafFieldComparator[] comparators = queue.getComparators(context); + final int[] reverseMul = queue.getReverseMul(); + + if (comparators.length == 1) { + return new OneComparatorLeafCollector(comparators[0], reverseMul[0]) { + + @Override + public void collect(int doc) throws IOException { + ++totalHits; + if (queueFull) { + // Fastmatch: return if this hit is not competitive + final int cmp = reverseMul * comparator.compareBottom(doc); + if (cmp < 0 || (cmp == 0 && doc + docBase > bottom.doc)) { + return; + } + + // Compute the score only if the hit is competitive. + final float score = scorer.score(); + + // This hit is competitive - replace bottom element in queue & adjustTop + comparator.copy(bottom.slot, doc); + updateBottom(doc, score); + comparator.setBottom(bottom.slot); + } else { + // Compute the score only if the hit is competitive. + final float score = scorer.score(); + + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + comparator.copy(slot, doc); + add(slot, doc, score); + if (queueFull) { + comparator.setBottom(bottom.slot); + } + } + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + + }; } else { - // Compute the score only if the hit is competitive. - final float score = scorer.score(); + return new MultiComparatorLeafCollector(comparators, reverseMul) { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - comparator.copy(slot, doc); - add(slot, doc, score); - if (queueFull) { - comparator.setBottom(bottom.slot); - } + @Override + public void collect(int doc) throws IOException { + ++totalHits; + if (queueFull) { + // Fastmatch: return if this hit is not competitive + final int cmp = compareBottom(doc); + if (cmp < 0 || (cmp == 0 && doc + docBase > bottom.doc)) { + return; + } + + // Compute the score only if the hit is competitive. + final float score = scorer.score(); + + // This hit is competitive - replace bottom element in queue & adjustTop + copy(bottom.slot, doc); + updateBottom(doc, score); + setBottom(bottom.slot); + } else { + // Compute the score only if the hit is competitive. + final float score = scorer.score(); + + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + copy(slot, doc); + add(slot, doc, score); + if (queueFull) { + setBottom(bottom.slot); + } + } + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + }; } } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } } @@ -268,61 +524,96 @@ public abstract class TopFieldCollector extends TopDocsCollector { * Implements a TopFieldCollector over one SortField criteria, with tracking * document scores and maxScore. */ - private static class OneComparatorScoringMaxScoreCollector extends - OneComparatorNonScoringCollector { + private static class ScoringMaxScoreCollector extends TopFieldCollector { - Scorer scorer; - - public OneComparatorScoringMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) { + final FieldValueHitQueue queue; + + public ScoringMaxScoreCollector(FieldValueHitQueue queue, int numHits, boolean fillFields) { super(queue, numHits, fillFields); - // Must set maxScore to NEG_INF, or otherwise Math.max always returns NaN. - maxScore = Float.NEGATIVE_INFINITY; - } - - final void updateBottom(int doc, float score) { - bottom.doc = docBase + doc; - bottom.score = score; - bottom = pq.updateTop(); + this.queue = queue; + maxScore = Float.MIN_NORMAL; // otherwise we would keep NaN } @Override - public void collect(int doc) throws IOException { - final float score = scorer.score(); - if (score > maxScore) { - maxScore = score; - } - ++totalHits; - if (queueFull) { - if ((reverseMul * comparator.compareBottom(doc)) <= 0) { - // since docs are visited in doc Id order, if compare is 0, it means - // this document is largest than anything else in the queue, and - // therefore not competitive. - return; - } - - // This hit is competitive - replace bottom element in queue & adjustTop - comparator.copy(bottom.slot, doc); - updateBottom(doc, score); - comparator.setBottom(bottom.slot); + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + docBase = context.docBase; + + final LeafFieldComparator[] comparators = queue.getComparators(context); + final int[] reverseMul = queue.getReverseMul(); + + if (comparators.length == 1) { + return new OneComparatorLeafCollector(comparators[0], reverseMul[0]) { + + @Override + public void collect(int doc) throws IOException { + final float score = scorer.score(); + if (score > maxScore) { + maxScore = score; + } + ++totalHits; + if (queueFull) { + if (reverseMul * comparator.compareBottom(doc) <= 0) { + // since docs are visited in doc Id order, if compare is 0, it means + // this document is largest than anything else in the queue, and + // therefore not competitive. + return; + } + + // This hit is competitive - replace bottom element in queue & adjustTop + comparator.copy(bottom.slot, doc); + updateBottom(doc, score); + comparator.setBottom(bottom.slot); + } else { + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + comparator.copy(slot, doc); + add(slot, doc, score); + if (queueFull) { + comparator.setBottom(bottom.slot); + } + } + } + + }; } else { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - comparator.copy(slot, doc); - add(slot, doc, score); - if (queueFull) { - comparator.setBottom(bottom.slot); - } - } + return new MultiComparatorLeafCollector(comparators, reverseMul) { + @Override + public void collect(int doc) throws IOException { + final float score = scorer.score(); + if (score > maxScore) { + maxScore = score; + } + ++totalHits; + if (queueFull) { + if (compareBottom(doc) <= 0) { + // since docs are visited in doc Id order, if compare is 0, it means + // this document is largest than anything else in the queue, and + // therefore not competitive. + return; + } + + // This hit is competitive - replace bottom element in queue & adjustTop + copy(bottom.slot, doc); + updateBottom(doc, score); + setBottom(bottom.slot); + } else { + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + copy(slot, doc); + add(slot, doc, score); + if (queueFull) { + setBottom(bottom.slot); + } + } + } + + }; + } } - - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - super.setScorer(scorer); - } + } /* @@ -330,515 +621,102 @@ public abstract class TopFieldCollector extends TopDocsCollector { * document scores and maxScore, and assumes out of orderness in doc Ids * collection. */ - private static class OutOfOrderOneComparatorScoringMaxScoreCollector extends - OneComparatorScoringMaxScoreCollector { + private static class OutOfOrderScoringMaxScoreCollector extends TopFieldCollector { - public OutOfOrderOneComparatorScoringMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) { - super(queue, numHits, fillFields); - } - - @Override - public void collect(int doc) throws IOException { - final float score = scorer.score(); - if (score > maxScore) { - maxScore = score; - } - ++totalHits; - if (queueFull) { - // Fastmatch: return if this hit is not competitive - final int cmp = reverseMul * comparator.compareBottom(doc); - if (cmp < 0 || (cmp == 0 && doc + docBase > bottom.doc)) { - return; - } - - // This hit is competitive - replace bottom element in queue & adjustTop - comparator.copy(bottom.slot, doc); - updateBottom(doc, score); - comparator.setBottom(bottom.slot); - } else { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - comparator.copy(slot, doc); - add(slot, doc, score); - if (queueFull) { - comparator.setBottom(bottom.slot); - } - } - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - - } - - /* - * Implements a TopFieldCollector over multiple SortField criteria, without - * tracking document scores and maxScore. - */ - private static class MultiComparatorNonScoringCollector extends TopFieldCollector { - - final FieldComparator[] comparators; - final int[] reverseMul; final FieldValueHitQueue queue; - public MultiComparatorNonScoringCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) { + + public OutOfOrderScoringMaxScoreCollector(FieldValueHitQueue queue, int numHits, boolean fillFields) { super(queue, numHits, fillFields); this.queue = queue; - comparators = queue.getComparators(); - reverseMul = queue.getReverseMul(); - } - - final void updateBottom(int doc) { - // bottom.score is already set to Float.NaN in add(). - bottom.doc = docBase + doc; - bottom = pq.updateTop(); + maxScore = Float.MIN_NORMAL; // otherwise we would keep NaN } @Override - public void collect(int doc) throws IOException { - ++totalHits; - if (queueFull) { - // Fastmatch: return if this hit is not competitive - for (int i = 0;; i++) { - final int c = reverseMul[i] * comparators[i].compareBottom(doc); - if (c < 0) { - // Definitely not competitive. - return; - } else if (c > 0) { - // Definitely competitive. - break; - } else if (i == comparators.length - 1) { - // Here c=0. If we're at the last comparator, this doc is not - // competitive, since docs are visited in doc Id order, which means - // this doc cannot compete with any other document in the queue. - return; - } - } - - // This hit is competitive - replace bottom element in queue & adjustTop - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(bottom.slot, doc); - } - - updateBottom(doc); - - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } else { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(slot, doc); - } - add(slot, doc, Float.NaN); - if (queueFull) { - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } - } - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { docBase = context.docBase; - for (int i = 0; i < comparators.length; i++) { - queue.setComparator(i, comparators[i].setNextReader(context)); - } - } - @Override - public void setScorer(Scorer scorer) throws IOException { - // set the scorer on all comparators - for (int i = 0; i < comparators.length; i++) { - comparators[i].setScorer(scorer); - } - } - } - - /* - * Implements a TopFieldCollector over multiple SortField criteria, without - * tracking document scores and maxScore, and assumes out of orderness in doc - * Ids collection. - */ - private static class OutOfOrderMultiComparatorNonScoringCollector extends - MultiComparatorNonScoringCollector { - - public OutOfOrderMultiComparatorNonScoringCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) { - super(queue, numHits, fillFields); - } - - @Override - public void collect(int doc) throws IOException { - ++totalHits; - if (queueFull) { - // Fastmatch: return if this hit is not competitive - for (int i = 0;; i++) { - final int c = reverseMul[i] * comparators[i].compareBottom(doc); - if (c < 0) { - // Definitely not competitive. - return; - } else if (c > 0) { - // Definitely competitive. - break; - } else if (i == comparators.length - 1) { - // This is the equals case. - if (doc + docBase > bottom.doc) { - // Definitely not competitive - return; + final LeafFieldComparator[] comparators = queue.getComparators(context); + final int[] reverseMul = queue.getReverseMul(); + + if (comparators.length == 1) { + return new OneComparatorLeafCollector(comparators[0], reverseMul[0]) { + + @Override + public void collect(int doc) throws IOException { + final float score = scorer.score(); + if (score > maxScore) { + maxScore = score; } - break; - } - } + ++totalHits; + if (queueFull) { + // Fastmatch: return if this hit is not competitive + final int cmp = reverseMul * comparator.compareBottom(doc); + if (cmp < 0 || (cmp == 0 && doc + docBase > bottom.doc)) { + return; + } - // This hit is competitive - replace bottom element in queue & adjustTop - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(bottom.slot, doc); - } - - updateBottom(doc); - - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } else { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(slot, doc); - } - add(slot, doc, Float.NaN); - if (queueFull) { - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } - } - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - - } - - /* - * Implements a TopFieldCollector over multiple SortField criteria, with - * tracking document scores and maxScore. - */ - private static class MultiComparatorScoringMaxScoreCollector extends MultiComparatorNonScoringCollector { - - Scorer scorer; - - public MultiComparatorScoringMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) { - super(queue, numHits, fillFields); - // Must set maxScore to NEG_INF, or otherwise Math.max always returns NaN. - maxScore = Float.NEGATIVE_INFINITY; - } - - final void updateBottom(int doc, float score) { - bottom.doc = docBase + doc; - bottom.score = score; - bottom = pq.updateTop(); - } - - @Override - public void collect(int doc) throws IOException { - final float score = scorer.score(); - if (score > maxScore) { - maxScore = score; - } - ++totalHits; - if (queueFull) { - // Fastmatch: return if this hit is not competitive - for (int i = 0;; i++) { - final int c = reverseMul[i] * comparators[i].compareBottom(doc); - if (c < 0) { - // Definitely not competitive. - return; - } else if (c > 0) { - // Definitely competitive. - break; - } else if (i == comparators.length - 1) { - // Here c=0. If we're at the last comparator, this doc is not - // competitive, since docs are visited in doc Id order, which means - // this doc cannot compete with any other document in the queue. - return; - } - } - - // This hit is competitive - replace bottom element in queue & adjustTop - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(bottom.slot, doc); - } - - updateBottom(doc, score); - - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } else { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(slot, doc); - } - add(slot, doc, score); - if (queueFull) { - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } - } - } - - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - super.setScorer(scorer); - } - } - - /* - * Implements a TopFieldCollector over multiple SortField criteria, with - * tracking document scores and maxScore, and assumes out of orderness in doc - * Ids collection. - */ - private final static class OutOfOrderMultiComparatorScoringMaxScoreCollector - extends MultiComparatorScoringMaxScoreCollector { - - public OutOfOrderMultiComparatorScoringMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) { - super(queue, numHits, fillFields); - } - - @Override - public void collect(int doc) throws IOException { - final float score = scorer.score(); - if (score > maxScore) { - maxScore = score; - } - ++totalHits; - if (queueFull) { - // Fastmatch: return if this hit is not competitive - for (int i = 0;; i++) { - final int c = reverseMul[i] * comparators[i].compareBottom(doc); - if (c < 0) { - // Definitely not competitive. - return; - } else if (c > 0) { - // Definitely competitive. - break; - } else if (i == comparators.length - 1) { - // This is the equals case. - if (doc + docBase > bottom.doc) { - // Definitely not competitive - return; + // This hit is competitive - replace bottom element in queue & adjustTop + comparator.copy(bottom.slot, doc); + updateBottom(doc, score); + comparator.setBottom(bottom.slot); + } else { + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + comparator.copy(slot, doc); + add(slot, doc, score); + if (queueFull) { + comparator.setBottom(bottom.slot); + } } - break; } - } - // This hit is competitive - replace bottom element in queue & adjustTop - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(bottom.slot, doc); - } - - updateBottom(doc, score); - - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + }; } else { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(slot, doc); - } - add(slot, doc, score); - if (queueFull) { - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } - } - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } + return new MultiComparatorLeafCollector(comparators, reverseMul) { - } - - /* - * Implements a TopFieldCollector over multiple SortField criteria, with - * tracking document scores and maxScore. - */ - private static class MultiComparatorScoringNoMaxScoreCollector extends MultiComparatorNonScoringCollector { - - Scorer scorer; - - public MultiComparatorScoringNoMaxScoreCollector(FieldValueHitQueue queue, - int numHits, boolean fillFields) { - super(queue, numHits, fillFields); - } - - final void updateBottom(int doc, float score) { - bottom.doc = docBase + doc; - bottom.score = score; - bottom = pq.updateTop(); - } - - @Override - public void collect(int doc) throws IOException { - ++totalHits; - if (queueFull) { - // Fastmatch: return if this hit is not competitive - for (int i = 0;; i++) { - final int c = reverseMul[i] * comparators[i].compareBottom(doc); - if (c < 0) { - // Definitely not competitive. - return; - } else if (c > 0) { - // Definitely competitive. - break; - } else if (i == comparators.length - 1) { - // Here c=0. If we're at the last comparator, this doc is not - // competitive, since docs are visited in doc Id order, which means - // this doc cannot compete with any other document in the queue. - return; - } - } - - // This hit is competitive - replace bottom element in queue & adjustTop - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(bottom.slot, doc); - } - - // Compute score only if it is competitive. - final float score = scorer.score(); - updateBottom(doc, score); - - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } else { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(slot, doc); - } - - // Compute score only if it is competitive. - final float score = scorer.score(); - add(slot, doc, score); - if (queueFull) { - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } - } - } - - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - super.setScorer(scorer); - } - } - - /* - * Implements a TopFieldCollector over multiple SortField criteria, with - * tracking document scores and maxScore, and assumes out of orderness in doc - * Ids collection. - */ - private final static class OutOfOrderMultiComparatorScoringNoMaxScoreCollector - extends MultiComparatorScoringNoMaxScoreCollector { - - public OutOfOrderMultiComparatorScoringNoMaxScoreCollector( - FieldValueHitQueue queue, int numHits, boolean fillFields) { - super(queue, numHits, fillFields); - } - - @Override - public void collect(int doc) throws IOException { - ++totalHits; - if (queueFull) { - // Fastmatch: return if this hit is not competitive - for (int i = 0;; i++) { - final int c = reverseMul[i] * comparators[i].compareBottom(doc); - if (c < 0) { - // Definitely not competitive. - return; - } else if (c > 0) { - // Definitely competitive. - break; - } else if (i == comparators.length - 1) { - // This is the equals case. - if (doc + docBase > bottom.doc) { - // Definitely not competitive - return; + @Override + public void collect(int doc) throws IOException { + final float score = scorer.score(); + if (score > maxScore) { + maxScore = score; + } + ++totalHits; + if (queueFull) { + // Fastmatch: return if this hit is not competitive + final int cmp = compareBottom(doc); + if (cmp < 0 || (cmp == 0 && doc + docBase > bottom.doc)) { + return; + } + + // This hit is competitive - replace bottom element in queue & adjustTop + copy(bottom.slot, doc); + updateBottom(doc, score); + setBottom(bottom.slot); + } else { + // Startup transient: queue hasn't gathered numHits yet + final int slot = totalHits - 1; + // Copy hit into queue + copy(slot, doc); + add(slot, doc, score); + if (queueFull) { + setBottom(bottom.slot); + } } - break; } - } - // This hit is competitive - replace bottom element in queue & adjustTop - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(bottom.slot, doc); - } - - // Compute score only if it is competitive. - final float score = scorer.score(); - updateBottom(doc, score); - - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } else { - // Startup transient: queue hasn't gathered numHits yet - final int slot = totalHits - 1; - // Copy hit into queue - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(slot, doc); - } - - // Compute score only if it is competitive. - final float score = scorer.score(); - add(slot, doc, score); - if (queueFull) { - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); + @Override + public boolean acceptsDocsOutOfOrder() { + return true; } - } + }; } } - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - super.setScorer(scorer); - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - } /* @@ -846,30 +724,24 @@ public abstract class TopFieldCollector extends TopDocsCollector { */ private final static class PagingFieldCollector extends TopFieldCollector { - Scorer scorer; int collectedHits; - final FieldComparator[] comparators; - final int[] reverseMul; final FieldValueHitQueue queue; final boolean trackDocScores; final boolean trackMaxScore; final FieldDoc after; - int afterDoc; - - public PagingFieldCollector( - FieldValueHitQueue queue, FieldDoc after, int numHits, boolean fillFields, + + public PagingFieldCollector(FieldValueHitQueue queue, FieldDoc after, int numHits, boolean fillFields, boolean trackDocScores, boolean trackMaxScore) { super(queue, numHits, fillFields); this.queue = queue; this.trackDocScores = trackDocScores; this.trackMaxScore = trackMaxScore; this.after = after; - comparators = queue.getComparators(); - reverseMul = queue.getReverseMul(); // Must set maxScore to NEG_INF, or otherwise Math.max always returns NaN. maxScore = Float.NEGATIVE_INFINITY; + FieldComparator[] comparators = queue.comparators; // Tell all comparators their top value: for(int i=0;i { comparator.setTopValue(after.fields[i]); } } - - void updateBottom(int doc, float score) { - bottom.doc = docBase + doc; - bottom.score = score; - bottom = pq.updateTop(); - } - @SuppressWarnings({"unchecked", "rawtypes"}) @Override - public void collect(int doc) throws IOException { - //System.out.println(" collect doc=" + doc); + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + docBase = context.docBase; + final int afterDoc = after.doc - docBase; + return new MultiComparatorLeafCollector(queue.getComparators(context), queue.getReverseMul()) { - totalHits++; + @Override + public void collect(int doc) throws IOException { + //System.out.println(" collect doc=" + doc); - float score = Float.NaN; - if (trackMaxScore) { - score = scorer.score(); - if (score > maxScore) { - maxScore = score; - } - } + totalHits++; - if (queueFull) { - // Fastmatch: return if this hit is no better than - // the worst hit currently in the queue: - for (int i = 0;; i++) { - final int c = reverseMul[i] * comparators[i].compareBottom(doc); - if (c < 0) { - // Definitely not competitive. - return; - } else if (c > 0) { - // Definitely competitive. - break; - } else if (i == comparators.length - 1) { - // This is the equals case. - if (doc + docBase > bottom.doc) { - // Definitely not competitive + float score = Float.NaN; + if (trackMaxScore) { + score = scorer.score(); + if (score > maxScore) { + maxScore = score; + } + } + + if (queueFull) { + // Fastmatch: return if this hit is no better than + // the worst hit currently in the queue: + final int cmp = compareBottom(doc); + if (cmp < 0 || (cmp == 0 && doc + docBase > bottom.doc)) { + // Definitely not competitive. return; } - break; + } + + final int topCmp = compareTop(doc); + if (topCmp > 0 || (topCmp == 0 && doc <= afterDoc)) { + // Already collected on a previous page + return; + } + + if (queueFull) { + // This hit is competitive - replace bottom element in queue & adjustTop + copy(bottom.slot, doc); + + // Compute score only if it is competitive. + if (trackDocScores && !trackMaxScore) { + score = scorer.score(); + } + updateBottom(doc, score); + + setBottom(bottom.slot); + } else { + collectedHits++; + + // Startup transient: queue hasn't gathered numHits yet + final int slot = collectedHits - 1; + //System.out.println(" slot=" + slot); + // Copy hit into queue + copy(slot, doc); + + // Compute score only if it is competitive. + if (trackDocScores && !trackMaxScore) { + score = scorer.score(); + } + bottom = pq.add(new Entry(slot, docBase + doc, score)); + queueFull = collectedHits == numHits; + if (queueFull) { + setBottom(bottom.slot); + } } } - } - // Check if this hit was already collected on a - // previous page: - boolean sameValues = true; - for(int compIDX=0;compIDX 0) { - // Already collected on a previous page - //System.out.println(" skip: before"); - return; - } else if (cmp < 0) { - // Not yet collected - sameValues = false; - //System.out.println(" keep: after; reverseMul=" + reverseMul[compIDX]); - break; + @Override + public boolean acceptsDocsOutOfOrder() { + return true; } - } - - // Tie-break by docID: - if (sameValues && doc <= afterDoc) { - // Already collected on a previous page - //System.out.println(" skip: tie-break"); - return; - } - - if (queueFull) { - // This hit is competitive - replace bottom element in queue & adjustTop - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(bottom.slot, doc); - } - - // Compute score only if it is competitive. - if (trackDocScores && !trackMaxScore) { - score = scorer.score(); - } - updateBottom(doc, score); - - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } else { - collectedHits++; - - // Startup transient: queue hasn't gathered numHits yet - final int slot = collectedHits - 1; - //System.out.println(" slot=" + slot); - // Copy hit into queue - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(slot, doc); - } - - // Compute score only if it is competitive. - if (trackDocScores && !trackMaxScore) { - score = scorer.score(); - } - bottom = pq.add(new Entry(slot, docBase + doc, score)); - queueFull = collectedHits == numHits; - if (queueFull) { - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } - } + }; } - @Override - public void setScorer(Scorer scorer) { - this.scorer = scorer; - for (int i = 0; i < comparators.length; i++) { - comparators[i].setScorer(scorer); - } - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - docBase = context.docBase; - afterDoc = after.doc - docBase; - for (int i = 0; i < comparators.length; i++) { - queue.setComparator(i, comparators[i].setNextReader(context)); - } - } } private static final ScoreDoc[] EMPTY_SCOREDOCS = new ScoreDoc[0]; - + private final boolean fillFields; /* @@ -1024,7 +841,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { FieldValueHitQueue.Entry bottom = null; boolean queueFull; int docBase; - + // Declaring the constructor private prevents extending this class by anyone // else. Note that the class cannot be final since it's extended by the // internal versions. If someone will define a constructor with any other @@ -1043,7 +860,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { *

NOTE: The instances returned by this method * pre-allocate a full array of length * numHits. - * + * * @param sort * the sort criteria (SortFields). * @param numHits @@ -1067,7 +884,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { * trackDocScores to true as well. * @param docsScoredInOrder * specifies whether documents are scored in doc Id order or not by - * the given {@link Scorer} in {@link #setScorer(Scorer)}. + * the given {@link Scorer} in {@link LeafCollector#setScorer(Scorer)}. * @return a {@link TopFieldCollector} instance which will sort the results by * the sort criteria. * @throws IOException if there is a low-level I/O error @@ -1086,7 +903,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { *

NOTE: The instances returned by this method * pre-allocate a full array of length * numHits. - * + * * @param sort * the sort criteria (SortFields). * @param numHits @@ -1112,7 +929,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { * trackDocScores to true as well. * @param docsScoredInOrder * specifies whether documents are scored in doc Id order or not by - * the given {@link Scorer} in {@link #setScorer(Scorer)}. + * the given {@link Scorer} in {@link LeafCollector#setScorer(Scorer)}. * @return a {@link TopFieldCollector} instance which will sort the results by * the sort criteria. * @throws IOException if there is a low-level I/O error @@ -1125,7 +942,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { if (sort.fields.length == 0) { throw new IllegalArgumentException("Sort must contain at least one field"); } - + if (numHits <= 0) { throw new IllegalArgumentException("numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count"); } @@ -1133,42 +950,21 @@ public abstract class TopFieldCollector extends TopDocsCollector { FieldValueHitQueue queue = FieldValueHitQueue.create(sort.fields, numHits); if (after == null) { - if (queue.getComparators().length == 1) { - if (docsScoredInOrder) { - if (trackMaxScore) { - return new OneComparatorScoringMaxScoreCollector(queue, numHits, fillFields); - } else if (trackDocScores) { - return new OneComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); - } else { - return new OneComparatorNonScoringCollector(queue, numHits, fillFields); - } - } else { - if (trackMaxScore) { - return new OutOfOrderOneComparatorScoringMaxScoreCollector(queue, numHits, fillFields); - } else if (trackDocScores) { - return new OutOfOrderOneComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); - } else { - return new OutOfOrderOneComparatorNonScoringCollector(queue, numHits, fillFields); - } - } - } - - // multiple comparators. if (docsScoredInOrder) { if (trackMaxScore) { - return new MultiComparatorScoringMaxScoreCollector(queue, numHits, fillFields); + return new ScoringMaxScoreCollector(queue, numHits, fillFields); } else if (trackDocScores) { - return new MultiComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); + return new ScoringNoMaxScoreCollector(queue, numHits, fillFields); } else { - return new MultiComparatorNonScoringCollector(queue, numHits, fillFields); + return new NonScoringCollector(queue, numHits, fillFields); } } else { if (trackMaxScore) { - return new OutOfOrderMultiComparatorScoringMaxScoreCollector(queue, numHits, fillFields); + return new OutOfOrderScoringMaxScoreCollector(queue, numHits, fillFields); } else if (trackDocScores) { - return new OutOfOrderMultiComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); + return new OutOfOrderScoringNoMaxScoreCollector(queue, numHits, fillFields); } else { - return new OutOfOrderMultiComparatorNonScoringCollector(queue, numHits, fillFields); + return new OutOfOrderNonScoringCollector(queue, numHits, fillFields); } } } else { @@ -1183,12 +979,24 @@ public abstract class TopFieldCollector extends TopDocsCollector { return new PagingFieldCollector(queue, after, numHits, fillFields, trackDocScores, trackMaxScore); } } - + final void add(int slot, int doc, float score) { bottom = pq.add(new Entry(slot, docBase + doc, score)); queueFull = totalHits == numHits; } + final void updateBottom(int doc) { + // bottom.score is already set to Float.NaN in add(). + bottom.doc = docBase + doc; + bottom = pq.updateTop(); + } + + final void updateBottom(int doc, float score) { + bottom.doc = docBase + doc; + bottom.score = score; + bottom = pq.updateTop(); + } + /* * Only the following callback methods need to be overridden since * topDocs(int, int) calls them to return the results. @@ -1209,7 +1017,7 @@ public abstract class TopFieldCollector extends TopDocsCollector { } } } - + @Override protected TopDocs newTopDocs(ScoreDoc[] results, int start) { if (results == null) { @@ -1218,12 +1026,8 @@ public abstract class TopFieldCollector extends TopDocsCollector { maxScore = Float.NaN; } - // If this is a maxScoring tracking collector and there were no results, + // If this is a maxScoring tracking collector and there were no results, return new TopFieldDocs(totalHits, results, ((FieldValueHitQueue) pq).getFields(), maxScore); } - - @Override - public boolean acceptsDocsOutOfOrder() { - return false; - } + } diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java index e343e677e39..32bae08ba62 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java +++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java @@ -36,198 +36,189 @@ import org.apache.lucene.index.LeafReaderContext; */ public abstract class TopScoreDocCollector extends TopDocsCollector { - // Assumes docs are scored in order. - private static class InOrderTopScoreDocCollector extends TopScoreDocCollector { - private InOrderTopScoreDocCollector(int numHits) { - super(numHits); + private abstract static class ScorerLeafCollector implements LeafCollector { + + final boolean scoreDocsInOrder; + + ScorerLeafCollector(boolean scoreDocsInOrder) { + this.scoreDocsInOrder = scoreDocsInOrder; } - + + Scorer scorer; + @Override - public void collect(int doc) throws IOException { - float score = scorer.score(); - - // This collector cannot handle these scores: - assert score != Float.NEGATIVE_INFINITY; - assert !Float.isNaN(score); - - totalHits++; - if (score <= pqTop.score) { - // Since docs are returned in-order (i.e., increasing doc Id), a document - // with equal score to pqTop.score cannot compete since HitQueue favors - // documents with lower doc Ids. Therefore reject those docs too. - return; - } - pqTop.doc = doc + docBase; - pqTop.score = score; - pqTop = pq.updateTop(); + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; } - + @Override public boolean acceptsDocsOutOfOrder() { - return false; + return scoreDocsInOrder == false; } + } - - // Assumes docs are scored in order. - private static class InOrderPagingScoreDocCollector extends TopScoreDocCollector { + + private static class SimpleTopScoreDocCollector extends TopScoreDocCollector { + + private final boolean scoreDocsInOrder; + + SimpleTopScoreDocCollector(int numHits, boolean scoreDocsInOrder) { + super(numHits); + this.scoreDocsInOrder = scoreDocsInOrder; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) + throws IOException { + final int docBase = context.docBase; + if (scoreDocsInOrder) { + return new ScorerLeafCollector(scoreDocsInOrder) { + + @Override + public void collect(int doc) throws IOException { + float score = scorer.score(); + + // This collector cannot handle these scores: + assert score != Float.NEGATIVE_INFINITY; + assert !Float.isNaN(score); + + totalHits++; + if (score <= pqTop.score) { + // Since docs are returned in-order (i.e., increasing doc Id), a document + // with equal score to pqTop.score cannot compete since HitQueue favors + // documents with lower doc Ids. Therefore reject those docs too. + return; + } + pqTop.doc = doc + docBase; + pqTop.score = score; + pqTop = pq.updateTop(); + } + + }; + } else { + return new ScorerLeafCollector(scoreDocsInOrder) { + + @Override + public void collect(int doc) throws IOException { + float score = scorer.score(); + + // This collector cannot handle NaN + assert !Float.isNaN(score); + + totalHits++; + if (score < pqTop.score) { + // Doesn't compete w/ bottom entry in queue + return; + } + doc += docBase; + if (score == pqTop.score && doc > pqTop.doc) { + // Break tie in score by doc ID: + return; + } + pqTop.doc = doc; + pqTop.score = score; + pqTop = pq.updateTop(); + } + + }; + } + } + + } + + private static class PagingTopScoreDocCollector extends TopScoreDocCollector { + + private final boolean scoreDocsInOrder; private final ScoreDoc after; - // this is always after.doc - docBase, to save an add when score == after.score - private int afterDoc; private int collectedHits; - private InOrderPagingScoreDocCollector(ScoreDoc after, int numHits) { + PagingTopScoreDocCollector(int numHits, boolean scoreDocsInOrder, ScoreDoc after) { super(numHits); + this.scoreDocsInOrder = scoreDocsInOrder; this.after = after; - } - - @Override - public void collect(int doc) throws IOException { - float score = scorer.score(); - - // This collector cannot handle these scores: - assert score != Float.NEGATIVE_INFINITY; - assert !Float.isNaN(score); - - totalHits++; - - if (score > after.score || (score == after.score && doc <= afterDoc)) { - // hit was collected on a previous page - return; - } - - if (score <= pqTop.score) { - // Since docs are returned in-order (i.e., increasing doc Id), a document - // with equal score to pqTop.score cannot compete since HitQueue favors - // documents with lower doc Ids. Therefore reject those docs too. - return; - } - collectedHits++; - pqTop.doc = doc + docBase; - pqTop.score = score; - pqTop = pq.updateTop(); - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return false; - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - super.doSetNextReader(context); - afterDoc = after.doc - context.docBase; + this.collectedHits = 0; } @Override protected int topDocsSize() { return collectedHits < pq.size() ? collectedHits : pq.size(); } - + @Override protected TopDocs newTopDocs(ScoreDoc[] results, int start) { return results == null ? new TopDocs(totalHits, new ScoreDoc[0], Float.NaN) : new TopDocs(totalHits, results); } - } - // Assumes docs are scored out of order. - private static class OutOfOrderTopScoreDocCollector extends TopScoreDocCollector { - private OutOfOrderTopScoreDocCollector(int numHits) { - super(numHits); - } - @Override - public void collect(int doc) throws IOException { - float score = scorer.score(); + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + final int docBase = context.docBase; + final int afterDoc = after.doc - context.docBase; + if (scoreDocsInOrder) { + return new ScorerLeafCollector(scoreDocsInOrder) { + @Override + public void collect(int doc) throws IOException { + float score = scorer.score(); - // This collector cannot handle NaN - assert !Float.isNaN(score); + // This collector cannot handle these scores: + assert score != Float.NEGATIVE_INFINITY; + assert !Float.isNaN(score); - totalHits++; - if (score < pqTop.score) { - // Doesn't compete w/ bottom entry in queue - return; + totalHits++; + + if (score > after.score || (score == after.score && doc <= afterDoc)) { + // hit was collected on a previous page + return; + } + + if (score <= pqTop.score) { + // Since docs are returned in-order (i.e., increasing doc Id), a document + // with equal score to pqTop.score cannot compete since HitQueue favors + // documents with lower doc Ids. Therefore reject those docs too. + return; + } + collectedHits++; + pqTop.doc = doc + docBase; + pqTop.score = score; + pqTop = pq.updateTop(); + } + }; + } else { + return new ScorerLeafCollector(scoreDocsInOrder) { + @Override + public void collect(int doc) throws IOException { + float score = scorer.score(); + + // This collector cannot handle NaN + assert !Float.isNaN(score); + + totalHits++; + if (score > after.score || (score == after.score && doc <= afterDoc)) { + // hit was collected on a previous page + return; + } + if (score < pqTop.score) { + // Doesn't compete w/ bottom entry in queue + return; + } + doc += docBase; + if (score == pqTop.score && doc > pqTop.doc) { + // Break tie in score by doc ID: + return; + } + collectedHits++; + pqTop.doc = doc; + pqTop.score = score; + pqTop = pq.updateTop(); + } + }; } - doc += docBase; - if (score == pqTop.score && doc > pqTop.doc) { - // Break tie in score by doc ID: - return; - } - pqTop.doc = doc; - pqTop.score = score; - pqTop = pq.updateTop(); - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - } - - // Assumes docs are scored out of order. - private static class OutOfOrderPagingScoreDocCollector extends TopScoreDocCollector { - private final ScoreDoc after; - // this is always after.doc - docBase, to save an add when score == after.score - private int afterDoc; - private int collectedHits; - - private OutOfOrderPagingScoreDocCollector(ScoreDoc after, int numHits) { - super(numHits); - this.after = after; - } - - @Override - public void collect(int doc) throws IOException { - float score = scorer.score(); - - // This collector cannot handle NaN - assert !Float.isNaN(score); - - totalHits++; - if (score > after.score || (score == after.score && doc <= afterDoc)) { - // hit was collected on a previous page - return; - } - if (score < pqTop.score) { - // Doesn't compete w/ bottom entry in queue - return; - } - doc += docBase; - if (score == pqTop.score && doc > pqTop.doc) { - // Break tie in score by doc ID: - return; - } - collectedHits++; - pqTop.doc = doc; - pqTop.score = score; - pqTop = pq.updateTop(); - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - super.doSetNextReader(context); - afterDoc = after.doc - context.docBase; - } - - @Override - protected int topDocsSize() { - return collectedHits < pq.size() ? collectedHits : pq.size(); - } - - @Override - protected TopDocs newTopDocs(ScoreDoc[] results, int start) { - return results == null ? new TopDocs(totalHits, new ScoreDoc[0], Float.NaN) : new TopDocs(totalHits, results); } } /** * Creates a new {@link TopScoreDocCollector} given the number of hits to * collect and whether documents are scored in order by the input - * {@link Scorer} to {@link #setScorer(Scorer)}. + * {@link Scorer} to {@link LeafCollector#setScorer(Scorer)}. * *

NOTE: The instances returned by this method * pre-allocate a full array of length @@ -237,11 +228,11 @@ public abstract class TopScoreDocCollector extends TopDocsCollector { public static TopScoreDocCollector create(int numHits, boolean docsScoredInOrder) { return create(numHits, null, docsScoredInOrder); } - + /** * Creates a new {@link TopScoreDocCollector} given the number of hits to * collect, the bottom of the previous page, and whether documents are scored in order by the input - * {@link Scorer} to {@link #setScorer(Scorer)}. + * {@link Scorer} to {@link LeafCollector#setScorer(Scorer)}. * *

NOTE: The instances returned by this method * pre-allocate a full array of length @@ -249,27 +240,20 @@ public abstract class TopScoreDocCollector extends TopDocsCollector { * objects. */ public static TopScoreDocCollector create(int numHits, ScoreDoc after, boolean docsScoredInOrder) { - + if (numHits <= 0) { throw new IllegalArgumentException("numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count"); } - - if (docsScoredInOrder) { - return after == null - ? new InOrderTopScoreDocCollector(numHits) - : new InOrderPagingScoreDocCollector(after, numHits); + + if (after == null) { + return new SimpleTopScoreDocCollector(numHits, docsScoredInOrder); } else { - return after == null - ? new OutOfOrderTopScoreDocCollector(numHits) - : new OutOfOrderPagingScoreDocCollector(after, numHits); + return new PagingTopScoreDocCollector(numHits, docsScoredInOrder, after); } - } - + ScoreDoc pqTop; - int docBase = 0; - Scorer scorer; - + // prevents instantiation private TopScoreDocCollector(int numHits) { super(new HitQueue(numHits, true)); @@ -283,7 +267,7 @@ public abstract class TopScoreDocCollector extends TopDocsCollector { if (results == null) { return EMPTY_TOPDOCS; } - + // We need to compute maxScore in order to set it in TopDocs. If start == 0, // it means the largest element is already in results, use its score as // maxScore. Otherwise pop everything else, until the largest element is @@ -295,17 +279,7 @@ public abstract class TopScoreDocCollector extends TopDocsCollector { for (int i = pq.size(); i > 1; i--) { pq.pop(); } maxScore = pq.pop().score; } - + return new TopDocs(totalHits, results, maxScore); } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - docBase = context.docBase; - } - - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } } diff --git a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java index cf11e265818..1f333398cca 100644 --- a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java +++ b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java @@ -98,43 +98,23 @@ final class JustCompileSearch { static final class JustCompileFieldComparator extends FieldComparator { - @Override - public int compare(int slot1, int slot2) { - throw new UnsupportedOperationException(UNSUPPORTED_MSG); - } - - @Override - public int compareBottom(int doc) { - throw new UnsupportedOperationException(UNSUPPORTED_MSG); - } - - @Override - public void copy(int slot, int doc) { - throw new UnsupportedOperationException(UNSUPPORTED_MSG); - } - - @Override - public void setBottom(int slot) { - throw new UnsupportedOperationException(UNSUPPORTED_MSG); - } - @Override public void setTopValue(Object value) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } - @Override - public FieldComparator setNextReader(LeafReaderContext context) { - throw new UnsupportedOperationException(UNSUPPORTED_MSG); - } - @Override public Object value(int slot) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @Override - public int compareTop(int doc) { + public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } + + @Override + public int compare(int slot1, int slot2) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } } @@ -260,23 +240,8 @@ final class JustCompileSearch { } @Override - public void collect(int doc) { - throw new UnsupportedOperationException(UNSUPPORTED_MSG); - } - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - throw new UnsupportedOperationException(UNSUPPORTED_MSG); - } - - @Override - public void setScorer(Scorer scorer) { - throw new UnsupportedOperationException(UNSUPPORTED_MSG); - } - - @Override - public boolean acceptsDocsOutOfOrder() { - throw new UnsupportedOperationException(UNSUPPORTED_MSG); + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + throw new UnsupportedOperationException( UNSUPPORTED_MSG ); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java b/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java index 8a397b60910..96f36650632 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.store.*; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.BytesRef; + import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -146,61 +147,67 @@ class ElevationComparatorSource extends FieldComparatorSource { public FieldComparator newComparator(final String fieldname, final int numHits, int sortPos, boolean reversed) throws IOException { return new FieldComparator() { - SortedDocValues idIndex; private final int[] values = new int[numHits]; int bottomVal; + @Override + public LeafFieldComparator getLeafComparator(LeafReaderContext context) + throws IOException { + final SortedDocValues idIndex = DocValues.getSorted(context.reader(), fieldname); + return new LeafFieldComparator() { + + @Override + public void setBottom(int slot) { + bottomVal = values[slot]; + } + + @Override + public int compareTop(int doc) { + throw new UnsupportedOperationException(); + } + + private int docVal(int doc) { + int ord = idIndex.getOrd(doc); + if (ord == -1) { + return 0; + } else { + final BytesRef term = idIndex.lookupOrd(ord); + Integer prio = priority.get(term); + return prio == null ? 0 : prio.intValue(); + } + } + + @Override + public int compareBottom(int doc) { + return docVal(doc) - bottomVal; + } + + @Override + public void copy(int slot, int doc) { + values[slot] = docVal(doc); + } + + @Override + public void setScorer(Scorer scorer) {} + }; + } + @Override public int compare(int slot1, int slot2) { return values[slot2] - values[slot1]; // values will be small enough that there is no overflow concern } - @Override - public void setBottom(int slot) { - bottomVal = values[slot]; - } - @Override public void setTopValue(Integer value) { throw new UnsupportedOperationException(); } - private int docVal(int doc) { - int ord = idIndex.getOrd(doc); - if (ord == -1) { - return 0; - } else { - final BytesRef term = idIndex.lookupOrd(ord); - Integer prio = priority.get(term); - return prio == null ? 0 : prio.intValue(); - } - } - - @Override - public int compareBottom(int doc) { - return docVal(doc) - bottomVal; - } - - @Override - public void copy(int slot, int doc) { - values[slot] = docVal(doc); - } - - @Override - public FieldComparator setNextReader(LeafReaderContext context) throws IOException { - idIndex = DocValues.getSorted(context.reader(), fieldname); - return this; - } - @Override public Integer value(int slot) { return Integer.valueOf(values[slot]); } - @Override - public int compareTop(int doc) { - throw new UnsupportedOperationException(); - } + }; } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSort.java b/lucene/core/src/test/org/apache/lucene/search/TestSort.java index fd8fe8b6366..3fb598c6e2b 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSort.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSort.java @@ -785,4 +785,63 @@ public class TestSort extends LuceneTestCase { ir.close(); dir.close(); } + + /** Tests sorting on multiple sort fields */ + public void testMultiSort() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(new SortedDocValuesField("value1", new BytesRef("foo"))); + doc.add(new NumericDocValuesField("value2", 0)); + doc.add(newStringField("value1", "foo", Field.Store.YES)); + doc.add(newStringField("value2", "0", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new SortedDocValuesField("value1", new BytesRef("bar"))); + doc.add(new NumericDocValuesField("value2", 1)); + doc.add(newStringField("value1", "bar", Field.Store.YES)); + doc.add(newStringField("value2", "1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new SortedDocValuesField("value1", new BytesRef("bar"))); + doc.add(new NumericDocValuesField("value2", 0)); + doc.add(newStringField("value1", "bar", Field.Store.YES)); + doc.add(newStringField("value2", "0", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(new SortedDocValuesField("value1", new BytesRef("foo"))); + doc.add(new NumericDocValuesField("value2", 1)); + doc.add(newStringField("value1", "foo", Field.Store.YES)); + doc.add(newStringField("value2", "1", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = newSearcher(ir); + Sort sort = new Sort( + new SortField("value1", SortField.Type.STRING), + new SortField("value2", SortField.Type.LONG)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(4, td.totalHits); + // 'bar' comes before 'foo' + assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value1")); + assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value1")); + assertEquals("foo", searcher.doc(td.scoreDocs[2].doc).get("value1")); + assertEquals("foo", searcher.doc(td.scoreDocs[3].doc).get("value1")); + // 0 comes before 1 + assertEquals("0", searcher.doc(td.scoreDocs[0].doc).get("value2")); + assertEquals("1", searcher.doc(td.scoreDocs[1].doc).get("value2")); + assertEquals("0", searcher.doc(td.scoreDocs[2].doc).get("value2")); + assertEquals("1", searcher.doc(td.scoreDocs[3].doc).get("value2")); + + // Now with overflow + td = searcher.search(new MatchAllDocsQuery(), 1, sort); + assertEquals(4, td.totalHits); + assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value1")); + assertEquals("0", searcher.doc(td.scoreDocs[0].doc).get("value2")); + + ir.close(); + dir.close(); + } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java index c3b9ec19c73..ab04ea79ef9 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java @@ -31,7 +31,6 @@ public class TestTopDocsCollector extends LuceneTestCase { private static final class MyTopsDocCollector extends TopDocsCollector { private int idx = 0; - private int base = 0; public MyTopsDocCollector(int size) { super(new HitQueue(size, false)); @@ -55,24 +54,26 @@ public class TestTopDocsCollector extends LuceneTestCase { } @Override - public void collect(int doc) { - ++totalHits; - pq.insertWithOverflow(new ScoreDoc(doc + base, scores[idx++])); - } + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + final int base = context.docBase; + return new LeafCollector() { + + @Override + public void collect(int doc) { + ++totalHits; + pq.insertWithOverflow(new ScoreDoc(doc + base, scores[idx++])); + } - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - base = context.docBase; - } - - @Override - public void setScorer(Scorer scorer) { - // Don't do anything. Assign scores in random - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; + @Override + public void setScorer(Scorer scorer) { + // Don't do anything. Assign scores in random + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + }; } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java index 0492c2dd59d..eea867dc354 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java @@ -171,14 +171,14 @@ public class TestTopFieldCollector extends LuceneTestCase { new boolean[] { true, true, true }, }; String[] actualTFCClasses = new String[] { - "OutOfOrderOneComparatorNonScoringCollector", - "OutOfOrderOneComparatorScoringMaxScoreCollector", - "OutOfOrderOneComparatorScoringNoMaxScoreCollector", - "OutOfOrderOneComparatorScoringMaxScoreCollector", - "OutOfOrderOneComparatorNonScoringCollector", - "OutOfOrderOneComparatorScoringMaxScoreCollector", - "OutOfOrderOneComparatorScoringNoMaxScoreCollector", - "OutOfOrderOneComparatorScoringMaxScoreCollector" + "OutOfOrderNonScoringCollector", + "OutOfOrderScoringMaxScoreCollector", + "OutOfOrderScoringNoMaxScoreCollector", + "OutOfOrderScoringMaxScoreCollector", + "OutOfOrderNonScoringCollector", + "OutOfOrderScoringMaxScoreCollector", + "OutOfOrderScoringNoMaxScoreCollector", + "OutOfOrderScoringMaxScoreCollector" }; BooleanQuery bq = new BooleanQuery(); @@ -220,14 +220,14 @@ public class TestTopFieldCollector extends LuceneTestCase { new boolean[] { true, true, true }, }; String[] actualTFCClasses = new String[] { - "OutOfOrderMultiComparatorNonScoringCollector", - "OutOfOrderMultiComparatorScoringMaxScoreCollector", - "OutOfOrderMultiComparatorScoringNoMaxScoreCollector", - "OutOfOrderMultiComparatorScoringMaxScoreCollector", - "OutOfOrderMultiComparatorNonScoringCollector", - "OutOfOrderMultiComparatorScoringMaxScoreCollector", - "OutOfOrderMultiComparatorScoringNoMaxScoreCollector", - "OutOfOrderMultiComparatorScoringMaxScoreCollector" + "OutOfOrderNonScoringCollector", + "OutOfOrderScoringMaxScoreCollector", + "OutOfOrderScoringNoMaxScoreCollector", + "OutOfOrderScoringMaxScoreCollector", + "OutOfOrderNonScoringCollector", + "OutOfOrderScoringMaxScoreCollector", + "OutOfOrderScoringNoMaxScoreCollector", + "OutOfOrderScoringMaxScoreCollector" }; BooleanQuery bq = new BooleanQuery(); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java index 409dd1a7e1b..563797bd571 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java @@ -34,10 +34,6 @@ public class TestTopScoreDocCollector extends LuceneTestCase { } boolean[] inOrder = new boolean[] { false, true }; - String[] actualTSDCClass = new String[] { - "OutOfOrderTopScoreDocCollector", - "InOrderTopScoreDocCollector" - }; BooleanQuery bq = new BooleanQuery(); // Add a Query with SHOULD, since bw.scorer() returns BooleanScorer2 @@ -50,7 +46,8 @@ public class TestTopScoreDocCollector extends LuceneTestCase { IndexSearcher searcher = newSearcher(reader); for (int i = 0; i < inOrder.length; i++) { TopDocsCollector tdc = TopScoreDocCollector.create(3, inOrder[i]); - assertEquals("org.apache.lucene.search.TopScoreDocCollector$" + actualTSDCClass[i], tdc.getClass().getName()); + LeafCollector leafCollector = tdc.getLeafCollector(reader.leaves().get(0)); + assertEquals(!inOrder[i], leafCollector.acceptsDocsOutOfOrder()); searcher.search(new MatchAllDocsQuery(), tdc); diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionComparator.java b/lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionComparator.java index ebeeca9af47..bf386bcee61 100644 --- a/lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionComparator.java +++ b/lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionComparator.java @@ -24,10 +24,11 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Scorer; /** A custom comparator for sorting documents by an expression */ -class ExpressionComparator extends FieldComparator { +class ExpressionComparator extends FieldComparator implements LeafFieldComparator { private final double[] values; private double bottom; private double topValue; @@ -44,7 +45,6 @@ class ExpressionComparator extends FieldComparator { // TODO: change FieldComparator.setScorer to throw IOException and remove this try-catch @Override public void setScorer(Scorer scorer) { - super.setScorer(scorer); // TODO: might be cleaner to lazy-init 'source' and set scorer after? assert readerContext != null; try { @@ -83,7 +83,7 @@ class ExpressionComparator extends FieldComparator { } @Override - public FieldComparator setNextReader(LeafReaderContext context) throws IOException { + public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { this.readerContext = context; return this; } diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java index c383042788b..01f71c9d958 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/AbstractFirstPassGroupingCollector.java @@ -37,6 +37,7 @@ abstract public class AbstractFirstPassGroupingCollector exten private final Sort groupSort; private final FieldComparator[] comparators; + private final LeafFieldComparator[] leafComparators; private final int[] reversed; private final int topNGroups; private final HashMap> groupMap; @@ -60,7 +61,6 @@ abstract public class AbstractFirstPassGroupingCollector exten * @param topNGroups How many top groups to keep. * @throws IOException If I/O related errors occur */ - @SuppressWarnings({"unchecked","rawtypes"}) public AbstractFirstPassGroupingCollector(Sort groupSort, int topNGroups) throws IOException { if (topNGroups < 1) { throw new IllegalArgumentException("topNGroups must be >= 1 (got " + topNGroups + ")"); @@ -74,6 +74,7 @@ abstract public class AbstractFirstPassGroupingCollector exten final SortField[] sortFields = groupSort.getSort(); comparators = new FieldComparator[sortFields.length]; + leafComparators = new LeafFieldComparator[sortFields.length]; compIDXEnd = comparators.length - 1; reversed = new int[sortFields.length]; for (int i = 0; i < sortFields.length; i++) { @@ -137,7 +138,7 @@ abstract public class AbstractFirstPassGroupingCollector exten @Override public void setScorer(Scorer scorer) throws IOException { - for (FieldComparator comparator : comparators) { + for (LeafFieldComparator comparator : leafComparators) { comparator.setScorer(scorer); } } @@ -157,7 +158,7 @@ abstract public class AbstractFirstPassGroupingCollector exten // wasted effort as we will most likely be updating an existing group. if (orderedGroups != null) { for (int compIDX = 0;; compIDX++) { - final int c = reversed[compIDX] * comparators[compIDX].compareBottom(doc); + final int c = reversed[compIDX] * leafComparators[compIDX].compareBottom(doc); if (c < 0) { // Definitely not competitive. So don't even bother to continue return; @@ -197,7 +198,7 @@ abstract public class AbstractFirstPassGroupingCollector exten sg.groupValue = copyDocGroupValue(groupValue, null); sg.comparatorSlot = groupMap.size(); sg.topDoc = docBase + doc; - for (FieldComparator fc : comparators) { + for (LeafFieldComparator fc : leafComparators) { fc.copy(sg.comparatorSlot, doc); } groupMap.put(sg.groupValue, sg); @@ -223,7 +224,7 @@ abstract public class AbstractFirstPassGroupingCollector exten bottomGroup.groupValue = copyDocGroupValue(groupValue, bottomGroup.groupValue); bottomGroup.topDoc = docBase + doc; - for (FieldComparator fc : comparators) { + for (LeafFieldComparator fc : leafComparators) { fc.copy(bottomGroup.comparatorSlot, doc); } @@ -232,7 +233,7 @@ abstract public class AbstractFirstPassGroupingCollector exten assert orderedGroups.size() == topNGroups; final int lastComparatorSlot = orderedGroups.last().comparatorSlot; - for (FieldComparator fc : comparators) { + for (LeafFieldComparator fc : leafComparators) { fc.setBottom(lastComparatorSlot); } @@ -241,17 +242,16 @@ abstract public class AbstractFirstPassGroupingCollector exten // Update existing group: for (int compIDX = 0;; compIDX++) { - final FieldComparator fc = comparators[compIDX]; - fc.copy(spareSlot, doc); + leafComparators[compIDX].copy(spareSlot, doc); - final int c = reversed[compIDX] * fc.compare(group.comparatorSlot, spareSlot); + final int c = reversed[compIDX] * comparators[compIDX].compare(group.comparatorSlot, spareSlot); if (c < 0) { // Definitely not competitive. return; } else if (c > 0) { // Definitely competitive; set remaining comparators: for (int compIDX2=compIDX+1; compIDX2 exten final CollectedSearchGroup newLast = orderedGroups.last(); // If we changed the value of the last group, or changed which group was last, then update bottom: if (group == newLast || prevLast != newLast) { - for (FieldComparator fc : comparators) { + for (LeafFieldComparator fc : leafComparators) { fc.setBottom(newLast.comparatorSlot); } } @@ -315,7 +315,7 @@ abstract public class AbstractFirstPassGroupingCollector exten orderedGroups.addAll(groupMap.values()); assert orderedGroups.size() > 0; - for (FieldComparator fc : comparators) { + for (LeafFieldComparator fc : leafComparators) { fc.setBottom(orderedGroups.last().comparatorSlot); } } @@ -329,7 +329,7 @@ abstract public class AbstractFirstPassGroupingCollector exten protected void doSetNextReader(LeafReaderContext readerContext) throws IOException { docBase = readerContext.docBase; for (int i=0; i exte @Override public void setScorer(Scorer scorer) throws IOException { for (SearchGroupDocs group : groupMap.values()) { - group.collector.setScorer(scorer); + group.leafCollector.setScorer(scorer); } } @@ -93,7 +93,7 @@ public abstract class AbstractSecondPassGroupingCollector exte SearchGroupDocs group = retrieveGroup(doc); if (group != null) { totalGroupedHitCount++; - group.collector.collect(doc); + group.leafCollector.collect(doc); } } @@ -110,7 +110,7 @@ public abstract class AbstractSecondPassGroupingCollector exte protected void doSetNextReader(LeafReaderContext readerContext) throws IOException { //System.out.println("SP.setNextReader"); for (SearchGroupDocs group : groupMap.values()) { - group.collector.getLeafCollector(readerContext); + group.leafCollector = group.collector.getLeafCollector(readerContext); } } @@ -151,6 +151,7 @@ public abstract class AbstractSecondPassGroupingCollector exte public final GROUP_VALUE_TYPE groupValue; public final TopDocsCollector collector; + public LeafCollector leafCollector; public SearchGroupDocs(GROUP_VALUE_TYPE groupValue, TopDocsCollector collector) { this.groupValue = groupValue; diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java index 4c0c6b54b64..509ab16edef 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java @@ -69,6 +69,7 @@ public class BlockGroupingCollector extends SimpleCollector { private final boolean needsScores; private final FieldComparator[] comparators; + private final LeafFieldComparator[] leafComparators; private final int[] reversed; private final int compIDXEnd; private int bottomSlot; @@ -202,7 +203,7 @@ public class BlockGroupingCollector extends SimpleCollector { bottomSlot = bottomGroup.comparatorSlot; //System.out.println(" set bottom=" + bottomSlot); for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottomSlot); + leafComparators[i].setBottom(bottomSlot); } //System.out.println(" QUEUE FULL"); } else { @@ -231,7 +232,7 @@ public class BlockGroupingCollector extends SimpleCollector { //System.out.println(" set bottom=" + bottomSlot); for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottomSlot); + leafComparators[i].setBottom(bottomSlot); } } } @@ -278,6 +279,7 @@ public class BlockGroupingCollector extends SimpleCollector { final SortField[] sortFields = groupSort.getSort(); comparators = new FieldComparator[sortFields.length]; + leafComparators = new LeafFieldComparator[sortFields.length]; compIDXEnd = comparators.length - 1; reversed = new int[sortFields.length]; for (int i = 0; i < sortFields.length; i++) { @@ -349,15 +351,15 @@ public class BlockGroupingCollector extends SimpleCollector { collector = TopFieldCollector.create(withinGroupSort, maxDocsPerGroup, fillSortFields, needsScores, needsScores, true); } - collector.setScorer(fakeScorer); - collector.getLeafCollector(og.readerContext); + LeafCollector leafCollector = collector.getLeafCollector(og.readerContext); + leafCollector.setScorer(fakeScorer); for(int docIDX=0;docIDX comparator : comparators) { + for (LeafFieldComparator comparator : leafComparators) { comparator.setScorer(scorer); } } @@ -443,7 +445,7 @@ public class BlockGroupingCollector extends SimpleCollector { assert !queueFull; //System.out.println(" init copy to bottomSlot=" + bottomSlot); - for (FieldComparator fc : comparators) { + for (LeafFieldComparator fc : leafComparators) { fc.copy(bottomSlot, doc); fc.setBottom(bottomSlot); } @@ -451,7 +453,7 @@ public class BlockGroupingCollector extends SimpleCollector { } else { // Compare to bottomSlot for (int compIDX = 0;; compIDX++) { - final int c = reversed[compIDX] * comparators[compIDX].compareBottom(doc); + final int c = reversed[compIDX] * leafComparators[compIDX].compareBottom(doc); if (c < 0) { // Definitely not competitive -- done return; @@ -468,7 +470,7 @@ public class BlockGroupingCollector extends SimpleCollector { //System.out.println(" best w/in group!"); - for (FieldComparator fc : comparators) { + for (LeafFieldComparator fc : leafComparators) { fc.copy(bottomSlot, doc); // Necessary because some comparators cache // details of bottom slot; this forces them to @@ -481,7 +483,7 @@ public class BlockGroupingCollector extends SimpleCollector { // We're not sure this group will make it into the // queue yet for (int compIDX = 0;; compIDX++) { - final int c = reversed[compIDX] * comparators[compIDX].compareBottom(doc); + final int c = reversed[compIDX] * leafComparators[compIDX].compareBottom(doc); if (c < 0) { // Definitely not competitive -- done //System.out.println(" doc doesn't compete w/ top groups"); @@ -498,7 +500,7 @@ public class BlockGroupingCollector extends SimpleCollector { } } groupCompetes = true; - for (FieldComparator fc : comparators) { + for (LeafFieldComparator fc : leafComparators) { fc.copy(bottomSlot, doc); // Necessary because some comparators cache // details of bottom slot; this forces them to @@ -528,7 +530,7 @@ public class BlockGroupingCollector extends SimpleCollector { currentReaderContext = readerContext; for (int i=0; i { public GroupComparator(Sort groupSort) throws IOException { final SortField[] sortFields = groupSort.getSort(); - comparators = new FieldComparator[sortFields.length]; + comparators = new FieldComparator[sortFields.length]; reversed = new int[sortFields.length]; for (int compIDX = 0; compIDX < sortFields.length; compIDX++) { final SortField sortField = sortFields[compIDX]; diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionAllGroupHeadsCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionAllGroupHeadsCollector.java index 771660d7978..e30a55186ae 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionAllGroupHeadsCollector.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionAllGroupHeadsCollector.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; @@ -94,7 +95,7 @@ public class FunctionAllGroupHeadsCollector extends AbstractAllGroupHeadsCollect public void setScorer(Scorer scorer) throws IOException { this.scorer = scorer; for (GroupHead groupHead : groups.values()) { - for (FieldComparator comparator : groupHead.comparators) { + for (LeafFieldComparator comparator : groupHead.leafComparators) { comparator.setScorer(scorer); } } @@ -109,7 +110,7 @@ public class FunctionAllGroupHeadsCollector extends AbstractAllGroupHeadsCollect for (GroupHead groupHead : groups.values()) { for (int i = 0; i < groupHead.comparators.length; i++) { - groupHead.comparators[i] = groupHead.comparators[i].setNextReader(context); + groupHead.leafComparators[i] = groupHead.comparators[i].getLeafComparator(context); } } } @@ -120,28 +121,31 @@ public class FunctionAllGroupHeadsCollector extends AbstractAllGroupHeadsCollect public class GroupHead extends AbstractAllGroupHeadsCollector.GroupHead { final FieldComparator[] comparators; + final LeafFieldComparator[] leafComparators; @SuppressWarnings({"unchecked","rawtypes"}) private GroupHead(MutableValue groupValue, Sort sort, int doc) throws IOException { super(groupValue, doc + readerContext.docBase); final SortField[] sortFields = sort.getSort(); comparators = new FieldComparator[sortFields.length]; + leafComparators = new LeafFieldComparator[sortFields.length]; for (int i = 0; i < sortFields.length; i++) { - comparators[i] = sortFields[i].getComparator(1, i).setNextReader(readerContext); - comparators[i].setScorer(scorer); - comparators[i].copy(0, doc); - comparators[i].setBottom(0); + comparators[i] = sortFields[i].getComparator(1, i); + leafComparators[i] = comparators[i].getLeafComparator(readerContext); + leafComparators[i].setScorer(scorer); + leafComparators[i].copy(0, doc); + leafComparators[i].setBottom(0); } } @Override public int compare(int compIDX, int doc) throws IOException { - return comparators[compIDX].compareBottom(doc); + return leafComparators[compIDX].compareBottom(doc); } @Override public void updateDocHead(int doc) throws IOException { - for (FieldComparator comparator : comparators) { + for (LeafFieldComparator comparator : leafComparators) { comparator.copy(0, doc); comparator.setBottom(0); } diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java index df0c0bb3c5f..61c981ad844 100644 --- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java +++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; @@ -164,7 +165,7 @@ public abstract class TermAllGroupHeadsCollector comparator : groupHead.comparators) { + for (LeafFieldComparator comparator : groupHead.leafComparators) { comparator.setScorer(scorer); } } @@ -181,28 +182,31 @@ public abstract class TermAllGroupHeadsCollector { - final FieldComparator[] comparators; + final FieldComparator[] comparators; + final LeafFieldComparator[] leafComparators; private GroupHead(BytesRef groupValue, Sort sort, int doc) throws IOException { super(groupValue, doc + readerContext.docBase); final SortField[] sortFields = sort.getSort(); comparators = new FieldComparator[sortFields.length]; + leafComparators = new LeafFieldComparator[sortFields.length]; for (int i = 0; i < sortFields.length; i++) { - comparators[i] = sortFields[i].getComparator(1, i).setNextReader(readerContext); - comparators[i].setScorer(scorer); - comparators[i].copy(0, doc); - comparators[i].setBottom(0); + comparators[i] = sortFields[i].getComparator(1, i); + leafComparators[i] = comparators[i].getLeafComparator(readerContext); + leafComparators[i].setScorer(scorer); + leafComparators[i].copy(0, doc); + leafComparators[i].setBottom(0); } } @Override public int compare(int compIDX, int doc) throws IOException { - return comparators[compIDX].compareBottom(doc); + return leafComparators[compIDX].compareBottom(doc); } @Override public void updateDocHead(int doc) throws IOException { - for (FieldComparator comparator : comparators) { + for (LeafFieldComparator comparator : leafComparators) { comparator.copy(0, doc); comparator.setBottom(0); } diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java index 6fe1fc492a0..fb5175f8846 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java @@ -74,7 +74,7 @@ import java.util.*; * * @lucene.experimental */ -public class ToParentBlockJoinCollector extends SimpleCollector { +public class ToParentBlockJoinCollector implements Collector { private final Sort sort; @@ -83,16 +83,11 @@ public class ToParentBlockJoinCollector extends SimpleCollector { private final Map joinQueryID = new HashMap<>(); private final int numParentHits; private final FieldValueHitQueue queue; - private final FieldComparator[] comparators; - private final int[] reverseMul; - private final int compEnd; + private final FieldComparator[] comparators; private final boolean trackMaxScore; private final boolean trackScores; - private int docBase; private ToParentBlockJoinQuery.BlockJoinScorer[] joinScorers = new ToParentBlockJoinQuery.BlockJoinScorer[0]; - private LeafReaderContext currentReaderContext; - private Scorer scorer; private boolean queueFull; private OneGroup bottom; @@ -116,8 +111,6 @@ public class ToParentBlockJoinCollector extends SimpleCollector { this.numParentHits = numParentHits; queue = FieldValueHitQueue.create(sort.getSort(), numParentHits); comparators = queue.getComparators(); - reverseMul = queue.getReverseMul(); - compEnd = comparators.length - 1; } private static final class OneGroup extends FieldValueHitQueue.Entry { @@ -143,143 +136,172 @@ public class ToParentBlockJoinCollector extends SimpleCollector { } @Override - public void collect(int parentDoc) throws IOException { - //System.out.println("\nC parentDoc=" + parentDoc); - totalHitCount++; + public LeafCollector getLeafCollector(final LeafReaderContext context) + throws IOException { + final LeafFieldComparator[] comparators = queue.getComparators(context); + final int[] reverseMul = queue.getReverseMul(); + final int docBase = context.docBase; + return new LeafCollector() { - float score = Float.NaN; + private Scorer scorer; - if (trackMaxScore) { - score = scorer.score(); - maxScore = Math.max(maxScore, score); - } - - // TODO: we could sweep all joinScorers here and - // aggregate total child hit count, so we can fill this - // in getTopGroups (we wire it to 0 now) - - if (queueFull) { - //System.out.println(" queueFull"); - // Fastmatch: return if this hit is not competitive - for (int i = 0;; i++) { - final int c = reverseMul[i] * comparators[i].compareBottom(parentDoc); - if (c < 0) { - // Definitely not competitive. - //System.out.println(" skip"); - return; - } else if (c > 0) { - // Definitely competitive. - break; - } else if (i == compEnd) { - // Here c=0. If we're at the last comparator, this doc is not - // competitive, since docs are visited in doc Id order, which means - // this doc cannot compete with any other document in the queue. - //System.out.println(" skip"); - return; + @Override + public void setScorer(Scorer scorer) throws IOException { + //System.out.println("C.setScorer scorer=" + scorer); + // Since we invoke .score(), and the comparators likely + // do as well, cache it so it's only "really" computed + // once: + if (scorer instanceof ScoreCachingWrappingScorer == false) { + scorer = new ScoreCachingWrappingScorer(scorer); } - } - - //System.out.println(" competes! doc=" + (docBase + parentDoc)); - - // This hit is competitive - replace bottom element in queue & adjustTop - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(bottom.slot, parentDoc); - } - if (!trackMaxScore && trackScores) { - score = scorer.score(); - } - bottom.doc = docBase + parentDoc; - bottom.readerContext = currentReaderContext; - bottom.score = score; - copyGroups(bottom); - bottom = queue.updateTop(); - - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); - } - } else { - // Startup transient: queue is not yet full: - final int comparatorSlot = totalHitCount - 1; - - // Copy hit into queue - for (int i = 0; i < comparators.length; i++) { - comparators[i].copy(comparatorSlot, parentDoc); - } - //System.out.println(" startup: new OG doc=" + (docBase+parentDoc)); - if (!trackMaxScore && trackScores) { - score = scorer.score(); - } - final OneGroup og = new OneGroup(comparatorSlot, docBase+parentDoc, score, joinScorers.length, trackScores); - og.readerContext = currentReaderContext; - copyGroups(og); - bottom = queue.add(og); - queueFull = totalHitCount == numParentHits; - if (queueFull) { - // End of startup transient: queue just filled up: - for (int i = 0; i < comparators.length; i++) { - comparators[i].setBottom(bottom.slot); + this.scorer = scorer; + for (LeafFieldComparator comparator : comparators) { + comparator.setScorer(scorer); } - } - } - } + Arrays.fill(joinScorers, null); - // Pulls out child doc and scores for all join queries: - private void copyGroups(OneGroup og) { - // While rare, it's possible top arrays could be too - // short if join query had null scorer on first - // segment(s) but then became non-null on later segments - final int numSubScorers = joinScorers.length; - if (og.docs.length < numSubScorers) { - // While rare, this could happen if join query had - // null scorer on first segment(s) but then became - // non-null on later segments - og.docs = ArrayUtil.grow(og.docs); - } - if (og.counts.length < numSubScorers) { - og.counts = ArrayUtil.grow(og.counts); - } - if (trackScores && og.scores.length < numSubScorers) { - og.scores = ArrayUtil.grow(og.scores); - } - - //System.out.println("\ncopyGroups parentDoc=" + og.doc); - for(int scorerIDX = 0;scorerIDX < numSubScorers;scorerIDX++) { - final ToParentBlockJoinQuery.BlockJoinScorer joinScorer = joinScorers[scorerIDX]; - //System.out.println(" scorer=" + joinScorer); - if (joinScorer != null && docBase + joinScorer.getParentDoc() == og.doc) { - og.counts[scorerIDX] = joinScorer.getChildCount(); - //System.out.println(" count=" + og.counts[scorerIDX]); - og.docs[scorerIDX] = joinScorer.swapChildDocs(og.docs[scorerIDX]); - assert og.docs[scorerIDX].length >= og.counts[scorerIDX]: "length=" + og.docs[scorerIDX].length + " vs count=" + og.counts[scorerIDX]; - //System.out.println(" len=" + og.docs[scorerIDX].length); - /* - for(int idx=0;idx= og.counts[scorerIDX]: "length=" + og.scores[scorerIDX].length + " vs count=" + og.counts[scorerIDX]; } - } else { - og.counts[scorerIDX] = 0; } - } - } + + @Override + public void collect(int parentDoc) throws IOException { + //System.out.println("\nC parentDoc=" + parentDoc); + totalHitCount++; - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - currentReaderContext = context; - docBase = context.docBase; - for (int compIDX = 0; compIDX < comparators.length; compIDX++) { - queue.setComparator(compIDX, comparators[compIDX].setNextReader(context)); - } - } + float score = Float.NaN; - @Override - public boolean acceptsDocsOutOfOrder() { - return false; + if (trackMaxScore) { + score = scorer.score(); + maxScore = Math.max(maxScore, score); + } + + // TODO: we could sweep all joinScorers here and + // aggregate total child hit count, so we can fill this + // in getTopGroups (we wire it to 0 now) + + if (queueFull) { + //System.out.println(" queueFull"); + // Fastmatch: return if this hit is not competitive + int c = 0; + for (int i = 0; i < comparators.length; ++i) { + c = reverseMul[i] * comparators[i].compareBottom(parentDoc); + if (c != 0) { + break; + } + } + if (c <= 0) { // in case of equality, this hit is not competitive as docs are visited in order + // Definitely not competitive. + //System.out.println(" skip"); + return; + } + + //System.out.println(" competes! doc=" + (docBase + parentDoc)); + + // This hit is competitive - replace bottom element in queue & adjustTop + for (LeafFieldComparator comparator : comparators) { + comparator.copy(bottom.slot, parentDoc); + } + if (!trackMaxScore && trackScores) { + score = scorer.score(); + } + bottom.doc = docBase + parentDoc; + bottom.readerContext = context; + bottom.score = score; + copyGroups(bottom); + bottom = queue.updateTop(); + + for (LeafFieldComparator comparator : comparators) { + comparator.setBottom(bottom.slot); + } + } else { + // Startup transient: queue is not yet full: + final int comparatorSlot = totalHitCount - 1; + + // Copy hit into queue + for (LeafFieldComparator comparator : comparators) { + comparator.copy(comparatorSlot, parentDoc); + } + //System.out.println(" startup: new OG doc=" + (docBase+parentDoc)); + if (!trackMaxScore && trackScores) { + score = scorer.score(); + } + final OneGroup og = new OneGroup(comparatorSlot, docBase+parentDoc, score, joinScorers.length, trackScores); + og.readerContext = context; + copyGroups(og); + bottom = queue.add(og); + queueFull = totalHitCount == numParentHits; + if (queueFull) { + // End of startup transient: queue just filled up: + for (LeafFieldComparator comparator : comparators) { + comparator.setBottom(bottom.slot); + } + } + } + } + + // Pulls out child doc and scores for all join queries: + private void copyGroups(OneGroup og) { + // While rare, it's possible top arrays could be too + // short if join query had null scorer on first + // segment(s) but then became non-null on later segments + final int numSubScorers = joinScorers.length; + if (og.docs.length < numSubScorers) { + // While rare, this could happen if join query had + // null scorer on first segment(s) but then became + // non-null on later segments + og.docs = ArrayUtil.grow(og.docs); + } + if (og.counts.length < numSubScorers) { + og.counts = ArrayUtil.grow(og.counts); + } + if (trackScores && og.scores.length < numSubScorers) { + og.scores = ArrayUtil.grow(og.scores); + } + + //System.out.println("\ncopyGroups parentDoc=" + og.doc); + for(int scorerIDX = 0;scorerIDX < numSubScorers;scorerIDX++) { + final ToParentBlockJoinQuery.BlockJoinScorer joinScorer = joinScorers[scorerIDX]; + //System.out.println(" scorer=" + joinScorer); + if (joinScorer != null && docBase + joinScorer.getParentDoc() == og.doc) { + og.counts[scorerIDX] = joinScorer.getChildCount(); + //System.out.println(" count=" + og.counts[scorerIDX]); + og.docs[scorerIDX] = joinScorer.swapChildDocs(og.docs[scorerIDX]); + assert og.docs[scorerIDX].length >= og.counts[scorerIDX]: "length=" + og.docs[scorerIDX].length + " vs count=" + og.counts[scorerIDX]; + //System.out.println(" len=" + og.docs[scorerIDX].length); + /* + for(int idx=0;idx= og.counts[scorerIDX]: "length=" + og.scores[scorerIDX].length + " vs count=" + og.counts[scorerIDX]; + } + } else { + og.counts[scorerIDX] = 0; + } + } + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return false; + } + }; } private void enroll(ToParentBlockJoinQuery query, ToParentBlockJoinQuery.BlockJoinScorer scorer) { @@ -296,34 +318,6 @@ public class ToParentBlockJoinCollector extends SimpleCollector { joinScorers[slot] = scorer; } } - - @Override - public void setScorer(Scorer scorer) { - //System.out.println("C.setScorer scorer=" + scorer); - // Since we invoke .score(), and the comparators likely - // do as well, cache it so it's only "really" computed - // once: - this.scorer = new ScoreCachingWrappingScorer(scorer); - for (int compIDX = 0; compIDX < comparators.length; compIDX++) { - comparators[compIDX].setScorer(this.scorer); - } - Arrays.fill(joinScorers, null); - - Queue queue = new LinkedList<>(); - //System.out.println("\nqueue: add top scorer=" + scorer); - queue.add(scorer); - while ((scorer = queue.poll()) != null) { - //System.out.println(" poll: " + scorer + "; " + scorer.getWeight().getQuery()); - if (scorer instanceof ToParentBlockJoinQuery.BlockJoinScorer) { - enroll((ToParentBlockJoinQuery) scorer.getWeight().getQuery(), (ToParentBlockJoinQuery.BlockJoinScorer) scorer); - } - - for (ChildScorer sub : scorer.getChildren()) { - //System.out.println(" add sub: " + sub.child + "; " + sub.child.getWeight().getQuery()); - queue.add(sub.child); - } - } - } private OneGroup[] sortedGroups; @@ -420,8 +414,8 @@ public class ToParentBlockJoinCollector extends SimpleCollector { collector = TopFieldCollector.create(withinGroupSort, numDocsInGroup, fillSortFields, trackScores, trackMaxScore, true); } - collector.setScorer(fakeScorer); - collector.getLeafCollector(og.readerContext); + LeafCollector leafCollector = collector.getLeafCollector(og.readerContext); + leafCollector.setScorer(fakeScorer); for(int docIDX=0;docIDX { +public abstract class ToParentBlockJoinFieldComparator extends SimpleFieldComparator implements LeafFieldComparator { // repeat LeafFieldComparator for javadocs private final BitDocIdSetFilter parentFilter; private final BitDocIdSetFilter childFilter; final int spareSlot; FieldComparator wrappedComparator; + LeafFieldComparator wrappedLeafComparator; BitSet parentDocuments; BitSet childDocuments; @@ -55,7 +58,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator setNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { BitDocIdSet children = childFilter.getDocIdSet(context); if (children == null) { childDocuments = null; @@ -77,8 +80,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator 0) { return cmp; } @@ -129,7 +131,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) { return cmp; } - int cmp1 = wrappedComparator.compareBottom(childDoc); + int cmp1 = wrappedLeafComparator.compareBottom(childDoc); if (cmp1 > 0) { return cmp1; } else { @@ -152,23 +154,22 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) { return; } - wrappedComparator.copy(spareSlot, childDoc); - wrappedComparator.copy(slot, childDoc); + wrappedLeafComparator.copy(spareSlot, childDoc); + wrappedLeafComparator.copy(slot, childDoc); while (true) { childDoc = childDocuments.nextSetBit(childDoc + 1); if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) { return; } - wrappedComparator.copy(spareSlot, childDoc); + wrappedLeafComparator.copy(spareSlot, childDoc); if (wrappedComparator.compare(spareSlot, slot) < 0) { - wrappedComparator.copy(slot, childDoc); + wrappedLeafComparator.copy(slot, childDoc); } } } @Override - @SuppressWarnings("unchecked") public int compareTop(int parentDoc) throws IOException { if (parentDoc == 0 || parentDocuments == null || childDocuments == null) { return 0; @@ -182,7 +183,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator 0) { return cmp; } @@ -192,7 +193,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) { return cmp; } - int cmp1 = wrappedComparator.compareTop(childDoc); + int cmp1 = wrappedLeafComparator.compareTop(childDoc); if (cmp1 > 0) { return cmp1; } else { @@ -209,12 +210,12 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) { return cmp; } - int cmp1 = wrappedComparator.compareBottom(childDoc); + int cmp1 = wrappedLeafComparator.compareBottom(childDoc); if (cmp1 < 0) { return cmp1; } else { @@ -268,23 +269,22 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) { return; } - wrappedComparator.copy(spareSlot, childDoc); - wrappedComparator.copy(slot, childDoc); + wrappedLeafComparator.copy(spareSlot, childDoc); + wrappedLeafComparator.copy(slot, childDoc); while (true) { childDoc = childDocuments.nextSetBit(childDoc + 1); if (childDoc >= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) { return; } - wrappedComparator.copy(spareSlot, childDoc); + wrappedLeafComparator.copy(spareSlot, childDoc); if (wrappedComparator.compare(spareSlot, slot) > 0) { - wrappedComparator.copy(slot, childDoc); + wrappedLeafComparator.copy(slot, childDoc); } } } @Override - @SuppressWarnings("unchecked") public int compareTop(int parentDoc) throws IOException { if (parentDoc == 0 || parentDocuments == null || childDocuments == null) { return 0; @@ -296,7 +296,7 @@ public abstract class ToParentBlockJoinFieldComparator extends FieldComparator= parentDoc || childDoc == DocIdSetIterator.NO_MORE_DOCS) { return cmp; } - int cmp1 = wrappedComparator.compareTop(childDoc); + int cmp1 = wrappedLeafComparator.compareTop(childDoc); if (cmp1 < 0) { return cmp1; } else { diff --git a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java index b4730a79756..4d6a6ed38db 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinSortField.java @@ -18,6 +18,7 @@ package org.apache.lucene.search.join; */ import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.SortField; import java.io.IOException; diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java index 795801c91d9..b94d430793c 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java @@ -52,9 +52,12 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Collector; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.FilterLeafCollector; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; @@ -482,30 +485,25 @@ public class TestJoinUtil extends LuceneTestCase { // Need to know all documents that have matches. TopDocs doesn't give me that and then I'd be also testing TopDocsCollector... final BitSet actualResult = new FixedBitSet(indexSearcher.getIndexReader().maxDoc()); final TopScoreDocCollector topScoreDocCollector = TopScoreDocCollector.create(10, false); - indexSearcher.search(joinQuery, new SimpleCollector() { - - int docBase; + indexSearcher.search(joinQuery, new Collector() { @Override - public void collect(int doc) throws IOException { - actualResult.set(doc + docBase); - topScoreDocCollector.collect(doc); - } + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + final int docBase = context.docBase; + final LeafCollector in = topScoreDocCollector.getLeafCollector(context); + return new FilterLeafCollector(in) { - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - docBase = context.docBase; - topScoreDocCollector.getLeafCollector(context); - } - - @Override - public void setScorer(Scorer scorer) throws IOException { - topScoreDocCollector.setScorer(scorer); - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return scoreDocsInOrder; + @Override + public void collect(int doc) throws IOException { + super.collect(doc); + actualResult.set(doc + docBase); + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return scoreDocsInOrder; + } + }; } }); // Asserting bit set... diff --git a/lucene/misc/src/java/org/apache/lucene/index/Sorter.java b/lucene/misc/src/java/org/apache/lucene/index/Sorter.java index 6ae99b0bf7f..22912bc67c9 100644 --- a/lucene/misc/src/java/org/apache/lucene/index/Sorter.java +++ b/lucene/misc/src/java/org/apache/lucene/index/Sorter.java @@ -20,9 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Comparator; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; @@ -214,12 +212,11 @@ final class Sorter { DocMap sort(LeafReader reader) throws IOException { SortField fields[] = sort.getSort(); final int reverseMul[] = new int[fields.length]; - final FieldComparator comparators[] = new FieldComparator[fields.length]; + final LeafFieldComparator comparators[] = new LeafFieldComparator[fields.length]; for (int i = 0; i < fields.length; i++) { reverseMul[i] = fields[i].getReverse() ? -1 : 1; - comparators[i] = fields[i].getComparator(1, i); - comparators[i].setNextReader(reader.getContext()); + comparators[i] = fields[i].getComparator(1, i).getLeafComparator(reader.getContext()); comparators[i].setScorer(FAKESCORER); } final DocComparator comparator = new DocComparator() { diff --git a/lucene/misc/src/java/org/apache/lucene/search/BlockJoinComparatorSource.java b/lucene/misc/src/java/org/apache/lucene/search/BlockJoinComparatorSource.java index 21143a2605a..bd7d5e4a919 100644 --- a/lucene/misc/src/java/org/apache/lucene/search/BlockJoinComparatorSource.java +++ b/lucene/misc/src/java/org/apache/lucene/search/BlockJoinComparatorSource.java @@ -21,18 +21,8 @@ import java.io.IOException; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortingMergePolicy; -import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.FieldComparatorSource; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.IndexSearcher; // javadocs -import org.apache.lucene.search.Query; // javadocs -import org.apache.lucene.search.ScoreDoc; // javadocs -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; import org.apache.lucene.util.BitDocIdSet; -import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.BitSet; /** * Helper class to sort readers that contain blocks of documents. @@ -51,22 +41,22 @@ public class BlockJoinComparatorSource extends FieldComparatorSource { final Filter parentsFilter; final Sort parentSort; final Sort childSort; - - /** + + /** * Create a new BlockJoinComparatorSource, sorting only blocks of documents * with {@code parentSort} and not reordering children with a block. - * + * * @param parentsFilter Filter identifying parent documents * @param parentSort Sort for parent documents */ public BlockJoinComparatorSource(Filter parentsFilter, Sort parentSort) { this(parentsFilter, parentSort, new Sort(SortField.FIELD_DOC)); } - - /** + + /** * Create a new BlockJoinComparatorSource, specifying the sort order for both * blocks of documents and children within a block. - * + * * @param parentsFilter Filter identifying parent documents * @param parentSort Sort for parent documents * @param childSort Sort for child documents in the same block @@ -82,7 +72,7 @@ public class BlockJoinComparatorSource extends FieldComparatorSource { // we keep parallel slots: the parent ids and the child ids final int parentSlots[] = new int[numHits]; final int childSlots[] = new int[numHits]; - + SortField parentFields[] = parentSort.getSort(); final int parentReverseMul[] = new int[parentFields.length]; final FieldComparator parentComparators[] = new FieldComparator[parentFields.length]; @@ -90,7 +80,7 @@ public class BlockJoinComparatorSource extends FieldComparatorSource { parentReverseMul[i] = parentFields[i].getReverse() ? -1 : 1; parentComparators[i] = parentFields[i].getComparator(1, i); } - + SortField childFields[] = childSort.getSort(); final int childReverseMul[] = new int[childFields.length]; final FieldComparator childComparators[] = new FieldComparator[childFields.length]; @@ -98,14 +88,16 @@ public class BlockJoinComparatorSource extends FieldComparatorSource { childReverseMul[i] = childFields[i].getReverse() ? -1 : 1; childComparators[i] = childFields[i].getComparator(1, i); } - + // NOTE: we could return parent ID as value but really our sort "value" is more complex... // So we throw UOE for now. At the moment you really should only use this at indexing time. return new FieldComparator() { int bottomParent; int bottomChild; - FixedBitSet parentBits; - + BitSet parentBits; + LeafFieldComparator[] parentLeafComparators; + LeafFieldComparator[] childLeafComparators; + @Override public int compare(int slot1, int slot2) { try { @@ -115,12 +107,6 @@ public class BlockJoinComparatorSource extends FieldComparatorSource { } } - @Override - public void setBottom(int slot) { - bottomParent = parentSlots[slot]; - bottomChild = childSlots[slot]; - } - @Override public void setTopValue(Integer value) { // we dont have enough information (the docid is needed) @@ -128,39 +114,63 @@ public class BlockJoinComparatorSource extends FieldComparatorSource { } @Override - public int compareBottom(int doc) throws IOException { - return compare(bottomChild, bottomParent, doc, parent(doc)); - } - - @Override - public int compareTop(int doc) throws IOException { - // we dont have enough information (the docid is needed) - throw new UnsupportedOperationException("this comparator cannot be used with deep paging"); - } - - @Override - public void copy(int slot, int doc) throws IOException { - childSlots[slot] = doc; - parentSlots[slot] = parent(doc); - } - - @Override - public FieldComparator setNextReader(LeafReaderContext context) throws IOException { + public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { + if (parentBits != null) { + throw new IllegalStateException("This comparator can only be used on a single segment"); + } final DocIdSet parents = parentsFilter.getDocIdSet(context, null); if (parents == null) { throw new IllegalStateException("LeafReader " + context.reader() + " contains no parents!"); } - if (!(parents instanceof BitDocIdSet)) { - throw new IllegalStateException("parentFilter must return FixedBitSet; got " + parents); + if (parents instanceof BitDocIdSet == false) { + throw new IllegalStateException("parentFilter must return BitSet; got " + parents); } - parentBits = (FixedBitSet) parents.bits(); + parentBits = (BitSet) parents.bits(); + parentLeafComparators = new LeafFieldComparator[parentComparators.length]; for (int i = 0; i < parentComparators.length; i++) { - parentComparators[i] = parentComparators[i].setNextReader(context); + parentLeafComparators[i] = parentComparators[i].getLeafComparator(context); } + childLeafComparators = new LeafFieldComparator[childComparators.length]; for (int i = 0; i < childComparators.length; i++) { - childComparators[i] = childComparators[i].setNextReader(context); + childLeafComparators[i] = childComparators[i].getLeafComparator(context); } - return this; + + return new LeafFieldComparator() { + + @Override + public int compareBottom(int doc) throws IOException { + return compare(bottomChild, bottomParent, doc, parent(doc)); + } + + @Override + public int compareTop(int doc) throws IOException { + // we dont have enough information (the docid is needed) + throw new UnsupportedOperationException("this comparator cannot be used with deep paging"); + } + + @Override + public void copy(int slot, int doc) throws IOException { + childSlots[slot] = doc; + parentSlots[slot] = parent(doc); + } + + @Override + public void setBottom(int slot) { + bottomParent = parentSlots[slot]; + bottomChild = childSlots[slot]; + } + + @Override + public void setScorer(Scorer scorer) { + for (LeafFieldComparator comp : parentLeafComparators) { + comp.setScorer(scorer); + } + for (LeafFieldComparator comp : childLeafComparators) { + comp.setScorer(scorer); + } + } + + }; } @Override @@ -168,32 +178,21 @@ public class BlockJoinComparatorSource extends FieldComparatorSource { // really our sort "value" is more complex... throw new UnsupportedOperationException("filling sort field values is not yet supported"); } - - @Override - public void setScorer(Scorer scorer) { - super.setScorer(scorer); - for (FieldComparator comp : parentComparators) { - comp.setScorer(scorer); - } - for (FieldComparator comp : childComparators) { - comp.setScorer(scorer); - } - } int parent(int doc) { return parentBits.nextSetBit(doc); } - + int compare(int docID1, int parent1, int docID2, int parent2) throws IOException { if (parent1 == parent2) { // both are in the same block if (docID1 == parent1 || docID2 == parent2) { // keep parents at the end of blocks return docID1 - docID2; } else { - return compare(docID1, docID2, childComparators, childReverseMul); + return compare(docID1, docID2, childLeafComparators, childReverseMul); } } else { - int cmp = compare(parent1, parent2, parentComparators, parentReverseMul); + int cmp = compare(parent1, parent2, parentLeafComparators, parentReverseMul); if (cmp == 0) { return parent1 - parent2; } else { @@ -201,8 +200,8 @@ public class BlockJoinComparatorSource extends FieldComparatorSource { } } } - - int compare(int docID1, int docID2, FieldComparator comparators[], int reverseMul[]) throws IOException { + + int compare(int docID1, int docID2, LeafFieldComparator comparators[], int reverseMul[]) throws IOException { for (int i = 0; i < comparators.length; i++) { // TODO: would be better if copy() didnt cause a term lookup in TermOrdVal & co, // the segments are always the same here... @@ -217,7 +216,7 @@ public class BlockJoinComparatorSource extends FieldComparatorSource { } }; } - + @Override public String toString() { return "blockJoin(parentSort=" + parentSort + ",childSort=" + childSort + ")"; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java index 3cf9081d235..14cdc1d7c6a 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java @@ -17,16 +17,17 @@ package org.apache.lucene.queries.function; * limitations under the License. */ +import java.io.IOException; +import java.util.IdentityHashMap; +import java.util.Map; + import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldComparatorSource; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.SimpleFieldComparator; import org.apache.lucene.search.SortField; -import java.io.IOException; -import java.util.IdentityHashMap; -import java.util.Map; - /** * Instantiates {@link FunctionValues} for a particular reader. *
@@ -126,7 +127,7 @@ public abstract class ValueSource { * off of the {@link FunctionValues} for a ValueSource * instead of the normal Lucene FieldComparator that works off of a FieldCache. */ - class ValueSourceComparator extends FieldComparator { + class ValueSourceComparator extends SimpleFieldComparator { private final double[] values; private FunctionValues docVals; private double bottom; @@ -154,9 +155,8 @@ public abstract class ValueSource { } @Override - public FieldComparator setNextReader(LeafReaderContext context) throws IOException { + public void doSetNextReader(LeafReaderContext context) throws IOException { docVals = getValues(fcontext, context); - return this; } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java index 311e128b246..d96017c844b 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java +++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java @@ -23,7 +23,7 @@ import java.text.Collator; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; -import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.SimpleFieldComparator; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -36,7 +36,7 @@ import org.apache.lucene.util.BytesRef; * This class will be removed in Lucene 5.0 */ @Deprecated -public final class SlowCollatedStringComparator extends FieldComparator { +public final class SlowCollatedStringComparator extends SimpleFieldComparator { private final String[] values; private BinaryDocValues currentDocTerms; @@ -93,10 +93,9 @@ public final class SlowCollatedStringComparator extends FieldComparator } @Override - public FieldComparator setNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { currentDocTerms = DocValues.getBinary(context.reader(), field); docsWithField = DocValues.getDocsWithField(context.reader(), field); - return this; } @Override diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 3fef56cec28..a3520ea57a6 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Scorer; @@ -598,7 +599,8 @@ public class QueryComponent extends SearchComponent // :TODO: would be simpler to always serialize every position of SortField[] if (type==SortField.Type.SCORE || type==SortField.Type.DOC) continue; - FieldComparator comparator = null; + FieldComparator comparator = null; + LeafFieldComparator leafComparator = null; Object[] vals = new Object[nDocs]; int lastIdx = -1; @@ -621,12 +623,12 @@ public class QueryComponent extends SearchComponent if (comparator == null) { comparator = sortField.getComparator(1,0); - comparator = comparator.setNextReader(currentLeaf); + leafComparator = comparator.getLeafComparator(currentLeaf); } doc -= currentLeaf.docBase; // adjust for what segment this is in - comparator.setScorer(new FakeScorer(doc, score)); - comparator.copy(0, doc); + leafComparator.setScorer(new FakeScorer(doc, score)); + leafComparator.copy(0, doc); Object val = comparator.value(0); if (null != ft) val = ft.marshalSortValue(val); vals[position] = val; diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java index 235c3e08333..3c5b24f1dba 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java @@ -36,6 +36,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldComparatorSource; import org.apache.lucene.search.Query; +import org.apache.lucene.search.SimpleFieldComparator; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; @@ -631,7 +632,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore @Override public FieldComparator newComparator(String fieldname, final int numHits, int sortPos, boolean reversed) throws IOException { - return new FieldComparator() { + return new SimpleFieldComparator() { private final int[] values = new int[numHits]; private int bottomVal; private int topVal; @@ -677,13 +678,13 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore } @Override - public FieldComparator setNextReader(LeafReaderContext context) throws IOException { + protected void doSetNextReader(LeafReaderContext context) throws IOException { //convert the ids to Lucene doc ids, the ordSet and termValues needs to be the same size as the number of elevation docs we have ordSet.clear(); Fields fields = context.reader().fields(); - if (fields == null) return this; + if (fields == null) return; Terms terms = fields.terms(idField); - if (terms == null) return this; + if (terms == null) return; termsEnum = terms.iterator(termsEnum); BytesRefBuilder term = new BytesRefBuilder(); Bits liveDocs = context.reader().getLiveDocs(); @@ -701,7 +702,6 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore } } } - return this; } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/RandomSortField.java b/solr/core/src/java/org/apache/solr/schema/RandomSortField.java index cb2adbbc447..5d097a207e0 100644 --- a/solr/core/src/java/org/apache/solr/schema/RandomSortField.java +++ b/solr/core/src/java/org/apache/solr/schema/RandomSortField.java @@ -109,7 +109,7 @@ public class RandomSortField extends FieldType { private static FieldComparatorSource randomComparatorSource = new FieldComparatorSource() { @Override public FieldComparator newComparator(final String fieldname, final int numHits, int sortPos, boolean reversed) { - return new FieldComparator() { + return new SimpleFieldComparator() { int seed; private final int[] values = new int[numHits]; int bottomVal; @@ -141,9 +141,8 @@ public class RandomSortField extends FieldType { } @Override - public FieldComparator setNextReader(LeafReaderContext context) { + protected void doSetNextReader(LeafReaderContext context) { seed = getSeed(fieldname, context); - return this; } @Override diff --git a/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java index 0bafbc30b3c..fb372148417 100644 --- a/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java @@ -26,7 +26,6 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.common.params.SolrParams; - import java.io.IOException; import java.util.Map; import java.util.Set; @@ -129,22 +128,32 @@ public class ExportQParserPlugin extends QParserPlugin { private class ExportCollector extends TopDocsCollector { private FixedBitSet[] sets; - private FixedBitSet set; public ExportCollector(FixedBitSet[] sets) { super(null); this.sets = sets; } - - public void doSetNextReader(LeafReaderContext context) throws IOException { - this.set = new FixedBitSet(context.reader().maxDoc()); - this.sets[context.ord] = set; - } - - public void collect(int docId) throws IOException{ - ++totalHits; - set.set(docId); + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + final FixedBitSet set = new FixedBitSet(context.reader().maxDoc()); + this.sets[context.ord] = set; + return new LeafCollector() { + + @Override + public void setScorer(Scorer scorer) throws IOException {} + + @Override + public void collect(int docId) throws IOException{ + ++totalHits; + set.set(docId); + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return false; + } + }; } private ScoreDoc[] getScoreDocs(int howMany) { @@ -170,12 +179,5 @@ public class ExportQParserPlugin extends QParserPlugin { return new TopDocs(totalHits, scoreDocs, 0.0f); } - public void setScorer(Scorer scorer) throws IOException { - - } - - public boolean acceptsDocsOutOfOrder() { - return false; - } } } \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java index 5e744be380b..d96b27f2877 100644 --- a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java @@ -18,6 +18,7 @@ package org.apache.solr.search; import com.carrotsearch.hppc.IntIntOpenHashMap; + import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.MatchAllDocsQuery; @@ -33,7 +34,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.handler.component.MergeStrategy; import org.apache.solr.handler.component.QueryElevationComponent; import org.apache.solr.request.SolrQueryRequest; - +import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Query; import org.apache.lucene.search.Weight; import org.apache.lucene.search.IndexSearcher; @@ -43,6 +44,7 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.ScoreDoc; + import com.carrotsearch.hppc.IntFloatOpenHashMap; import org.apache.lucene.util.Bits; @@ -255,26 +257,15 @@ public class ReRankQParserPlugin extends QParserPlugin { this.reRankWeight = reRankWeight; } - public boolean acceptsDocsOutOfOrder() { - return false; - } - - public void collect(int doc) throws IOException { - mainCollector.collect(doc); - } - - public void setScorer(Scorer scorer) throws IOException{ - mainCollector.setScorer(scorer); - } - - public void doSetNextReader(LeafReaderContext context) throws IOException{ - mainCollector.getLeafCollector(context); - } - public int getTotalHits() { return mainCollector.getTotalHits(); } + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + return mainCollector.getLeafCollector(context); + } + public TopDocs topDocs(int start, int howMany) { try { @@ -387,6 +378,7 @@ public class ReRankQParserPlugin extends QParserPlugin { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); } } + } public class BoostedComp implements Comparator { diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 1efa33b5332..fbe737fcfba 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -2055,16 +2055,17 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn int end=0; int readerIndex = 0; + LeafCollector leafCollector = null; while (iter.hasNext()) { int doc = iter.nextDoc(); while (doc>=end) { LeafReaderContext leaf = leafContexts.get(readerIndex++); base = leaf.docBase; end = base + leaf.reader().maxDoc(); - topCollector.getLeafCollector(leaf); + leafCollector = topCollector.getLeafCollector(leaf); // we should never need to set the scorer given the settings for the collector } - topCollector.collect(doc-base); + leafCollector.collect(doc-base); } TopDocs topDocs = topCollector.topDocs(0, nDocs); diff --git a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java b/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java index 2e9a11a5096..e368eb92d15 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java @@ -23,6 +23,8 @@ import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; @@ -51,7 +53,6 @@ import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.request.SolrQueryRequest; - import org.junit.Ignore; import java.io.IOException; @@ -410,7 +411,8 @@ public class TestRankQueryPlugin extends QParserPlugin { // :TODO: would be simpler to always serialize every position of SortField[] if (type==SortField.Type.SCORE || type==SortField.Type.DOC) continue; - FieldComparator comparator = null; + FieldComparator comparator = null; + LeafFieldComparator leafComparator = null; Object[] vals = new Object[nDocs]; int lastIdx = -1; @@ -433,12 +435,12 @@ public class TestRankQueryPlugin extends QParserPlugin { if (comparator == null) { comparator = sortField.getComparator(1,0); - comparator = comparator.setNextReader(currentLeaf); + leafComparator = comparator.getLeafComparator(currentLeaf); } doc -= currentLeaf.docBase; // adjust for what segment this is in - comparator.setScorer(new FakeScorer(doc, score)); - comparator.copy(0, doc); + leafComparator.setScorer(new FakeScorer(doc, score)); + leafComparator.copy(0, doc); Object val = comparator.value(0); if (null != ft) val = ft.marshalSortValue(val); vals[position] = val; @@ -705,24 +707,28 @@ public class TestRankQueryPlugin extends QParserPlugin { class TestCollector extends TopDocsCollector { private List list = new ArrayList(); - private NumericDocValues values; - private int base; public TestCollector(PriorityQueue pq) { super(pq); } - public boolean acceptsDocsOutOfOrder() { - return false; - } + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + final int base = context.docBase; + final NumericDocValues values = DocValues.getNumeric(context.reader(), "sort_i"); + return new LeafCollector() { + + @Override + public void setScorer(Scorer scorer) throws IOException {} + + public boolean acceptsDocsOutOfOrder() { + return false; + } - public void doSetNextReader(LeafReaderContext context) throws IOException { - values = DocValues.getNumeric(context.reader(), "sort_i"); - base = context.docBase; - } - - public void collect(int doc) { - list.add(new ScoreDoc(doc+base, (float)values.get(doc))); + public void collect(int doc) { + list.add(new ScoreDoc(doc+base, (float)values.get(doc))); + } + }; } public int topDocsSize() { @@ -759,27 +765,32 @@ public class TestRankQueryPlugin extends QParserPlugin { class TestCollector1 extends TopDocsCollector { private List list = new ArrayList(); - private int base; - private Scorer scorer; public TestCollector1(PriorityQueue pq) { super(pq); } - public boolean acceptsDocsOutOfOrder() { - return false; - } - - public void doSetNextReader(LeafReaderContext context) throws IOException { - base = context.docBase; - } - - public void setScorer(Scorer scorer) { - this.scorer = scorer; - } - - public void collect(int doc) throws IOException { - list.add(new ScoreDoc(doc+base, scorer.score())); + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + final int base = context.docBase; + return new LeafCollector() { + + Scorer scorer; + + @Override + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + } + + public void collect(int doc) throws IOException { + list.add(new ScoreDoc(doc+base, scorer.score())); + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return false; + } + }; } public int topDocsSize() { @@ -813,7 +824,4 @@ public class TestRankQueryPlugin extends QParserPlugin { } } - - - }