mirror of https://github.com/apache/lucene.git
Merge branch 'master' of https://gitbox.apache.org/repos/asf/lucene-solr
This commit is contained in:
Erick Erickson
commit
3db79ed9ff
|
|
@ -133,10 +133,10 @@ Optimizations
|
|||
|
||||
* LUCENE-7788: fail precommit on unparameterised log messages and examine for wasted work/objects (Erick Erickson)
|
||||
|
||||
* SOLR-13289: When the "minExactHits" parameters is provided in queries and it's value is lower than the number of hits,
|
||||
* SOLR-13289: When the "minExactCount" parameters is provided in queries and it's value is lower than the number of hits,
|
||||
Solr can speedup the query resolution by using the Block-Max WAND algorithm (see LUCENE-8135). When doing this, the
|
||||
value of matching documents in the response (numFound) will be an approximation.
|
||||
(Ishan Chattopadhyaya, Munendra S N, Tomás Fernández Löbbe)
|
||||
(Ishan Chattopadhyaya, Munendra S N, Tomás Fernández Löbbe, David Smiley)
|
||||
|
||||
* SOLR-14472: Autoscaling "cores" preference now retrieves the core count more efficiently, and counts all cores.
|
||||
(David Smiley)
|
||||
|
|
@ -192,6 +192,9 @@ Bug Fixes
|
|||
* SOLR-14477: Fix incorrect 'relatedness()' calculations in json.facet 'terms' when 'prefix' option is used
|
||||
(hossman)
|
||||
|
||||
* SOLR-14504: ZkController LiveNodesListener has NullPointerException in startup race.
|
||||
(Colvin Cowie via ab)
|
||||
|
||||
Other Changes
|
||||
---------------------
|
||||
* SOLR-14197: SolrResourceLoader: marked many methods as deprecated, and in some cases rerouted exiting logic to avoid
|
||||
|
|
|
|||
|
|
@ -1016,7 +1016,7 @@ public class ZkController implements Closeable {
|
|||
log.warn("Unable to read autoscaling.json", e1);
|
||||
}
|
||||
if (createNodes) {
|
||||
byte[] json = Utils.toJSON(Collections.singletonMap("timestamp", cloudManager.getTimeSource().getEpochTimeNs()));
|
||||
byte[] json = Utils.toJSON(Collections.singletonMap("timestamp", getSolrCloudManager().getTimeSource().getEpochTimeNs()));
|
||||
for (String n : oldNodes) {
|
||||
String path = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + n;
|
||||
|
||||
|
|
|
|||
|
|
@ -366,7 +366,7 @@ public class QueryComponent extends SearchComponent
|
|||
|
||||
QueryCommand cmd = rb.createQueryCommand();
|
||||
cmd.setTimeAllowed(timeAllowed);
|
||||
cmd.setMinExactHits(getMinExactHits(params));
|
||||
cmd.setMinExactCount(getMinExactCount(params));
|
||||
|
||||
req.getContext().put(SolrIndexSearcher.STATS_SOURCE, statsCache.get(req));
|
||||
|
||||
|
|
@ -403,12 +403,12 @@ public class QueryComponent extends SearchComponent
|
|||
doProcessUngroupedSearch(rb, cmd, result);
|
||||
}
|
||||
|
||||
private int getMinExactHits(SolrParams params) {
|
||||
long minExactHits = params.getLong(CommonParams.MIN_EXACT_HITS, Integer.MAX_VALUE);
|
||||
if (minExactHits < 0 || minExactHits > Integer.MAX_VALUE) {
|
||||
minExactHits = Integer.MAX_VALUE;
|
||||
private int getMinExactCount(SolrParams params) {
|
||||
long minExactCount = params.getLong(CommonParams.MIN_EXACT_COUNT, Integer.MAX_VALUE);
|
||||
if (minExactCount < 0 || minExactCount > Integer.MAX_VALUE) {
|
||||
minExactCount = Integer.MAX_VALUE;
|
||||
}
|
||||
return (int)minExactHits;
|
||||
return (int)minExactCount;
|
||||
}
|
||||
|
||||
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ public class QueryCommand {
|
|||
private int supersetMaxDoc;
|
||||
private int flags;
|
||||
private long timeAllowed = -1;
|
||||
private int minExactHits = Integer.MAX_VALUE;
|
||||
private int minExactCount = Integer.MAX_VALUE;
|
||||
private CursorMark cursorMark;
|
||||
|
||||
public CursorMark getCursorMark() {
|
||||
|
|
@ -184,12 +184,12 @@ public class QueryCommand {
|
|||
return this;
|
||||
}
|
||||
|
||||
public int getMinExactHits() {
|
||||
return minExactHits;
|
||||
public int getMinExactCount() {
|
||||
return minExactCount;
|
||||
}
|
||||
|
||||
public QueryCommand setMinExactHits(int hits) {
|
||||
this.minExactHits = hits;
|
||||
public QueryCommand setMinExactCount(int count) {
|
||||
this.minExactCount = count;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ public final class QueryResultKey implements Accountable {
|
|||
final SortField[] sfields;
|
||||
final List<Query> filters;
|
||||
final int nc_flags; // non-comparable flags... ignored by hashCode and equals
|
||||
final int minExactHits;
|
||||
final int minExactCount;
|
||||
|
||||
private final int hc; // cached hashCode
|
||||
private final long ramBytesUsed; // cached
|
||||
|
|
@ -48,12 +48,12 @@ public final class QueryResultKey implements Accountable {
|
|||
this(query, filters, sort, nc_flags, Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
public QueryResultKey(Query query, List<Query> filters, Sort sort, int nc_flags, int minExactHits) {
|
||||
public QueryResultKey(Query query, List<Query> filters, Sort sort, int nc_flags, int minExactCount) {
|
||||
this.query = query;
|
||||
this.sort = sort;
|
||||
this.filters = filters;
|
||||
this.nc_flags = nc_flags;
|
||||
this.minExactHits = minExactHits;
|
||||
this.minExactCount = minExactCount;
|
||||
|
||||
int h = query.hashCode();
|
||||
|
||||
|
|
@ -70,7 +70,7 @@ public final class QueryResultKey implements Accountable {
|
|||
h = h*29 + sf.hashCode();
|
||||
ramSfields += BASE_SF_RAM_BYTES_USED + RamUsageEstimator.sizeOfObject(sf.getField());
|
||||
}
|
||||
h = h*31 + minExactHits;
|
||||
h = h*31 + minExactCount;
|
||||
|
||||
hc = h;
|
||||
|
||||
|
|
@ -102,7 +102,7 @@ public final class QueryResultKey implements Accountable {
|
|||
if (this.sfields.length != other.sfields.length) return false;
|
||||
if (!this.query.equals(other.query)) return false;
|
||||
if (!unorderedCompare(this.filters, other.filters)) return false;
|
||||
if (this.minExactHits != other.minExactHits) return false;
|
||||
if (this.minExactCount != other.minExactCount) return false;
|
||||
|
||||
for (int i=0; i<sfields.length; i++) {
|
||||
SortField sf1 = this.sfields[i];
|
||||
|
|
|
|||
|
|
@ -160,13 +160,14 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
|
|||
UninvertingReader.wrap(reader, core.getLatestSchema().getUninversionMapper()),
|
||||
SolrQueryTimeoutImpl.getInstance());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Builds the necessary collector chain (via delegate wrapping) and executes the query against it. This method takes
|
||||
* into consideration both the explicitly provided collector and postFilter as well as any needed collector wrappers
|
||||
* for dealing with options specified in the QueryCommand.
|
||||
* @return The collector used for search
|
||||
*/
|
||||
private void buildAndRunCollectorChain(QueryResult qr, Query query, Collector collector, QueryCommand cmd,
|
||||
private Collector buildAndRunCollectorChain(QueryResult qr, Query query, Collector collector, QueryCommand cmd,
|
||||
DelegatingCollector postFilter) throws IOException {
|
||||
|
||||
EarlyTerminatingSortingCollector earlyTerminatingSortingCollector = null;
|
||||
|
|
@ -216,6 +217,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
|
|||
if (collector instanceof DelegatingCollector) {
|
||||
((DelegatingCollector) collector).finish();
|
||||
}
|
||||
return collector;
|
||||
}
|
||||
|
||||
public SolrIndexSearcher(SolrCore core, String path, IndexSchema schema, SolrIndexConfig config, String name,
|
||||
|
|
@ -1306,7 +1308,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
|
|||
&& (flags & (NO_CHECK_QCACHE | NO_SET_QCACHE)) != ((NO_CHECK_QCACHE | NO_SET_QCACHE))) {
|
||||
// all of the current flags can be reused during warming,
|
||||
// so set all of them on the cache key.
|
||||
key = new QueryResultKey(q, cmd.getFilterList(), cmd.getSort(), flags, cmd.getMinExactHits());
|
||||
key = new QueryResultKey(q, cmd.getFilterList(), cmd.getSort(), flags, cmd.getMinExactCount());
|
||||
if ((flags & NO_CHECK_QCACHE) == 0) {
|
||||
superset = queryResultCache.get(key);
|
||||
|
||||
|
|
@ -1483,7 +1485,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
|
|||
* The Command whose properties should determine the type of TopDocsCollector to use.
|
||||
*/
|
||||
private TopDocsCollector buildTopDocsCollector(int len, QueryCommand cmd) throws IOException {
|
||||
int minNumFound = cmd.getMinExactHits();
|
||||
int minNumFound = cmd.getMinExactCount();
|
||||
Query q = cmd.getQuery();
|
||||
if (q instanceof RankQuery) {
|
||||
RankQuery rq = (RankQuery) q;
|
||||
|
|
@ -1580,11 +1582,15 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
|
|||
maxScoreCollector = new MaxScoreCollector();
|
||||
collector = MultiCollector.wrap(topCollector, maxScoreCollector);
|
||||
}
|
||||
buildAndRunCollectorChain(qr, query, collector, cmd, pf.postFilter);
|
||||
ScoreMode scoreModeUsed = buildAndRunCollectorChain(qr, query, collector, cmd, pf.postFilter).scoreMode();
|
||||
|
||||
totalHits = topCollector.getTotalHits();
|
||||
TopDocs topDocs = topCollector.topDocs(0, len);
|
||||
hitsRelation = topDocs.totalHits.relation;
|
||||
if (scoreModeUsed == ScoreMode.COMPLETE || scoreModeUsed == ScoreMode.COMPLETE_NO_SCORES) {
|
||||
hitsRelation = TotalHits.Relation.EQUAL_TO;
|
||||
} else {
|
||||
hitsRelation = topDocs.totalHits.relation;
|
||||
}
|
||||
if (cmd.getSort() != null && query instanceof RankQuery == false && (cmd.getFlags() & GET_SCORES) != 0) {
|
||||
TopFieldCollector.populateScores(topDocs.scoreDocs, this, query);
|
||||
}
|
||||
|
|
@ -1622,7 +1628,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
|
|||
|
||||
boolean needScores = (cmd.getFlags() & GET_SCORES) != 0;
|
||||
int maxDoc = maxDoc();
|
||||
cmd.setMinExactHits(Integer.MAX_VALUE);// We need the full DocSet
|
||||
cmd.setMinExactCount(Integer.MAX_VALUE);// We need the full DocSet
|
||||
|
||||
ProcessedFilter pf = getProcessedFilter(cmd.getFilter(), cmd.getFilterList());
|
||||
final Query query = QueryUtils.combineQueryAndFilter(QueryUtils.makeQueryable(cmd.getQuery()), pf.filter);
|
||||
|
|
|
|||
|
|
@ -211,7 +211,7 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
|
|||
query("q","*:*", "sort","n_tl1 desc");
|
||||
|
||||
handle.put("maxScore", SKIPVAL);
|
||||
testMinExactHits();
|
||||
testMinExactCount();
|
||||
|
||||
query("q","{!func}"+i1);// does not expect maxScore. So if it comes ,ignore it. JavaBinCodec.writeSolrDocumentList()
|
||||
//is agnostic of request params.
|
||||
|
|
@ -1090,13 +1090,13 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
|
|||
"stats.facet", fieldName);
|
||||
}
|
||||
|
||||
private void testMinExactHits() throws Exception {
|
||||
assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_HITS, "200", CommonParams.ROWS, "2", CommonParams.SORT, "score desc, id asc");
|
||||
assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_HITS, "-1", CommonParams.ROWS, "2", CommonParams.SORT, "score desc, id asc");
|
||||
assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_HITS, "1", CommonParams.ROWS, "200", CommonParams.SORT, "score desc, id asc");
|
||||
assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", "facet", "true", "facet.field", s1, CommonParams.MIN_EXACT_HITS,"1", CommonParams.ROWS, "200", CommonParams.SORT, "score desc, id asc");
|
||||
assertIsExactHitCount("q","{!cache=false}id:1", CommonParams.MIN_EXACT_HITS,"1", CommonParams.ROWS, "1");
|
||||
assertApproximatedHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_HITS,"2", CommonParams.ROWS, "2", CommonParams.SORT, "score desc, id asc");
|
||||
private void testMinExactCount() throws Exception {
|
||||
assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_COUNT, "200", CommonParams.ROWS, "2", CommonParams.SORT, "score desc, id asc");
|
||||
assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_COUNT, "-1", CommonParams.ROWS, "2", CommonParams.SORT, "score desc, id asc");
|
||||
assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_COUNT, "1", CommonParams.ROWS, "200", CommonParams.SORT, "score desc, id asc");
|
||||
assertIsExactHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", "facet", "true", "facet.field", s1, CommonParams.MIN_EXACT_COUNT,"1", CommonParams.ROWS, "200", CommonParams.SORT, "score desc, id asc");
|
||||
assertIsExactHitCount("q","{!cache=false}id:1", CommonParams.MIN_EXACT_COUNT,"1", CommonParams.ROWS, "1");
|
||||
assertApproximatedHitCount("q","{!cache=false}dog OR men OR cow OR country OR dumpty", CommonParams.MIN_EXACT_COUNT,"2", CommonParams.ROWS, "2", CommonParams.SORT, "score desc, id asc");
|
||||
}
|
||||
|
||||
private void assertIsExactHitCount(Object... requestParams) throws Exception {
|
||||
|
|
|
|||
|
|
@ -950,7 +950,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
|
|||
}
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
params.set("q", FOO_STRING_FIELD + ":Book1");
|
||||
assertQ(req(params, CommonParams.MIN_EXACT_HITS, "2", CommonParams.ROWS, "2")
|
||||
assertQ(req(params, CommonParams.MIN_EXACT_COUNT, "2", CommonParams.ROWS, "2")
|
||||
,"/response/result[@numFoundExact='false']"
|
||||
);
|
||||
params.set("group", true);
|
||||
|
|
@ -959,7 +959,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
|
|||
,"/response/lst[@name='grouped']/lst[@name='"+FOO_STRING_FIELD+"']/arr[@name='groups']/lst[1]/result[@numFoundExact='true']"
|
||||
);
|
||||
|
||||
assertQ(req(params, CommonParams.MIN_EXACT_HITS, "2", CommonParams.ROWS, "2")
|
||||
assertQ(req(params, CommonParams.MIN_EXACT_COUNT, "2", CommonParams.ROWS, "2")
|
||||
,"/response/lst[@name='grouped']/lst[@name='"+FOO_STRING_FIELD+"']/arr[@name='groups']/lst[1]/result[@numFoundExact='true']"
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ public class QueryResultKeyTest extends SolrTestCaseJ4 {
|
|||
assert minIters <= iter;
|
||||
}
|
||||
|
||||
public void testMinExactHits() {
|
||||
public void testMinExactCount() {
|
||||
int[] nums = smallArrayOfRandomNumbers();
|
||||
final Query base = new FlatHashTermQuery("base");
|
||||
assertKeyEquals(new QueryResultKey(base, buildFiltersFromNumbers(nums), null, 0, 10),
|
||||
|
|
|
|||
|
|
@ -935,7 +935,7 @@ public class TestFaceting extends SolrTestCaseJ4 {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testFacetCountsWithMinExactHits() throws Exception {
|
||||
public void testFacetCountsWithMinExactCount() throws Exception {
|
||||
final int NUM_DOCS = 20;
|
||||
for (int i = 0; i < NUM_DOCS ; i++) {
|
||||
assertU(adoc("id", String.valueOf(i), "title_ws", "Book1"));
|
||||
|
|
@ -950,8 +950,8 @@ public class TestFaceting extends SolrTestCaseJ4 {
|
|||
,"//*[@numFoundExact='true']"
|
||||
,"//*[@numFound='" + NUM_DOCS + "']");
|
||||
|
||||
// It doesn't matter if we request minExactHits, when requesting facets, the numFound value is precise
|
||||
assertQ(req(params, CommonParams.MIN_EXACT_HITS, "2", CommonParams.ROWS, "2"),
|
||||
// It doesn't matter if we request minExactCount, when requesting facets, the numFound value is precise
|
||||
assertQ(req(params, CommonParams.MIN_EXACT_COUNT, "2", CommonParams.ROWS, "2"),
|
||||
"//lst[@name='facet_fields']/lst[@name='title_ws']/int[1][@name='Book1'][.='20']"
|
||||
,"//*[@numFoundExact='true']"
|
||||
,"//*[@numFound='" + NUM_DOCS + "']");
|
||||
|
|
|
|||
|
|
@ -16,185 +16,315 @@
|
|||
*/
|
||||
package org.apache.solr.search;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.ScoreMode;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.search.TotalHits;
|
||||
import org.apache.lucene.search.Weight;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class SolrIndexSearcherTest extends SolrTestCaseJ4 {
|
||||
|
||||
|
||||
private final static int NUM_DOCS = 20;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUpClass() throws Exception {
|
||||
initCore("solrconfig.xml", "schema.xml");
|
||||
for (int i = 0 ; i < NUM_DOCS ; i ++) {
|
||||
assertU(adoc("id", String.valueOf(i), "field1_s", "foo", "field2_s", String.valueOf(i % 2), "field3_s", String.valueOf(i)));
|
||||
assertU(commit());
|
||||
assertU(adoc("id", String.valueOf(i),
|
||||
"field1_s", "foo",
|
||||
"field2_s", String.valueOf(i % 2),
|
||||
"field3_i_dvo", String.valueOf(i),
|
||||
"field4_t", numbersTo(i)));
|
||||
assertU(commit()); //commit inside the loop to get multiple segments
|
||||
}
|
||||
}
|
||||
|
||||
private static String numbersTo(int i) {
|
||||
StringBuilder numbers = new StringBuilder();
|
||||
for (int j = 0; j <= i ; j++) {
|
||||
numbers.append(String.valueOf(j) + " ");
|
||||
}
|
||||
return numbers.toString();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
assertU(adoc("id", "1", "field1_s", "foo", "field2_s", "1", "field3_s", "1"));
|
||||
assertU(adoc("id", "1",
|
||||
"field1_s", "foo",
|
||||
"field2_s", "1",
|
||||
"field3_i_dvo", "1",
|
||||
"field4_t", numbersTo(1)));
|
||||
assertU(commit());
|
||||
super.setUp();
|
||||
}
|
||||
|
||||
public void testMinExactHitsLongValue() {
|
||||
public void testMinExactCountLongValue() {
|
||||
assertQ("test query on empty index",
|
||||
req("q", "field1_s:foo",
|
||||
"minExactHits", Long.toString(10L * Integer.MAX_VALUE),
|
||||
"minExactCount", Long.toString(10L * Integer.MAX_VALUE),
|
||||
"rows", "2")
|
||||
,"//*[@numFoundExact='true']"
|
||||
,"//*[@numFound='" + NUM_DOCS + "']"
|
||||
);
|
||||
}
|
||||
|
||||
public void testMinExactHits() {
|
||||
assertQ("minExactHits is lower than numFound,should produce approximated results",
|
||||
public void testMinExactCount() {
|
||||
assertQ("minExactCount is lower than numFound,should produce approximated results",
|
||||
req("q", "field1_s:foo",
|
||||
"minExactHits", "2",
|
||||
"minExactCount", "2",
|
||||
"rows", "2")
|
||||
,"//*[@numFoundExact='false']"
|
||||
,"//*[@numFound<='" + NUM_DOCS + "']"
|
||||
);
|
||||
assertQ("minExactHits is higher than numFound,should produce exact results",
|
||||
assertQ("minExactCount is higher than numFound,should produce exact results",
|
||||
req("q", "field1_s:foo",
|
||||
"minExactHits", "200",
|
||||
"minExactCount", "200",
|
||||
"rows", "2")
|
||||
,"//*[@numFoundExact='true']"
|
||||
,"//*[@numFound='" + NUM_DOCS + "']"
|
||||
);
|
||||
}
|
||||
|
||||
private void assertMatchesEqual(int expectedCount, QueryResult qr) {
|
||||
private void assertMatchesEqual(int expectedCount, SolrIndexSearcher searcher, QueryCommand cmd) throws IOException {
|
||||
QueryResult qr = new QueryResult();
|
||||
searcher.search(qr, cmd);
|
||||
assertEquals(expectedCount, qr.getDocList().matches());
|
||||
assertEquals(TotalHits.Relation.EQUAL_TO, qr.getDocList().hitCountRelation());
|
||||
}
|
||||
|
||||
private void assertMatchesGraterThan(int expectedCount, QueryResult qr) {
|
||||
private QueryResult assertMatchesGreaterThan(int expectedCount, SolrIndexSearcher searcher, QueryCommand cmd) throws IOException {
|
||||
QueryResult qr = new QueryResult();
|
||||
searcher.search(qr, cmd);
|
||||
assertTrue("Expecting returned matches to be greater than " + expectedCount + " but got " + qr.getDocList().matches(),
|
||||
expectedCount >= qr.getDocList().matches());
|
||||
assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, qr.getDocList().hitCountRelation());
|
||||
return qr;
|
||||
}
|
||||
|
||||
public void testLowMinExactHitsGeneratesApproximation() throws IOException {
|
||||
public void testLowMinExactCountGeneratesApproximation() throws IOException {
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = new QueryCommand();
|
||||
cmd.setMinExactHits(NUM_DOCS / 2);
|
||||
cmd.setQuery(new TermQuery(new Term("field1_s", "foo")));
|
||||
QueryResult qr = new QueryResult();
|
||||
searcher.search(qr, cmd);
|
||||
assertMatchesGraterThan(NUM_DOCS, qr);
|
||||
QueryCommand cmd = createBasicQueryCommand(NUM_DOCS / 2, 10, "field1_s", "foo");
|
||||
assertMatchesGreaterThan(NUM_DOCS, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = new QueryCommand();
|
||||
cmd.setMinExactHits(1);
|
||||
cmd.setLen(1);
|
||||
// We need to disable cache, otherwise the search will be done for 20 docs (cache window size) which brings up the minExactHits
|
||||
cmd.setFlags(SolrIndexSearcher.NO_CHECK_QCACHE | SolrIndexSearcher.NO_SET_QCACHE);
|
||||
cmd.setQuery(new TermQuery(new Term("field2_s", "1")));
|
||||
QueryResult qr = new QueryResult();
|
||||
searcher.search(qr, cmd);
|
||||
assertMatchesGraterThan(NUM_DOCS/2, qr);
|
||||
QueryCommand cmd = createBasicQueryCommand(1, 1, "field2_s", "1");
|
||||
assertMatchesGreaterThan(NUM_DOCS/2, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
public void testHighMinExactHitsGeneratesExactCount() throws IOException {
|
||||
|
||||
public void testHighMinExactCountGeneratesExactCount() throws IOException {
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = new QueryCommand();
|
||||
cmd.setMinExactHits(NUM_DOCS);
|
||||
cmd.setQuery(new TermQuery(new Term("field1_s", "foo")));
|
||||
QueryResult qr = new QueryResult();
|
||||
searcher.search(qr, cmd);
|
||||
assertMatchesEqual(NUM_DOCS, qr);
|
||||
QueryCommand cmd = createBasicQueryCommand(NUM_DOCS, 10, "field1_s", "foo");
|
||||
assertMatchesEqual(NUM_DOCS, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = new QueryCommand();
|
||||
cmd.setMinExactHits(NUM_DOCS);
|
||||
cmd.setQuery(new TermQuery(new Term("field2_s", "1")));
|
||||
QueryResult qr = new QueryResult();
|
||||
searcher.search(qr, cmd);
|
||||
assertMatchesEqual(NUM_DOCS/2, qr);
|
||||
QueryCommand cmd = createBasicQueryCommand(NUM_DOCS, 10, "field2_s", "1");
|
||||
assertMatchesEqual(NUM_DOCS/2, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public void testLowMinExactHitsWithQueryResultCache() throws IOException {
|
||||
|
||||
public void testLowMinExactCountWithQueryResultCache() throws IOException {
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = new QueryCommand();
|
||||
cmd.setMinExactHits(NUM_DOCS / 2);
|
||||
cmd.setQuery(new TermQuery(new Term("field1_s", "foo")));
|
||||
QueryCommand cmd = createBasicQueryCommand(NUM_DOCS / 2, 10, "field1_s", "foo");
|
||||
cmd.clearFlags(SolrIndexSearcher.NO_CHECK_QCACHE | SolrIndexSearcher.NO_SET_QCACHE);
|
||||
searcher.search(new QueryResult(), cmd);
|
||||
QueryResult qr = new QueryResult();
|
||||
searcher.search(qr, cmd);
|
||||
assertMatchesGraterThan(NUM_DOCS, qr);
|
||||
assertMatchesGreaterThan(NUM_DOCS, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
public void testHighMinExactHitsWithQueryResultCache() throws IOException {
|
||||
public void testHighMinExactCountWithQueryResultCache() throws IOException {
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = new QueryCommand();
|
||||
cmd.setMinExactHits(NUM_DOCS);
|
||||
cmd.setQuery(new TermQuery(new Term("field1_s", "foo")));
|
||||
QueryCommand cmd = createBasicQueryCommand(NUM_DOCS, 2, "field1_s", "foo");
|
||||
cmd.clearFlags(SolrIndexSearcher.NO_CHECK_QCACHE | SolrIndexSearcher.NO_SET_QCACHE);
|
||||
searcher.search(new QueryResult(), cmd);
|
||||
QueryResult qr = new QueryResult();
|
||||
searcher.search(qr, cmd);
|
||||
assertMatchesEqual(NUM_DOCS, qr);
|
||||
assertMatchesEqual(NUM_DOCS, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
public void testMinExactHitsMoreRows() throws IOException {
|
||||
public void testMinExactCountMoreRows() throws IOException {
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = new QueryCommand();
|
||||
cmd.setMinExactHits(2);
|
||||
cmd.setLen(NUM_DOCS);
|
||||
cmd.setQuery(new TermQuery(new Term("field1_s", "foo")));
|
||||
QueryResult qr = new QueryResult();
|
||||
searcher.search(qr, cmd);
|
||||
assertMatchesEqual(NUM_DOCS, qr);
|
||||
QueryCommand cmd = createBasicQueryCommand(2, NUM_DOCS, "field1_s", "foo");
|
||||
assertMatchesEqual(NUM_DOCS, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
public void testMinExactHitsMatchWithDocSet() throws IOException {
|
||||
public void testMinExactCountMatchWithDocSet() throws IOException {
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = new QueryCommand();
|
||||
QueryCommand cmd = createBasicQueryCommand(2, 2, "field1_s", "foo");
|
||||
assertMatchesGreaterThan(NUM_DOCS, searcher, cmd);
|
||||
|
||||
cmd.setNeedDocSet(true);
|
||||
cmd.setMinExactHits(2);
|
||||
cmd.setQuery(new TermQuery(new Term("field1_s", "foo")));
|
||||
searcher.search(new QueryResult(), cmd);
|
||||
QueryResult qr = new QueryResult();
|
||||
searcher.search(qr, cmd);
|
||||
assertMatchesEqual(NUM_DOCS, qr);
|
||||
assertMatchesEqual(NUM_DOCS, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
public void testMinExactHitsWithMaxScoreRequested() throws IOException {
|
||||
public void testMinExactCountWithMaxScoreRequested() throws IOException {
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = new QueryCommand();
|
||||
cmd.setMinExactHits(2);
|
||||
QueryCommand cmd = createBasicQueryCommand(2, 2, "field1_s", "foo");
|
||||
cmd.setFlags(SolrIndexSearcher.GET_SCORES);
|
||||
cmd.setQuery(new TermQuery(new Term("field1_s", "foo")));
|
||||
searcher.search(new QueryResult(), cmd);
|
||||
QueryResult qr = new QueryResult();
|
||||
searcher.search(qr, cmd);
|
||||
assertMatchesGraterThan(NUM_DOCS, qr);
|
||||
QueryResult qr = assertMatchesGreaterThan(NUM_DOCS, searcher, cmd);
|
||||
assertNotEquals(Float.NaN, qr.getDocList().maxScore());
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
public void testMinExactWithFilters() throws Exception {
|
||||
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
//Sanity Check - No Filter
|
||||
QueryCommand cmd = createBasicQueryCommand(1, 1, "field4_t", "0");
|
||||
assertMatchesGreaterThan(NUM_DOCS, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
|
||||
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = createBasicQueryCommand(1, 1, "field4_t", "0");
|
||||
Query filterQuery = new TermQuery(new Term("field4_t", "19"));
|
||||
cmd.setFilterList(filterQuery);
|
||||
assertNull(searcher.getProcessedFilter(null, cmd.getFilterList()).postFilter);
|
||||
assertMatchesEqual(1, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
public void testMinExactWithPostFilters() throws Exception {
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
//Sanity Check - No Filter
|
||||
QueryCommand cmd = createBasicQueryCommand(1, 1, "field4_t", "0");
|
||||
assertMatchesGreaterThan(NUM_DOCS, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
|
||||
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = createBasicQueryCommand(1, 1, "field4_t", "0");
|
||||
MockPostFilter filterQuery = new MockPostFilter(1, 101);
|
||||
cmd.setFilterList(filterQuery);
|
||||
assertNotNull(searcher.getProcessedFilter(null, cmd.getFilterList()).postFilter);
|
||||
assertMatchesEqual(1, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = createBasicQueryCommand(1, 1, "field4_t", "0");
|
||||
MockPostFilter filterQuery = new MockPostFilter(100, 101);
|
||||
cmd.setFilterList(filterQuery);
|
||||
assertNotNull(searcher.getProcessedFilter(null, cmd.getFilterList()).postFilter);
|
||||
assertMatchesGreaterThan(NUM_DOCS, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
public void testMinExactWithPostFilterThatChangesScoreMode() throws Exception {
|
||||
h.getCore().withSearcher(searcher -> {
|
||||
QueryCommand cmd = createBasicQueryCommand(1, 1, "field4_t", "0");
|
||||
// Use ScoreMode.COMPLETE for the PostFilter
|
||||
MockPostFilter filterQuery = new MockPostFilter(100, 101, ScoreMode.COMPLETE);
|
||||
cmd.setFilterList(filterQuery);
|
||||
assertNotNull(searcher.getProcessedFilter(null, cmd.getFilterList()).postFilter);
|
||||
assertMatchesEqual(NUM_DOCS, searcher, cmd);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private QueryCommand createBasicQueryCommand(int minExactCount, int length, String field, String q) {
|
||||
QueryCommand cmd = new QueryCommand();
|
||||
cmd.setMinExactCount(minExactCount);
|
||||
cmd.setLen(length);
|
||||
cmd.setFlags(SolrIndexSearcher.NO_CHECK_QCACHE | SolrIndexSearcher.NO_SET_QCACHE);
|
||||
cmd.setQuery(new TermQuery(new Term(field, q)));
|
||||
return cmd;
|
||||
}
|
||||
|
||||
private final static class MockPostFilter extends TermQuery implements PostFilter {
|
||||
|
||||
private final int cost;
|
||||
private final int maxDocsToCollect;
|
||||
private final ScoreMode scoreMode;
|
||||
|
||||
public MockPostFilter(int maxDocsToCollect, int cost, ScoreMode scoreMode) {
|
||||
super(new Term("foo", "bar"));//The term won't really be used. just the collector
|
||||
assert cost > 100;
|
||||
this.cost = cost;
|
||||
this.maxDocsToCollect = maxDocsToCollect;
|
||||
this.scoreMode = scoreMode;
|
||||
}
|
||||
|
||||
public MockPostFilter(int maxDocsToCollect, int cost) {
|
||||
this(maxDocsToCollect, cost, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
|
||||
throw new UnsupportedOperationException("This class is only intended to be used as a PostFilter");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean getCache() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setCache(boolean cache) {}
|
||||
|
||||
@Override
|
||||
public int getCost() {
|
||||
return cost;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setCost(int cost) {}
|
||||
|
||||
@Override
|
||||
public boolean getCacheSep() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setCacheSep(boolean cacheSep) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public DelegatingCollector getFilterCollector(IndexSearcher searcher) {
|
||||
return new DelegatingCollector() {
|
||||
private int collected = 0;
|
||||
@Override
|
||||
public void collect(int doc) throws IOException {
|
||||
if (++collected <= maxDocsToCollect) {
|
||||
super.collect(doc);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScoreMode scoreMode() {
|
||||
if (scoreMode != null) {
|
||||
return scoreMode;
|
||||
}
|
||||
return super.scoreMode();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1039,4 +1039,33 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
|
|||
assertQEx("Should Fail For collapsing on Date fields", "Collapsing field should be of either String, Int or Float type",
|
||||
req("q", "*:*", "fq", "{!collapse field=group_dt}"), SolrException.ErrorCode.BAD_REQUEST);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMinExactCountDisabledByCollapse() throws Exception {
|
||||
int numDocs = 10;
|
||||
String collapseFieldInt = "field_ti_dv";
|
||||
String collapseFieldFloat = "field_tf_dv";
|
||||
String collapseFieldString = "field_s_dv";
|
||||
for (int i = 0 ; i < numDocs ; i ++) {
|
||||
assertU(adoc(
|
||||
"id", String.valueOf(i),
|
||||
"field_s", String.valueOf(i % 2),
|
||||
collapseFieldInt, String.valueOf(i),
|
||||
collapseFieldFloat, String.valueOf(i),
|
||||
collapseFieldString, String.valueOf(i)));
|
||||
assertU(commit());
|
||||
}
|
||||
|
||||
for (String collapseField : new String[] {collapseFieldInt, collapseFieldFloat, collapseFieldString}) {
|
||||
assertQ(req(
|
||||
"q", "{!cache=false}field_s:1",
|
||||
"rows", "1",
|
||||
"minExactCount", "1",
|
||||
// this collapse will end up matching all docs
|
||||
"fq", "{!collapse field=" + collapseField + " nullPolicy=expand}"// nullPolicy needed due to a bug when val=0
|
||||
),"//*[@numFoundExact='true']"
|
||||
,"//*[@numFound='" + (numDocs/2) + "']"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -361,3 +361,43 @@ This is what happens if a similar request is sent that adds `echoParams=all` to
|
|||
}
|
||||
}
|
||||
----
|
||||
|
||||
== minExactCount Parameter
|
||||
When this parameter is used, Solr will count the number of hits accurately at least until this value. After that, Solr can skip over documents that don't have a score high enough to enter in the top N. This can greatly improve performance of search queries. On the other hand, when this parameter is used, the `numFound` may not be exact, and may instead be an approximation.
|
||||
The `numFoundExact` boolean attribute is included in all responses, indicating if the `numFound` value is exact or an approximation. If it's an approximation, the real number of hits for the query is guaranteed to be greater or equal `numFound`.
|
||||
|
||||
More about approximate document counting and `minExactCount`:
|
||||
|
||||
* The documents returned in the response are guaranteed to be the docs with the top scores. This parameter will not make Solr skip documents that are to be returned in the response, it will only allow Solr to skip counting docs that, while they match the query, their score is low enough to not be in the top N.
|
||||
* Providing `minExactCount` doesn't guarantee that Solr will use approximate hit counting (and thus, provide the speedup). Some types of queries, or other parameters (like if facets are requested) will require accurate counting. The value of `numFoundExact` indicates if the approximation was used or not.
|
||||
* Approximate counting can only be used when sorting by `score desc` first (which is the default sort in Solr). Other fields can be used after `score desc`, but if any other type of sorting is used before score, then the approximation won't be applied.
|
||||
* When doing distributed queries across multiple shards, each shard will accurately count hits until `minExactCount` (which means the query could be hitting `numShards * minExactCount` docs and `numFound` in the response would still be accurate)
|
||||
For example:
|
||||
|
||||
[source,text]
|
||||
q=quick brown fox&minExactCount=100&rows=10
|
||||
|
||||
[source,json]
|
||||
----
|
||||
"response": {
|
||||
"numFound": 153,
|
||||
"start": 0,
|
||||
"numFoundExact": false,
|
||||
"docs": Array[10]
|
||||
...
|
||||
----
|
||||
Since `numFoundExact=false`, we know the number of documents matching the query is greater or equal to 153. If we specify a higher value for `minExactCount`:
|
||||
|
||||
[source,text]
|
||||
q=quick brown fox&minExactCount=200&rows=10
|
||||
|
||||
[source,json]
|
||||
----
|
||||
"response": {
|
||||
"numFound": 163,
|
||||
"start": 0,
|
||||
"numFoundExact": true,
|
||||
"docs": Array[10]
|
||||
...
|
||||
----
|
||||
In this case we know that `163` is the exact number of hits for the query. Both queries must have returned the same number of documents in the top 10.
|
||||
|
|
|
|||
|
|
@ -164,10 +164,10 @@ public interface CommonParams {
|
|||
String TIME_ALLOWED = "timeAllowed";
|
||||
|
||||
/**
|
||||
* The number of hits that need to be counted accurately. If more than {@link #MIN_EXACT_HITS} documents
|
||||
* The number of hits that need to be counted accurately. If more than {@link #MIN_EXACT_COUNT} documents
|
||||
* match a query, then the value in "numFound" may be an estimate to speedup search.
|
||||
*/
|
||||
String MIN_EXACT_HITS = "minExactHits";
|
||||
String MIN_EXACT_COUNT = "minExactCount";
|
||||
|
||||
/** 'true' if the header should include the handler name */
|
||||
String HEADER_ECHO_HANDLER = "echoHandler";
|
||||
|
|
|
|||
|
|
@ -34,5 +34,5 @@ public class CommonParamsTest extends SolrTestCase
|
|||
|
||||
public void testPreferLocalShards() { assertEquals("preferLocalShards", CommonParams.PREFER_LOCAL_SHARDS); }
|
||||
|
||||
public void testMinExactHits() { assertEquals("minExactHits", CommonParams.MIN_EXACT_HITS); }
|
||||
public void testMinExactCount() { assertEquals("minExactCount", CommonParams.MIN_EXACT_COUNT); }
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue