From b912f6da984ea75d550f76f6d68aa0322777066f Mon Sep 17 00:00:00 2001 From: Grant Ingersoll Date: Wed, 1 Feb 2012 16:01:34 +0000 Subject: [PATCH] SOLR-1726: add deep paging support git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1239181 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 2 + .../handler/component/QueryComponent.java | 3 +- .../handler/component/ResponseBuilder.java | 17 ++++++- .../java/org/apache/solr/search/QParser.java | 27 ++++++++++- .../apache/solr/search/SolrIndexSearcher.java | 21 +++++++-- .../apache/solr/BasicFunctionalityTest.java | 45 +++++++++++++++++++ .../solr/common/params/CommonParams.java | 8 ++++ 7 files changed, 117 insertions(+), 6 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 8d333ae2a15..090211a8592 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -199,6 +199,8 @@ New Features * SOLR-3069: Ability to add openSearcher=false to not open a searcher when doing a hard commit. commitWithin now only invokes a softCommit. (yonik) +* SOLR-1726: Added deep paging support to search (sort by score only) which should use less memory when paging deeply into results + by keeping the priority queue small. (Manojkumar Rangasamy Kannadasan, gsingers) Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 32a9436f74c..6a68b6681f0 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -128,7 +128,8 @@ public class QueryComponent extends SearchComponent rb.setQuery( q ); rb.setSortSpec( parser.getSort(true) ); rb.setQparser(parser); - + rb.setScoreDoc(parser.getPaging()); + String[] fqs = req.getParams().getParams(CommonParams.FQ); if (fqs!=null && fqs.length!=0) { List filters = rb.getFilters(); diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java index cb55e2247a0..d798509e375 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java @@ -18,6 +18,7 @@ package org.apache.solr.handler.component; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.grouping.SearchGroup; import org.apache.lucene.search.grouping.TopGroups; @@ -69,6 +70,9 @@ public class ResponseBuilder private List filters = null; private SortSpec sortSpec = null; private GroupingSpecification groupingSpec; + //used for handling deep paging + private ScoreDoc scoreDoc; + private DocListAndSet results = null; private NamedList debugInfo = null; @@ -377,7 +381,8 @@ public class ResponseBuilder .setOffset(getSortSpec().getOffset()) .setLen(getSortSpec().getCount()) .setFlags(getFieldFlags()) - .setNeedDocSet(isNeedDocSet()); + .setNeedDocSet(isNeedDocSet()) + .setScoreDoc(getScoreDoc()); //Issue 1726 return cmd; } @@ -390,4 +395,14 @@ public class ResponseBuilder rsp.getResponseHeader().add("partialResults", Boolean.TRUE); } } + + public ScoreDoc getScoreDoc() + { + return scoreDoc; + } + + public void setScoreDoc(ScoreDoc scoreDoc) + { + this.scoreDoc = scoreDoc; + } } diff --git a/solr/core/src/java/org/apache/solr/search/QParser.java b/solr/core/src/java/org/apache/solr/search/QParser.java index 8b3cabce852..72bb18c971c 100755 --- a/solr/core/src/java/org/apache/solr/search/QParser.java +++ b/solr/core/src/java/org/apache/solr/search/QParser.java @@ -18,6 +18,7 @@ package org.apache.solr.search; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; //Issue 1726 import org.apache.lucene.search.Sort; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.MapSolrParams; @@ -209,7 +210,31 @@ public abstract class QParser { return nestedParser; } - + /** + * use common params to look up pageScore and pageDoc in global params + * @return the ScoreDoc + */ + public ScoreDoc getPaging() throws ParseException + { + String pageScoreS = null; + String pageDocS = null; + + pageScoreS = params.get(CommonParams.PAGESCORE); + pageDocS = params.get(CommonParams.PAGEDOC); + + if (pageScoreS == null || pageDocS == null) + return null; + + int pageDoc = pageDocS != null ? Integer.parseInt(pageDocS) : -1; + float pageScore = pageScoreS != null ? new Float(pageScoreS) : -1; + if(pageDoc != -1 && pageScore != -1){ + return new ScoreDoc(pageDoc, pageScore); + } + else { + return null; + } + } + /** * @param useGlobalParams look up sort, start, rows in global params if not in local params * @return the sort specification diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 3ae1bd8785f..d05f42dbfff 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -1358,7 +1358,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn } else { TopDocsCollector topCollector; if (cmd.getSort() == null) { - topCollector = TopScoreDocCollector.create(len, true); + if(cmd.getScoreDoc() != null) { + topCollector = TopScoreDocCollector.create(len, cmd.getScoreDoc(), true); //create the Collector with InOrderPagingCollector + } else { + topCollector = TopScoreDocCollector.create(len, true); + } + } else { topCollector = TopFieldCollector.create(weightSort(cmd.getSort()), len, false, needScores, needScores, true); } @@ -1382,7 +1387,6 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn TopDocs topDocs = topCollector.topDocs(0, len); maxScore = totalHits>0 ? topDocs.getMaxScore() : 0.0f; nDocsReturned = topDocs.scoreDocs.length; - ids = new int[nDocsReturned]; scores = (cmd.getFlags()&GET_SCORES)!=0 ? new float[nDocsReturned] : null; for (int i=0; i groupCommands; diff --git a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java index 6d56067697e..299ee484c42 100644 --- a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java +++ b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java @@ -33,6 +33,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LogMergePolicy; +import org.apache.lucene.util.English; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.MapSolrParams; @@ -761,6 +762,50 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { } + @Test + public void testDeepPaging() throws Exception { + for (int i = 0; i < 1000; i++){ + assertU(adoc("id", String.valueOf(i), "foo_t", English.intToEnglish(i))); + } + assertU(commit()); + SolrQueryRequest goldReq = null; + try { + goldReq = req("q", "foo_t:one", "rows", "50", "fl", "docid, score"); + SolrQueryResponse gold = h.queryAndResponse("standard", goldReq); + ResultContext response = (ResultContext) gold.getValues().get("response"); + assertQ("page: " + 0 + " failed", + req("q", "foo_t:one", "rows", "10", CommonParams.QT, "standard", "fl", "[docid], score"), + "*[count(//doc)=10]"); + //ugh, what a painful way to get the document + DocIterator iterator = response.docs.subset(9, 1).iterator(); + int lastDoc = iterator.nextDoc(); + float lastScore = iterator.score(); + for (int i = 1; i < 5; i++){ + //page through some results + DocList subset = response.docs.subset(i * 10, 1); + iterator = subset.iterator(); + int compareDoc = iterator.nextDoc(); + float compareScore = iterator.score(); + assertQ("page: " + i + " failed", + req("q", "foo_t:one", CommonParams.QT, "standard", "fl", "[docid], score", + "start", String.valueOf(i * 10), "rows", "1", //only get one doc, and then compare it to gold + CommonParams.PAGEDOC, String.valueOf(lastDoc), CommonParams.PAGESCORE, String.valueOf(lastScore)), + "*[count(//doc)=1]", + "//float[@name='score'][.='" + compareScore + "']", + "//int[@name='[docid]'][.='" + compareDoc + "']" + ); + lastScore = compareScore; + lastDoc = compareDoc; + + } + } finally { + if (goldReq != null ) { + goldReq.close(); + } + } + } + + // /** this doesn't work, but if it did, this is how we'd test it. */ // public void testOverwriteFalse() { diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java index bd047757de5..eafbbb41951 100755 --- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java @@ -46,6 +46,14 @@ public interface CommonParams { /** number of documents to return starting at "start" */ public static final String ROWS ="rows"; + //Issue 1726 start + /** score of the last document of the previous page */ + public static final String PAGESCORE ="pageScore"; + + /** docid of the last document of the previous page */ + public static final String PAGEDOC ="pageDoc"; + //Issue 1726 end + /** stylesheet to apply to XML results */ public static final String XSL ="xsl";