SOLR-1726: add deep paging support

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1239181 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -199,6 +199,8 @@ New Features
 * SOLR-3069: Ability to add openSearcher=false to not open a searcher when doing
   a hard commit.  commitWithin now only invokes a softCommit. (yonik)
 
+* SOLR-1726: Added deep paging support to search (sort by score only) which should use less memory when paging deeply into results
+ by keeping the priority queue small. (Manojkumar Rangasamy Kannadasan, gsingers)
 
 Optimizations
 ----------------------

solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java

@@ -128,6 +128,7 @@ public class QueryComponent extends SearchComponent
       rb.setQuery( q );
       rb.setSortSpec( parser.getSort(true) );
       rb.setQparser(parser);
+      rb.setScoreDoc(parser.getPaging());
       
       String[] fqs = req.getParams().getParams(CommonParams.FQ);
       if (fqs!=null && fqs.length!=0) {

solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java

@@ -18,6 +18,7 @@
 package org.apache.solr.handler.component;
 
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.grouping.SearchGroup;
 import org.apache.lucene.search.grouping.TopGroups;
@@ -69,6 +70,9 @@ public class ResponseBuilder
   private List<Query> filters = null;
   private SortSpec sortSpec = null;
   private GroupingSpecification groupingSpec;
+  //used for handling deep paging
+  private ScoreDoc scoreDoc;
+
 
   private DocListAndSet results = null;
   private NamedList<Object> debugInfo = null;
@@ -377,7 +381,8 @@ public class ResponseBuilder
             .setOffset(getSortSpec().getOffset())
             .setLen(getSortSpec().getCount())
             .setFlags(getFieldFlags())
-            .setNeedDocSet(isNeedDocSet());
+            .setNeedDocSet(isNeedDocSet())
+            .setScoreDoc(getScoreDoc()); //Issue 1726
     return cmd;
   }
 
@@ -390,4 +395,14 @@ public class ResponseBuilder
       rsp.getResponseHeader().add("partialResults", Boolean.TRUE);
     }
   }
+
+  public ScoreDoc getScoreDoc()
+  {
+	  return scoreDoc;
+  }
+  
+  public void setScoreDoc(ScoreDoc scoreDoc)
+  {
+	  this.scoreDoc = scoreDoc;
+  }
 }

solr/core/src/java/org/apache/solr/search/QParser.java

@@ -18,6 +18,7 @@ package org.apache.solr.search;
 
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc; //Issue 1726
 import org.apache.lucene.search.Sort;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.MapSolrParams;
@@ -209,6 +210,30 @@ public abstract class QParser {
     return nestedParser;
   }
 
+  /**
+   * use common params to look up pageScore and pageDoc in global params
+   * @return the ScoreDoc
+   */
+  public ScoreDoc getPaging() throws ParseException
+  {
+	  String pageScoreS = null;
+	  String pageDocS = null;
+	  
+	  pageScoreS = params.get(CommonParams.PAGESCORE);
+	  pageDocS = params.get(CommonParams.PAGEDOC);
+		  
+	  if (pageScoreS == null || pageDocS == null)
+		  return null;
+	  
+	  int pageDoc = pageDocS != null ? Integer.parseInt(pageDocS) : -1;
+	  float pageScore = pageScoreS != null ? new Float(pageScoreS) : -1;
+	  if(pageDoc != -1 && pageScore != -1){
+      return new ScoreDoc(pageDoc, pageScore);
+    }
+	  else {
+      return null;
+    }
+  }
   
   /**
    * @param useGlobalParams look up sort, start, rows in global params if not in local params

solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java

@@ -1358,7 +1358,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
     } else {
       TopDocsCollector topCollector;
       if (cmd.getSort() == null) {
-        topCollector = TopScoreDocCollector.create(len, true);
+        if(cmd.getScoreDoc() != null) {
+        	topCollector = TopScoreDocCollector.create(len, cmd.getScoreDoc(), true); //create the Collector with InOrderPagingCollector
+        } else {
+          topCollector = TopScoreDocCollector.create(len, true);
+        }
+
       } else {
         topCollector = TopFieldCollector.create(weightSort(cmd.getSort()), len, false, needScores, needScores, true);
       }
@@ -1382,7 +1387,6 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
       TopDocs topDocs = topCollector.topDocs(0, len);
       maxScore = totalHits>0 ? topDocs.getMaxScore() : 0.0f;
       nDocsReturned = topDocs.scoreDocs.length;
-
       ids = new int[nDocsReturned];
       scores = (cmd.getFlags()&GET_SCORES)!=0 ? new float[nDocsReturned] : null;
       for (int i=0; i<nDocsReturned; i++) {
@@ -1392,7 +1396,6 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
       }
     }
 
-
     int sliceLen = Math.min(lastDocRequested,nDocsReturned);
     if (sliceLen < 0) sliceLen=0;
     qr.setDocList(new DocSlice(0,sliceLen,ids,scores,totalHits,maxScore));
@@ -2011,6 +2014,18 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
     private int supersetMaxDoc;
     private int flags;
     private long timeAllowed = -1;
+    //Issue 1726 start
+    private ScoreDoc scoreDoc;
+    
+    public ScoreDoc getScoreDoc()
+    {
+    	return scoreDoc;
+    }
+    public void setScoreDoc(ScoreDoc scoreDoc)
+    {
+    	this.scoreDoc = scoreDoc;
+    }
+    //Issue 1726 end
 
     // public List<Grouping.Command> groupCommands;
 

solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java

@@ -33,6 +33,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.util.English;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.MapSolrParams;
@@ -761,6 +762,50 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
   }
 
 
+  @Test
+  public void testDeepPaging() throws Exception {
+    for (int i = 0; i < 1000; i++){
+      assertU(adoc("id", String.valueOf(i),  "foo_t", English.intToEnglish(i)));
+    }
+    assertU(commit());
+    SolrQueryRequest goldReq = null;
+    try {
+      goldReq = req("q", "foo_t:one", "rows", "50", "fl", "docid, score");
+      SolrQueryResponse gold = h.queryAndResponse("standard", goldReq);
+      ResultContext response = (ResultContext) gold.getValues().get("response");
+      assertQ("page: " + 0 + " failed",
+          req("q", "foo_t:one", "rows", "10", CommonParams.QT, "standard", "fl", "[docid], score"),
+          "*[count(//doc)=10]");
+      //ugh, what a painful way to get the document
+      DocIterator iterator = response.docs.subset(9, 1).iterator();
+      int lastDoc = iterator.nextDoc();
+      float lastScore = iterator.score();
+      for (int i = 1; i < 5; i++){
+        //page through some results
+        DocList subset = response.docs.subset(i * 10, 1);
+        iterator = subset.iterator();
+        int compareDoc = iterator.nextDoc();
+        float compareScore = iterator.score();
+        assertQ("page: " + i + " failed",
+            req("q", "foo_t:one", CommonParams.QT, "standard", "fl", "[docid], score",
+                "start", String.valueOf(i * 10), "rows", "1",  //only get one doc, and then compare it to gold
+                CommonParams.PAGEDOC, String.valueOf(lastDoc), CommonParams.PAGESCORE, String.valueOf(lastScore)),
+            "*[count(//doc)=1]",
+            "//float[@name='score'][.='" + compareScore + "']",
+            "//int[@name='[docid]'][.='" + compareDoc + "']"
+        );
+        lastScore = compareScore;
+        lastDoc = compareDoc;
+
+      }
+    } finally {
+      if (goldReq != null ) {
+        goldReq.close();
+      }
+    }
+  }
+
+
 //   /** this doesn't work, but if it did, this is how we'd test it. */
 //   public void testOverwriteFalse() {
 

solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java

@@ -46,6 +46,14 @@ public interface CommonParams {
   /** number of documents to return starting at "start" */
   public static final String ROWS ="rows";
   
+  //Issue 1726 start
+  /** score of the last document of the previous page */
+  public static final String PAGESCORE ="pageScore";
+  
+  /** docid of the last document of the previous page */
+  public static final String PAGEDOC ="pageDoc";
+  //Issue 1726 end
+  
   /** stylesheet to apply to XML results */
   public static final String XSL ="xsl";