SOLR-5244: Exporting Full Sorted Result Sets

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1618068 13f79535-47bb-0310-9956-ffa450edef68

solr/core/src/java/org/apache/solr/core/SolrCore.java

@@ -64,6 +64,7 @@ import org.apache.solr.response.RawResponseWriter;
 import org.apache.solr.response.RubyResponseWriter;
 import org.apache.solr.response.SchemaXmlResponseWriter;
 import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.response.SortingResponseWriter;
 import org.apache.solr.response.XMLResponseWriter;
 import org.apache.solr.response.transform.TransformerFactory;
 import org.apache.solr.rest.ManagedResourceStorage;
@@ -2075,6 +2076,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
     m.put("raw", new RawResponseWriter());
     m.put("javabin", new BinaryResponseWriter());
     m.put("csv", new CSVResponseWriter());
+    m.put("xsort", new SortingResponseWriter());
     m.put("schema.xml", new SchemaXmlResponseWriter());
     DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m);
   }

solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java

@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import com.carrotsearch.hppc.IntArrayList;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.solr.handler.component.MergeStrategy;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.lucene.search.*;
+import org.apache.lucene.index.*;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.lucene.util.OpenBitSet;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+public class ExportQParserPlugin extends QParserPlugin {
+
+  public static final String NAME = "xport";
+  
+  public void init(NamedList namedList) {
+  }
+  
+  public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest request) {
+    return new ExportQParser(qstr, localParams, params, request);
+  }
+
+  public class ExportQParser extends QParser {
+    
+    public ExportQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest request) {
+      super(qstr, localParams, params, request);
+    }
+    
+    public Query parse() throws SyntaxError {
+      try {
+          return new ExportQuery(localParams, params, req);
+        } catch (Exception e) {
+          throw new SyntaxError(e.getMessage(), e);
+        }
+    }
+  }
+
+  public class ExportQuery extends RankQuery {
+    
+    private int leafCount;
+    private Query mainQuery;
+    private Object id;
+
+    public Query clone() {
+      ExportQuery clone = new ExportQuery();
+      clone.id = id;
+      clone.leafCount = leafCount;
+      clone.mainQuery = mainQuery;
+      return clone;
+    }
+
+    public RankQuery wrap(Query mainQuery) {
+      this.mainQuery = mainQuery;
+      return this;
+    }
+
+    public MergeStrategy getMergeStrategy() {
+      return null;
+    }
+
+    public Weight createWeight(IndexSearcher searcher) throws IOException {
+      return mainQuery.createWeight(searcher);
+    }
+
+    public Query rewrite(IndexReader reader) throws IOException {
+      return this.mainQuery.rewrite(reader);
+    }
+
+    public void extractTerms(Set<Term> terms) {
+      this.mainQuery.extractTerms(terms);
+    }
+
+    public TopDocsCollector getTopDocsCollector(int len,
+                                                SolrIndexSearcher.QueryCommand cmd,
+                                                IndexSearcher searcher) throws IOException {
+      FixedBitSet[] sets = new FixedBitSet[this.leafCount];
+      return new ExportCollector(sets);
+    }
+
+    public int hashCode() {
+      return id.hashCode()+((int)getBoost());
+    }
+    
+    public boolean equals(Object o) {
+      if(o instanceof ExportQuery) {
+        ExportQuery q = (ExportQuery)o;
+        return (this.id == q.id && getBoost() == q.getBoost());
+      } else {
+        return false;
+      }
+    }
+    
+    public String toString(String s) {
+      return s;
+    }
+
+    public ExportQuery() {
+
+    }
+    
+    public ExportQuery(SolrParams localParams, SolrParams params, SolrQueryRequest request) throws IOException {
+      this.leafCount = request.getSearcher().getTopReaderContext().leaves().size();
+      id = new Object();
+    }
+  }
+  
+  private class ExportCollector extends TopDocsCollector  {
+
+    private FixedBitSet[] sets;
+    private FixedBitSet set;
+
+    public ExportCollector(FixedBitSet[] sets) {
+      super(null);
+      this.sets = sets;
+    }
+    
+    public void doSetNextReader(AtomicReaderContext context) throws IOException {
+      this.set = new FixedBitSet(context.reader().maxDoc());
+      this.sets[context.ord] = set;
+
+    }
+    
+    public void collect(int docId) throws IOException{
+      ++totalHits;
+      set.set(docId);
+    }
+
+    private ScoreDoc[] getScoreDocs(int howMany) {
+      ScoreDoc[] docs = new ScoreDoc[howMany];
+      for(int i=0; i<docs.length; i++) {
+        docs[i] = new ScoreDoc(i,0);
+      }
+      return docs;
+    }
+
+    public TopDocs topDocs(int start, int howMany) {
+      SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
+      SolrQueryRequest req = null;
+      if(info != null && ((req = info.getReq()) != null)) {
+        Map context = req.getContext();
+        context.put("export", sets);
+        context.put("totalHits", totalHits);
+
+      }
+      return new TopDocs(totalHits, getScoreDocs(howMany), 0.0f);
+    }
+
+    public void setScorer(Scorer scorer) throws IOException {
+
+    }
+    
+    public boolean acceptsDocsOutOfOrder() {
+      return false;
+    }
+  }
+}

solr/core/src/java/org/apache/solr/search/QParserPlugin.java

@@ -62,7 +62,8 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI
     CollapsingQParserPlugin.NAME, CollapsingQParserPlugin.class,
     SimpleQParserPlugin.NAME, SimpleQParserPlugin.class,
     ComplexPhraseQParserPlugin.NAME, ComplexPhraseQParserPlugin.class,
-    ReRankQParserPlugin.NAME, ReRankQParserPlugin.class
+    ReRankQParserPlugin.NAME, ReRankQParserPlugin.class,
+    ExportQParserPlugin.NAME, ExportQParserPlugin.class
   };
 
   /** return a {@link QParser} */

solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml

@@ -0,0 +1,60 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="schema-docValuesMulti" version="1.5">
+  <types>
+
+    <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <!-- format for date is 1995-12-31T23:59:59.999Z and only the fractional
+         seconds part (.999) is optional.
+      -->
+    <fieldtype name="date" class="solr.TrieDateField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+
+    <fieldtype name="boolean" class="solr.BoolField" />
+    <fieldtype name="string" class="solr.StrField" />
+
+    <fieldType name="uuid" class="solr.UUIDField" />
+
+  </types>
+
+
+  <fields>
+    <field name="id" type="string" required="true" indexed="true" />
+    <field name="floatdv_m" type="float" indexed="false" stored="false" docValues="true" multiValued="true" />
+    <field name="intdv_m" type="int" indexed="false" stored="false" docValues="true" multiValued="true" />
+    <field name="doubledv_m" type="double" indexed="false" stored="false" docValues="true" multiValued="true" />
+    <field name="longdv_m" type="long" indexed="false" stored="false" docValues="true" multiValued="true" />
+    <field name="datedv_m" type="date" indexed="false" stored="false" docValues="true" multiValued="true" />
+    <field name="stringdv_m" type="string" indexed="false" stored="false" docValues="true" multiValued="true" />
+
+    <field name="floatdv" type="float" indexed="false" stored="false" docValues="true" />
+    <field name="intdv" type="int" indexed="false" stored="false" docValues="true"/>
+    <field name="doubledv" type="double" indexed="false" stored="false" docValues="true"/>
+    <field name="longdv" type="long" indexed="false" stored="false" docValues="true"/>
+    <field name="datedv" type="date" indexed="false" stored="false" docValues="true"/>
+    <field name="stringdv" type="string" indexed="false" stored="false" docValues="true"/>
+    
+    
+  </fields>
+
+  <uniqueKey>id</uniqueKey>
+
+</schema>

solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingresponse.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- a basic solrconfig that tests can use when they want simple minimal solrconfig/schema
+     DO NOT ADD THINGS TO THIS CONFIG! -->
+<config>
+  <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+  <dataDir>${solr.data.dir:}</dataDir>
+  <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
+  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
+  <requestHandler name="standard" class="solr.StandardRequestHandler" />
+  <requestHandler name="/update" class="solr.UpdateRequestHandler" />
+
+  <requestHandler name="/export" class="solr.SearchHandler">
+    <lst name="invariants">
+    <str name="rq">{!xport}</str>
+    <str name="wt">xsort</str>
+    <str name="distrib">false</str>
+  </lst>
+
+
+  <arr name="components">
+      <str>query</str>
+    </arr>
+  </requestHandler>
+
+</config>

solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java

@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.response;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.*;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+
+@SuppressCodecs({"Lucene3x", "Lucene40","Lucene41","Lucene42","Lucene45"})
+public class TestSortingResponseWriter extends SolrTestCaseJ4 {
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    initCore("solrconfig-sortingresponse.xml","schema-sortingresponse.xml");
+    createIndex();
+  }
+
+  public static void createIndex() {
+    assertU(adoc("id","1",
+                 "floatdv","2.1",
+                 "intdv", "1",
+                 "stringdv", "hello world",
+                 "longdv", "323223232323",
+                 "doubledv","2344.345",
+                 "intdv_m","100",
+                 "intdv_m","250",
+                 "floatdv_m", "123.321",
+                 "floatdv_m", "345.123",
+                 "doubledv_m", "3444.222",
+                 "doubledv_m", "23232.2",
+                 "longdv_m", "43434343434",
+                 "longdv_m", "343332",
+                 "stringdv_m", "manchester city",
+                 "stringdv_m", "liverpool",
+                 "stringdv_m", "Everton"));
+
+    assertU(commit());
+    assertU(adoc("id","2", "floatdv","2.1", "intdv", "2", "stringdv", "hello world", "longdv", "323223232323","doubledv","2344.344"));
+    assertU(commit());
+    assertU(adoc("id","3",
+        "floatdv","2.1",
+        "intdv", "3",
+        "stringdv", "chello world",
+        "longdv", "323223232323",
+        "doubledv","2344.346",
+        "intdv_m","100",
+        "intdv_m","250",
+        "floatdv_m", "123.321",
+        "floatdv_m", "345.123",
+        "doubledv_m", "3444.222",
+        "doubledv_m", "23232.2",
+        "longdv_m", "43434343434",
+        "longdv_m", "343332",
+        "stringdv_m", "manchester city",
+        "stringdv_m", "liverpool",
+        "stringdv_m", "everton"));
+    assertU(commit());
+
+
+  }
+
+  @Test
+  public void testSortingOutput() throws Exception {
+
+    //Test single value DocValue output
+    String s =  h.query(req("q", "id:1", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
+    assertEquals(s, "{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":1,\"stringdv\":\"hello world\",\"longdv\":323223232323,\"doubledv\":2344.345}]}");
+
+    //Test multiValue docValues output
+    s =  h.query(req("q", "id:1", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
+    System.out.println(s);
+    assertEquals(s, "{\"numFound\":1, \"docs\":[{\"intdv_m\":[100,250],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[\"Everton\",\"liverpool\",\"manchester city\"]}]}");
+    //Test single sort param is working
+    s =  h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv desc"));
+    System.out.println("Output:"+s);
+    assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}");
+
+    s =  h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv asc"));
+    assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}");
+
+    //Test multi-sort params
+    s =  h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc"));
+    assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}");
+
+    s =  h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "floatdv desc,intdv asc"));
+    assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}");
+
+    //Test three sort fields
+    s =  h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,stringdv asc,intdv desc"));
+    assertEquals(s, "{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}");
+
+    //Test three sort fields
+    s =  h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,stringdv desc,intdv asc"));
+    assertEquals(s, "{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}");
+
+    //Test four sort fields
+    s =  h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,floatdv desc,floatdv asc,intdv desc"));
+    assertEquals(s, "{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}");
+
+    s =  h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
+    System.out.println("Results:"+s);
+    assertEquals(s, "{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":1},{\"intdv\":2}]}");
+
+  }
+}

solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java

@@ -160,6 +160,15 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
     }
   }
 
+  public void testExportQuery() throws Exception {
+    SolrQueryRequest req = req("q", "*:*");
+    try {
+      assertQueryEquals("xport", req, "{!xport}");
+    } finally {
+      req.close();
+    }
+  }
+
   public void testQuerySwitch() throws Exception {
     SolrQueryRequest req = req("myXXX", "XXX", 
                                "myField", "foo_s",

solr/example/solr/collection1/conf/solrconfig.xml

@@ -912,6 +912,27 @@
      </lst>
   </requestHandler>
 
+  <!--
+    The export request handler is used to export full sorted result sets.
+    Do not change these defaults.
+  -->
+
+  <requestHandler name="/export" class="solr.SearchHandler">
+    <lst name="invariants">
+      <str name="rq">{!xport}</str>
+      <str name="wt">xsort</str>
+      <str name="distrib">false</str>
+    </lst>
+
+    <arr name="components">
+      <str>query</str>
+    </arr>
+  </requestHandler>
+
+
+
+
+
 
   <!-- A Robust Example