Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-5244: Exporting Full Sorted Result Sets 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1618224 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Joel Bernstein 2014-08-15 16:23:36 +00:00
parent 9b692b270b
commit 41a0e71185
3 changed files with 79 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -193,6 +193,8 @@ New Features
* SOLR-5656: Add autoAddReplicas feature for shared file systems. (Mark Miller, Gregory Chanan)
* SOLR-5244: Exporting Full Sorted Result Sets (Erik Hatcher, Joel Bernstein)
Bug Fixes
----------------------

59
solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
View File

@ -17,7 +17,6 @@
package org.apache.solr.response;
import com.carrotsearch.hppc.IntArrayList;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.MultiDocValues;
@ -48,15 +47,19 @@ import org.apache.solr.schema.StrField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SortSpec;
import org.apache.solr.search.SyntaxError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Writer;
import java.io.PrintWriter;
import java.net.SocketException;
import java.util.List;
public class SortingResponseWriter implements QueryResponseWriter {
private final static Logger logger = LoggerFactory.getLogger(SortingResponseWriter.class);
public void init(NamedList args) {
/* NOOP */
@ -157,16 +160,33 @@ public class SortingResponseWriter implements QueryResponseWriter {
count += (outDocsIndex+1);
for(int i=outDocsIndex; i>=0; --i) {
SortDoc s = outDocs[i];
if(commaNeeded){writer.write(',');}
writer.write('{');
writeDoc(s, leaves, fieldWriters, sets, writer);
writer.write('}');
commaNeeded = true;
s.reset();
try {
for(int i=outDocsIndex; i>=0; --i) {
SortDoc s = outDocs[i];
if(commaNeeded){writer.write(',');}
writer.write('{');
writeDoc(s, leaves, fieldWriters, sets, writer);
writer.write('}');
commaNeeded = true;
s.reset();
}
} catch(Throwable e) {
Throwable ex = e;
while(ex != null) {
String m = ex.getMessage();
if(m != null && m.contains("Broken pipe")) {
logger.info("Early client disconnect during export");
return;
}
ex = ex.getCause();
}
if(e instanceof IOException) {
throw ((IOException)e);
} else {
throw new IOException(e);
}
}
//total+=end-begin;
}
//System.out.println("Sort Time 2:"+Long.toString(total/1000000));
@ -240,6 +260,8 @@ public class SortingResponseWriter implements QueryResponseWriter {
} else {
writers[i] = new StringFieldWriter(field, fieldType);
}
} else {
throw new IOException("Export fields must either be one of the following types: int,float,long,double,string");
}
}
return writers;
@ -252,7 +274,13 @@ public class SortingResponseWriter implements QueryResponseWriter {
SortField sf = sortFields[i];
String field = sf.getField();
boolean reverse = sf.getReverse();
FieldType ft = schema.getField(field).getType();
SchemaField schemaField = schema.getField(field);
FieldType ft = schemaField.getType();
if(!schemaField.hasDocValues()) {
throw new IOException(field+" must have DocValues to use this feature.");
}
if(ft instanceof TrieIntField) {
if(reverse) {
sortValues[i] = new IntValue(field, new IntDesc());
@ -285,6 +313,8 @@ public class SortingResponseWriter implements QueryResponseWriter {
} else {
sortValues[i] = new StringValue(vals, field, new IntAsc());
}
} else {
throw new IOException("Sort fields must be one of the following types: int,float,long,double,string");
}
}
@ -1060,7 +1090,12 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
public void setCurrentValue(int docId) {
currentOrd = (int)globalOrds.get(currentVals.getOrd(docId));
int ord = currentVals.getOrd(docId);
if(ord < 0) {
currentOrd = -1;
} else {
currentOrd = (int)globalOrds.get(ord);
}
}
public void setCurrentValue(SortValue sv) {

33
solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
View File

@ -48,6 +48,18 @@ public class TestSortingResponseWriter extends SolrTestCaseJ4 {
"stringdv_m", "liverpool",
"stringdv_m", "Everton"));
assertU(adoc("id","7",
"floatdv","2.1",
"intdv", "7",
"longdv", "323223232323",
"doubledv","2344.345",
"floatdv_m", "123.321",
"floatdv_m", "345.123",
"doubledv_m", "3444.222",
"doubledv_m", "23232.2",
"longdv_m", "43434343434",
"longdv_m", "343332"));
assertU(commit());
assertU(adoc("id","2", "floatdv","2.1", "intdv", "2", "stringdv", "hello world", "longdv", "323223232323","doubledv","2344.344"));
assertU(commit());
@ -80,18 +92,34 @@ public class TestSortingResponseWriter extends SolrTestCaseJ4 {
String s = h.query(req("q", "id:1", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
assertEquals(s, "{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":1,\"stringdv\":\"hello world\",\"longdv\":323223232323,\"doubledv\":2344.345}]}");
//Test null value string:
s = h.query(req("q", "id:7", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
assertEquals(s, "{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":7,\"stringdv\":\"\",\"longdv\":323223232323,\"doubledv\":2344.345}]}");
//Test multiValue docValues output
s = h.query(req("q", "id:1", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
System.out.println(s);
assertEquals(s, "{\"numFound\":1, \"docs\":[{\"intdv_m\":[100,250],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[\"Everton\",\"liverpool\",\"manchester city\"]}]}");
//Test multiValues docValues output with nulls
s = h.query(req("q", "id:7", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
assertEquals(s, "{\"numFound\":1, \"docs\":[{\"intdv_m\":[],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[]}]}");
//Test single sort param is working
s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv desc"));
System.out.println("Output:"+s);
assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}");
s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv asc"));
assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}");
// Test sort on String will null value. Null value should sort last on desc and first on asc.
s = h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv desc"));
assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":7}]}");
s = h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv asc"));
assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":7},{\"intdv\":1}]}");
//Test multi-sort params
s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc"));
assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}");
@ -112,7 +140,6 @@ public class TestSortingResponseWriter extends SolrTestCaseJ4 {
assertEquals(s, "{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}");
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
System.out.println("Results:"+s);
assertEquals(s, "{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":1},{\"intdv\":2}]}");
}