From caaa23a1b7672132b81e9a5058975598229eb227 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Fri, 25 Mar 2011 16:24:21 +0000 Subject: [PATCH] SOLR-1566: Transforming documents in the ResponseWriters. This will allow for more complex results in responses and open the door for function queries as results. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1085450 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 5 + .../org/apache/solr/common/SolrDocument.java | 5 + .../apache/solr/common/util/JavaBinCodec.java | 24 +- .../solr/handler/MoreLikeThisHandler.java | 8 +- .../handler/component/QueryComponent.java | 32 +- .../solr/response/BaseResponseWriter.java | 26 +- .../solr/response/BinaryResponseWriter.java | 108 ++++--- .../solr/response/CSVResponseWriter.java | 95 ++---- .../solr/response/JSONResponseWriter.java | 238 ++------------ .../response/PHPSerializedResponseWriter.java | 176 ++--------- .../apache/solr/response/ResultContext.java | 31 ++ .../apache/solr/response/ReturnFields.java | 168 ++++++++++ .../solr/response/SolrQueryResponse.java | 20 +- .../solr/response/TextResponseWriter.java | 134 ++++++-- .../org/apache/solr/response/XMLWriter.java | 298 +++--------------- .../response/transform/DocIdAugmenter.java | 42 +++ .../response/transform/DocTransformer.java | 33 ++ .../response/transform/DocTransformers.java | 62 ++++ .../transform/DocValuesAugmenter.java | 45 +++ .../response/transform/ExplainAugmenter.java | 55 ++++ .../response/transform/ScoreAugmenter.java | 44 +++ .../response/transform/TransformContext.java | 35 ++ .../transform/TransformerWithContext.java | 32 ++ .../response/transform/ValueAugmenter.java | 40 +++ .../org/apache/solr/util/SolrPluginUtils.java | 45 +-- .../apache/solr/BasicFunctionalityTest.java | 5 +- .../solr/client/solrj/SolrExampleTests.java | 90 ++++++ .../solr/response/TestCSVResponseWriter.java | 9 +- .../apache/solr/search/TestRangeQuery.java | 5 +- .../solrj/embedded/EmbeddedSolrServer.java | 28 +- 30 files changed, 1065 insertions(+), 873 deletions(-) create mode 100644 solr/src/java/org/apache/solr/response/ResultContext.java create mode 100644 solr/src/java/org/apache/solr/response/ReturnFields.java create mode 100644 solr/src/java/org/apache/solr/response/transform/DocIdAugmenter.java create mode 100644 solr/src/java/org/apache/solr/response/transform/DocTransformer.java create mode 100644 solr/src/java/org/apache/solr/response/transform/DocTransformers.java create mode 100644 solr/src/java/org/apache/solr/response/transform/DocValuesAugmenter.java create mode 100644 solr/src/java/org/apache/solr/response/transform/ExplainAugmenter.java create mode 100644 solr/src/java/org/apache/solr/response/transform/ScoreAugmenter.java create mode 100644 solr/src/java/org/apache/solr/response/transform/TransformContext.java create mode 100644 solr/src/java/org/apache/solr/response/transform/TransformerWithContext.java create mode 100644 solr/src/java/org/apache/solr/response/transform/ValueAugmenter.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index b5e3ebaf5f1..1168810c58e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -108,6 +108,11 @@ New Features Adding a parameter NOW= to the request will override the current time. (Peter Sturge, yonik) +* SOLR-1566: Transforming documents in the ResponseWriters. This will allow + for more complex results in responses and open the door for function queries + as results. (ryan) + + Optimizations ---------------------- diff --git a/solr/src/common/org/apache/solr/common/SolrDocument.java b/solr/src/common/org/apache/solr/common/SolrDocument.java index 813326e40f9..1864e09ea2a 100644 --- a/solr/src/common/org/apache/solr/common/SolrDocument.java +++ b/solr/src/common/org/apache/solr/common/SolrDocument.java @@ -26,6 +26,8 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; +import org.apache.solr.common.util.NamedList; + /** * A concrete representation of a document within a Solr index. Unlike a lucene @@ -88,6 +90,9 @@ public class SolrDocument implements Map, Iterable lst = new ArrayList(); for( Object o : (Iterable)value ) { diff --git a/solr/src/common/org/apache/solr/common/util/JavaBinCodec.java b/solr/src/common/org/apache/solr/common/util/JavaBinCodec.java index 2cb2104aecb..121820f16e1 100755 --- a/solr/src/common/org/apache/solr/common/util/JavaBinCodec.java +++ b/solr/src/common/org/apache/solr/common/util/JavaBinCodec.java @@ -295,27 +295,13 @@ public class JavaBinCodec { } public void writeSolrDocument(SolrDocument doc) throws IOException { - writeSolrDocument(doc, null); - } - - public void writeSolrDocument(SolrDocument doc, Set fields) throws IOException { - int count = 0; - if (fields == null) { - count = doc.getFieldNames().size(); - } else { - for (Map.Entry entry : doc) { - if (fields.contains(entry.getKey())) count++; - } - } writeTag(SOLRDOC); - writeTag(ORDERED_MAP, count); + writeTag(ORDERED_MAP, doc.size()); for (Map.Entry entry : doc) { - if (fields == null || fields.contains(entry.getKey())) { - String name = entry.getKey(); - writeExternString(name); - Object val = entry.getValue(); - writeVal(val); - } + String name = entry.getKey(); + writeExternString(name); + Object val = entry.getValue(); + writeVal(val); } } diff --git a/solr/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index fcd41e24dd9..31ee6c197b8 100644 --- a/solr/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -47,6 +47,7 @@ import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SimpleFacets; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.ReturnFields; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; @@ -77,10 +78,11 @@ public class MoreLikeThisHandler extends RequestHandlerBase SolrParams params = req.getParams(); // Set field flags - String fl = params.get(CommonParams.FL); + ReturnFields returnFields = ReturnFields.getReturnFields( req ); + rsp.setReturnFields( returnFields ); int flags = 0; - if (fl != null) { - flags |= SolrPluginUtils.setReturnFields(fl, rsp); + if (returnFields.getWantsScore()) { + flags |= SolrIndexSearcher.GET_SCORES; } String defType = params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE); diff --git a/solr/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/src/java/org/apache/solr/handler/component/QueryComponent.java index 10049937917..8695fd6afc4 100644 --- a/solr/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -39,6 +39,8 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.ResultContext; +import org.apache.solr.response.ReturnFields; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.SchemaField; @@ -72,13 +74,14 @@ public class QueryComponent extends SearchComponent } SolrQueryResponse rsp = rb.rsp; - // Set field flags - String fl = params.get(CommonParams.FL); - int fieldFlags = 0; - if (fl != null) { - fieldFlags |= SolrPluginUtils.setReturnFields(fl, rsp); + // Set field flags + ReturnFields returnFields = ReturnFields.getReturnFields( req ); + rsp.setReturnFields( returnFields ); + int flags = 0; + if (returnFields.getWantsScore()) { + flags |= SolrIndexSearcher.GET_SCORES; } - rb.setFieldFlags( fieldFlags ); + rb.setFieldFlags( flags ); String defType = params.get(QueryParsing.DEFTYPE,QParserPlugin.DEFAULT_QTYPE); @@ -294,7 +297,11 @@ public class QueryComponent extends SearchComponent res.docSet = searcher.getDocSet(queries); } rb.setResults(res); - rsp.add("response",rb.getResults().docList); + + ResultContext ctx = new ResultContext(); + ctx.docs = rb.getResults().docList; + ctx.query = null; // anything? + rsp.add("response", ctx); return; } @@ -416,7 +423,10 @@ public class QueryComponent extends SearchComponent // TODO: get "hits" a different way to log if (grouping.mainResult != null) { - rsp.add("response",grouping.mainResult); + ResultContext ctx = new ResultContext(); + ctx.docs = grouping.mainResult; + ctx.query = null; // TODO? add the query? + rsp.add("response", ctx); rsp.getToLog().add("hits", grouping.mainResult.matches()); } @@ -431,7 +441,11 @@ public class QueryComponent extends SearchComponent searcher.search(result,cmd); rb.setResult( result ); - rsp.add("response",rb.getResults().docList); + + ResultContext ctx = new ResultContext(); + ctx.docs = rb.getResults().docList; + ctx.query = rb.getQuery(); + rsp.add("response", ctx); rsp.getToLog().add("hits", rb.getResults().docList.matches()); doFieldSortValues(rb, searcher); diff --git a/solr/src/java/org/apache/solr/response/BaseResponseWriter.java b/solr/src/java/org/apache/solr/response/BaseResponseWriter.java index d86de764fee..ecdf900ea13 100644 --- a/solr/src/java/org/apache/solr/response/BaseResponseWriter.java +++ b/solr/src/java/org/apache/solr/response/BaseResponseWriter.java @@ -38,10 +38,14 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; -import java.util.Set; import java.util.ArrayList; /** + * THIS HAS NO TESTS and is not used anywhere.... no idea how or if it should work... + * + * I think we should drop it - along with {@link GenericBinaryResponseWriter} and {@link GenericBinaryResponseWriter} + * + * unless I'm missing something (ryan, March 2011) * * * This class serves as a basis from which {@link QueryResponseWriter}s can be @@ -60,7 +64,6 @@ public abstract class BaseResponseWriter { private static final Logger LOG = LoggerFactory .getLogger(BaseResponseWriter.class); - private static final String SCORE_FIELD = "score"; /** * @@ -110,9 +113,6 @@ public abstract class BaseResponseWriter { responseWriter.startDocumentList(name,info); for (int j = 0; j < sz; j++) { SolrDocument sdoc = getDoc(iterator.nextDoc(), idxInfo); - if (idxInfo.includeScore && docList.hasScores()) { - sdoc.addField(SCORE_FIELD, iterator.score()); - } responseWriter.writeDoc(sdoc); } } else { @@ -120,9 +120,6 @@ public abstract class BaseResponseWriter { .size()); for (int j = 0; j < sz; j++) { SolrDocument sdoc = getDoc(iterator.nextDoc(), idxInfo); - if (idxInfo.includeScore && docList.hasScores()) { - sdoc.addField(SCORE_FIELD, iterator.score()); - } list.add(sdoc); } responseWriter.writeAllDocs(info, list); @@ -144,22 +141,13 @@ public abstract class BaseResponseWriter { private static class IdxInfo { IndexSchema schema; SolrIndexSearcher searcher; - Set returnFields; - boolean includeScore; + ReturnFields returnFields; private IdxInfo(IndexSchema schema, SolrIndexSearcher searcher, - Set returnFields) { + ReturnFields returnFields) { this.schema = schema; this.searcher = searcher; - this.includeScore = returnFields != null - && returnFields.contains(SCORE_FIELD); - if (returnFields != null) { - if (returnFields.size() == 0 || (returnFields.size() == 1 && includeScore) || returnFields.contains("*")) { - returnFields = null; // null means return all stored fields - } - } this.returnFields = returnFields; - } } diff --git a/solr/src/java/org/apache/solr/response/BinaryResponseWriter.java b/solr/src/java/org/apache/solr/response/BinaryResponseWriter.java index 4dac4ab6ce5..0f2a6fe645e 100755 --- a/solr/src/java/org/apache/solr/response/BinaryResponseWriter.java +++ b/solr/src/java/org/apache/solr/response/BinaryResponseWriter.java @@ -19,12 +19,14 @@ package org.apache.solr.response; import org.apache.lucene.document.Document; import org.apache.lucene.document.Fieldable; import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.transform.DocTransformer; +import org.apache.solr.response.transform.TransformContext; import org.apache.solr.schema.*; -import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.search.SolrIndexSearcher; import org.slf4j.Logger; @@ -62,79 +64,101 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter { protected final SolrQueryRequest solrQueryRequest; protected IndexSchema schema; protected SolrIndexSearcher searcher; - protected final Set returnFields; - protected final boolean includeScore; + protected final ReturnFields returnFields; // transmit field values using FieldType.toObject() // rather than the String from FieldType.toExternal() boolean useFieldObjects = true; - public Resolver(SolrQueryRequest req, Set returnFields) { + public Resolver(SolrQueryRequest req, ReturnFields returnFields) { solrQueryRequest = req; - this.includeScore = returnFields != null && returnFields.contains("score"); - - if (returnFields != null) { - if (returnFields.size() == 0 || (returnFields.size() == 1 && includeScore) || returnFields.contains("*")) { - returnFields = null; // null means return all stored fields - } - } this.returnFields = returnFields; } public Object resolve(Object o, JavaBinCodec codec) throws IOException { - if (o instanceof DocList) { - writeDocList((DocList) o, codec); + if (o instanceof ResultContext) { + writeResults((ResultContext) o, codec); return null; // null means we completely handled it } - if (o instanceof SolrDocument) { - SolrDocument solrDocument = (SolrDocument) o; - codec.writeSolrDocument(solrDocument, returnFields); - return null; - } - if (o instanceof Document) { - return getDoc((Document) o); + if (o instanceof DocList) { + ResultContext ctx = new ResultContext(); + ctx.docs = (DocList) o; + writeResults(ctx, codec); + return null; // null means we completely handled it } + if (o instanceof SolrDocument) { + // Remove any fields that were not requested + // This typically happens when distributed search adds extra fields to an internal request + SolrDocument doc = (SolrDocument)o; + for( String fname : doc.getFieldNames() ) { + if( !returnFields.contains( fname ) ) { + doc.removeFields( fname ); + } + } + return doc; + } return o; } - public void writeDocList(DocList ids, JavaBinCodec codec) throws IOException { - codec.writeTag(JavaBinCodec.SOLRDOCLST); - List l = new ArrayList(3); - l.add((long) ids.matches()); - l.add((long) ids.offset()); - Float maxScore = null; - if (includeScore && ids.hasScores()) { - maxScore = ids.maxScore(); - } - l.add(maxScore); - codec.writeArray(l); - + protected void writeResultsBody( ResultContext res, JavaBinCodec codec ) throws IOException + { + DocList ids = res.docs; + TransformContext context = new TransformContext(); + context.query = res.query; + context.wantsScores = returnFields.getWantsScore() && ids.hasScores(); + int sz = ids.size(); codec.writeTag(JavaBinCodec.ARR, sz); if(searcher == null) searcher = solrQueryRequest.getSearcher(); if(schema == null) schema = solrQueryRequest.getSchema(); - DocIterator iterator = ids.iterator(); + + context.searcher = searcher; + DocTransformer transformer = returnFields.getTransformer(); + if( transformer != null ) { + transformer.setContext( context ); + } + + Set fnames = returnFields.getFieldNames(); + context.iterator = ids.iterator(); for (int i = 0; i < sz; i++) { - int id = iterator.nextDoc(); - Document doc = searcher.doc(id, returnFields); - + int id = context.iterator.nextDoc(); + Document doc = searcher.doc(id, fnames); SolrDocument sdoc = getDoc(doc); - - if (includeScore && ids.hasScores()) { - sdoc.addField("score", iterator.score()); + if( transformer != null ) { + transformer.transform(sdoc, id ); } - codec.writeSolrDocument(sdoc); } + if( transformer != null ) { + transformer.setContext( null ); + } + } + + public void writeResults(ResultContext ctx, JavaBinCodec codec) throws IOException { + codec.writeTag(JavaBinCodec.SOLRDOCLST); + boolean wantsScores = returnFields.getWantsScore() && ctx.docs.hasScores(); + List l = new ArrayList(3); + l.add((long) ctx.docs.matches()); + l.add((long) ctx.docs.offset()); + + Float maxScore = null; + if (wantsScores) { + maxScore = ctx.docs.maxScore(); + } + l.add(maxScore); + codec.writeArray(l); + + // this is a seprate function so that streaming responses can use just that part + writeResultsBody( ctx, codec ); } - public SolrDocument getDoc(Document doc) { SolrDocument solrDoc = new SolrDocument(); for (Fieldable f : doc.getFields()) { String fieldName = f.name(); - if (returnFields != null && !returnFields.contains(fieldName)) continue; + if( !returnFields.contains(fieldName) ) + continue; SchemaField sf = schema.getFieldOrNull(fieldName); FieldType ft = null; if(sf != null) ft =sf.getType(); diff --git a/solr/src/java/org/apache/solr/response/CSVResponseWriter.java b/solr/src/java/org/apache/solr/response/CSVResponseWriter.java index 2d14c804720..9ab9c37dfd9 100755 --- a/solr/src/java/org/apache/solr/response/CSVResponseWriter.java +++ b/solr/src/java/org/apache/solr/response/CSVResponseWriter.java @@ -159,7 +159,6 @@ class CSVWriter extends TextResponseWriter { ResettableFastWriter mvWriter = new ResettableFastWriter(); // writer used for multi-valued fields String NullValue; - boolean returnScore = false; public CSVWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) { @@ -236,12 +235,9 @@ class CSVWriter extends TextResponseWriter { // encapsulator will already be disabled if it wasn't specified } - returnScore = returnFields != null && returnFields.contains("score"); - boolean needListOfFields = returnFields==null || returnFields.size()==0 || (returnFields.size()==1 && returnScore) || returnFields.contains("*"); - Collection fields = returnFields; - + Collection fields = returnFields.getFieldNames(); Object responseObj = rsp.getValues().get("response"); - if (needListOfFields) { + if (fields==null) { if (responseObj instanceof SolrDocumentList) { // get the list of fields from the SolrDocumentList fields = new LinkedHashSet(); @@ -252,7 +248,7 @@ class CSVWriter extends TextResponseWriter { // get the list of fields from the index fields = req.getSearcher().getFieldNames(); } - if (returnScore) { + if (returnFields.getWantsScore()) { fields.add("score"); } else { fields.remove("score"); @@ -327,11 +323,15 @@ class CSVWriter extends TextResponseWriter { printer.println(); } - - if (responseObj instanceof DocList) { - writeDocList(null, (DocList)responseObj, null, null); + if (responseObj instanceof ResultContext ) { + writeDocuments(null, (ResultContext)responseObj, returnFields ); + } + else if (responseObj instanceof DocList) { + ResultContext ctx = new ResultContext(); + ctx.docs = (DocList)responseObj; + writeDocuments(null, ctx, returnFields ); } else if (responseObj instanceof SolrDocumentList) { - writeSolrDocumentList(null, (SolrDocumentList)responseObj, null, null); + writeSolrDocumentList(null, (SolrDocumentList)responseObj, returnFields ); } } @@ -346,56 +346,21 @@ class CSVWriter extends TextResponseWriter { public void writeNamedList(String name, NamedList val) throws IOException { } - @Override - public void writeDoc(String name, Document doc, Set returnFields, float score, boolean includeScore) throws IOException { - pass++; + public void writeStartDocumentList(String name, + long start, int size, long numFound, Float maxScore) throws IOException + { + // nothing + } - for (Fieldable field: doc.getFields()) { - CSVField csvField = csvFields.get(field.name()); - if (csvField == null) continue; - if (csvField.tmp != pass) { - csvField.tmp = pass; - csvField.values.clear(); - } - csvField.values.add(field); - } - - for (CSVField csvField : csvFields.values()) { - if (csvField.name.equals("score")) { - writeFloat("score", score); - continue; - } - if (csvField.tmp != pass) { - writeNull(csvField.name); - continue; - } - - if (csvField.sf.multiValued() || csvField.values.size() > 1) { - mvWriter.reset(); - csvField.mvPrinter.reset(); - // switch the printer to use the multi-valued one - CSVPrinter tmp = printer; - printer = csvField.mvPrinter; - for (Fieldable fval : csvField.values) { - csvField.sf.getType().write(this, csvField.name, fval); - } - printer = tmp; // restore the original printer - - mvWriter.freeze(); - printer.print(mvWriter.getFrozenBuf(), 0, mvWriter.getFrozenSize(), true); - } else { - assert csvField.values.size() == 1; - csvField.sf.getType().write(this,csvField.name,csvField.values.get(0)); - } - } - - printer.println(); + public void writeEndDocumentList() throws IOException + { + // nothing } //NOTE: a document cannot currently contain another document List tmpList; @Override - public void writeSolrDocument(String name, SolrDocument doc, Set returnFields, Map pseudoFields) throws IOException { + public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx ) throws IOException { if (tmpList == null) { tmpList = new ArrayList(1); tmpList.add(null); @@ -445,26 +410,6 @@ class CSVWriter extends TextResponseWriter { printer.println(); } - @Override - public void writeDocList(String name, DocList ids, Set fields, Map otherFields) throws IOException { - int sz=ids.size(); - SolrIndexSearcher searcher = req.getSearcher(); - DocIterator iterator = ids.iterator(); - for (int i=0; i fields, Map otherFields) throws IOException { - for (SolrDocument doc : docs) { - writeSolrDocument(name, doc, fields, otherFields); - } - } - @Override public void writeStr(String name, String val, boolean needsEscaping) throws IOException { printer.print(val, needsEscaping); diff --git a/solr/src/java/org/apache/solr/response/JSONResponseWriter.java b/solr/src/java/org/apache/solr/response/JSONResponseWriter.java index 48e373f5e7e..d2f1d8753d6 100644 --- a/solr/src/java/org/apache/solr/response/JSONResponseWriter.java +++ b/solr/src/java/org/apache/solr/response/JSONResponseWriter.java @@ -17,24 +17,23 @@ package org.apache.solr.response; -import org.apache.lucene.document.Document; +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + import org.apache.lucene.document.Fieldable; import org.apache.lucene.util.StringHelper; import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.SchemaField; -import org.apache.solr.schema.TextField; -import org.apache.solr.search.DocIterator; -import org.apache.solr.search.DocList; -import org.apache.solr.search.SolrIndexSearcher; - -import java.io.IOException; -import java.io.Writer; -import java.util.*; /** * @version $Id$ @@ -73,7 +72,6 @@ class JSONWriter extends TextResponseWriter { private static final String JSON_NL_ARROFMAP="arrmap"; private static final String JSON_WRAPPER_FUNCTION="json.wrf"; - public JSONWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) { super(writer, req, rsp); namedListStyle = StringHelper.intern(req.getParams().get(JSON_NL_STYLE, JSON_NL_FLAT)); @@ -312,94 +310,21 @@ class JSONWriter extends TextResponseWriter { } } - public void writeDoc(String name, Collection fields, Set returnFields, Map pseudoFields) throws IOException { - writeMapOpener(-1); // no trivial way to determine map size - incLevel(); - - HashMap multi = new HashMap(); - - boolean first=true; - - for (Fieldable ff : fields) { - String fname = ff.name(); - if (returnFields!=null && !returnFields.contains(fname)) { - continue; - } - - // if the field is multivalued, it may have other values further on... so - // build up a list for each multi-valued field. - SchemaField sf = schema.getField(fname); - if (sf.multiValued()) { - MultiValueField mf = multi.get(fname); - if (mf==null) { - mf = new MultiValueField(sf, ff); - multi.put(fname, mf); - } else { - mf.fields.add(ff); - } - } else { - // not multi-valued, so write it immediately. - if (first) { - first=false; - } else { - writeMapSeparator(); - } - indent(); - writeKey(fname,true); - sf.write(this, fname, ff); - } - } - - for(MultiValueField mvf : multi.values()) { - if (first) { - first=false; - } else { - writeMapSeparator(); - } - - indent(); - writeKey(mvf.sfield.getName(), true); - - boolean indentArrElems=false; - if (doIndent) { - // heuristic... TextField is probably the only field type likely to be long enough - // to warrant indenting individual values. - indentArrElems = (mvf.sfield.getType() instanceof TextField); - } - - writeArrayOpener(-1); // no trivial way to determine array size - boolean firstArrElem=true; - incLevel(); - - for (Fieldable ff : mvf.fields) { - if (firstArrElem) { - firstArrElem=false; - } else { - writeArraySeparator(); - } - if (indentArrElems) indent(); - mvf.sfield.write(this, null, ff); - } - writeArrayCloser(); - decLevel(); - } - - if (pseudoFields !=null && pseudoFields.size()>0) { - writeMap(null,pseudoFields,true,first); - } - - decLevel(); - writeMapCloser(); - } - @Override - public void writeSolrDocument(String name, SolrDocument doc, Set returnFields, Map pseudoFields) throws IOException { - writeMapOpener(-1); // no trivial way to determine map size - // TODO: could easily figure out size for SolrDocument if needed... + public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx) throws IOException { + if( idx > 0 ) { + writeArraySeparator(); + } + + writeMapOpener(doc.size()); incLevel(); boolean first=true; for (String fname : doc.getFieldNames()) { + if (!returnFields.contains(fname)) { + continue; + } + if (first) { first=false; } @@ -424,144 +349,43 @@ class JSONWriter extends TextResponseWriter { writeVal(fname, val); } } - - if (pseudoFields !=null && pseudoFields.size()>0) { - writeMap(null,pseudoFields,true,first); - } } - + decLevel(); writeMapCloser(); } - - // reusable map to store the "score" pseudo-field. - // if a Doc can ever contain another doc, this optimization would have to go. - private final HashMap scoreMap = new HashMap(1); - @Override - public void writeDoc(String name, Document doc, Set returnFields, float score, boolean includeScore) throws IOException { - Map other = null; - if (includeScore) { - other = scoreMap; - scoreMap.put("score",score); - } - writeDoc(name, doc.getFields(), returnFields, other); - } - - @Override - public void writeDocList(String name, DocList ids, Set fields, Map otherFields) throws IOException { - boolean includeScore=false; - if (fields!=null) { - includeScore = fields.contains("score"); - if (fields.size()==0 || (fields.size()==1 && includeScore) || fields.contains("*")) { - fields=null; // null means return all stored fields - } - } - - int sz=ids.size(); - - writeMapOpener(includeScore ? 4 : 3); + public void writeStartDocumentList(String name, + long start, int size, long numFound, Float maxScore) throws IOException + { + writeMapOpener((maxScore==null) ? 3 : 4); incLevel(); writeKey("numFound",false); - writeInt(null,ids.matches()); + writeLong(null,numFound); writeMapSeparator(); writeKey("start",false); - writeInt(null,ids.offset()); + writeLong(null,start); - if (includeScore) { + if (maxScore!=null) { writeMapSeparator(); writeKey("maxScore",false); - writeFloat(null,ids.maxScore()); + writeFloat(null,maxScore); } writeMapSeparator(); // indent(); writeKey("docs",false); - writeArrayOpener(sz); + writeArrayOpener(size); incLevel(); - boolean first=true; - - SolrIndexSearcher searcher = req.getSearcher(); - // be defensive... write out the doc even if we don't have the scores like we should - includeScore = includeScore && ids.hasScores(); - DocIterator iterator = ids.iterator(); - for (int i=0; i fields, Map otherFields) throws IOException { - boolean includeScore=false; - if (fields!=null) { - includeScore = fields.contains("score"); - if (fields.size()==0 || (fields.size()==1 && includeScore) || fields.contains("*")) { - fields=null; // null means return all stored fields - } - } - - int sz=docs.size(); - - writeMapOpener(includeScore ? 4 : 3); - incLevel(); - writeKey("numFound",false); - writeLong(null,docs.getNumFound()); - writeMapSeparator(); - writeKey("start",false); - writeLong(null,docs.getStart()); - - if (includeScore && docs.getMaxScore() != null) { - writeMapSeparator(); - writeKey("maxScore",false); - writeFloat(null,docs.getMaxScore()); - } - writeMapSeparator(); - // indent(); - writeKey("docs",false); - writeArrayOpener(sz); - - incLevel(); - boolean first=true; - - SolrIndexSearcher searcher = req.getSearcher(); - for (SolrDocument doc : docs) { - - if (first) { - first=false; - } else { - writeArraySeparator(); - } - indent(); - writeSolrDocument(null, doc, fields, otherFields); - } + public void writeEndDocumentList() throws IOException + { decLevel(); writeArrayCloser(); - if (otherFields !=null) { - writeMap(null, otherFields, true, false); - } - decLevel(); indent(); writeMapCloser(); diff --git a/solr/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java b/solr/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java index da832ebead9..11f9ee0ee1a 100755 --- a/solr/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java +++ b/solr/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java @@ -17,23 +17,22 @@ package org.apache.solr.response; -import java.io.Writer; import java.io.IOException; -import java.util.*; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashMap; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Fieldable; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.UnicodeUtil; +import org.apache.solr.common.SolrDocument; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.SchemaField; -import org.apache.solr.search.DocIterator; -import org.apache.solr.search.DocList; -import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; + + /** * A description of the PHP serialization format can be found here: * http://www.hurring.com/scott/code/perl/serialize/ @@ -80,126 +79,53 @@ class PHPSerializedWriter extends JSONWriter { writeNamedListAsMapMangled(name,val); } - @Override - public void writeDoc(String name, Collection fields, Set returnFields, Map pseudoFields) throws IOException { - ArrayList single = new ArrayList(); - LinkedHashMap multi - = new LinkedHashMap(); - - for (Fieldable ff : fields) { - String fname = ff.name(); - if (returnFields!=null && !returnFields.contains(fname)) { - continue; - } - // if the field is multivalued, it may have other values further on... so - // build up a list for each multi-valued field. - SchemaField sf = schema.getField(fname); - if (sf.multiValued()) { - MultiValueField mf = multi.get(fname); - if (mf==null) { - mf = new MultiValueField(sf, ff); - multi.put(fname, mf); - } else { - mf.fields.add(ff); - } - } else { - single.add(ff); - } - } - - // obtain number of fields in doc - writeArrayOpener(single.size() + multi.size() + ((pseudoFields!=null) ? pseudoFields.size() : 0)); - - // output single value fields - for(Fieldable ff : single) { - SchemaField sf = schema.getField(ff.name()); - writeKey(ff.name(),true); - sf.write(this, ff.name(), ff); - } - - // output multi value fields - for(MultiValueField mvf : multi.values()) { - writeKey(mvf.sfield.getName(), true); - writeArrayOpener(mvf.fields.size()); - int i = 0; - for (Fieldable ff : mvf.fields) { - writeKey(i++, false); - mvf.sfield.write(this, null, ff); - } - writeArrayCloser(); - } - - // output pseudo fields - if (pseudoFields !=null && pseudoFields.size()>0) { - writeMap(null,pseudoFields,true,false); - } - writeArrayCloser(); - } - @Override - public void writeDocList(String name, DocList ids, Set fields, Map otherFields) throws IOException { - boolean includeScore=false; - - if (fields!=null) { - includeScore = fields.contains("score"); - if (fields.size()==0 || (fields.size()==1 && includeScore) || fields.contains("*")) { - fields=null; // null means return all stored fields - } - } - int sz=ids.size(); - - writeMapOpener(includeScore ? 4 : 3); + public void writeStartDocumentList(String name, + long start, int size, long numFound, Float maxScore) throws IOException + { + writeMapOpener((maxScore==null) ? 3 : 4); writeKey("numFound",false); - writeInt(null,ids.matches()); + writeLong(null,numFound); writeKey("start",false); - writeInt(null,ids.offset()); + writeLong(null,start); - if (includeScore) { + if (maxScore!=null) { writeKey("maxScore",false); - writeFloat(null,ids.maxScore()); + writeFloat(null,maxScore); } writeKey("docs",false); - writeArrayOpener(sz); - - SolrIndexSearcher searcher = req.getSearcher(); - DocIterator iterator = ids.iterator(); - for (int i=0; i returnFields, Map pseudoFields) throws IOException { + public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx) throws IOException + { + writeKey(idx, false); + LinkedHashMap single = new LinkedHashMap(); LinkedHashMap multi = new LinkedHashMap(); - int pseudoSize = pseudoFields != null ? pseudoFields.size() : 0; for (String fname : doc.getFieldNames()) { - if(returnFields != null && !returnFields.contains(fname)){ + if(!returnFields.contains(fname)){ continue; } Object val = doc.getFieldValue(fname); - SchemaField sf = schema.getFieldOrNull(fname); - if (sf != null && sf.multiValued()) { + if (val instanceof Collection) { multi.put(fname, val); }else{ single.put(fname, val); } } - writeMapOpener(single.size() + multi.size() + pseudoSize); + writeMapOpener(single.size() + multi.size()); for(String fname: single.keySet()){ Object val = single.get(fname); writeKey(fname, true); @@ -220,51 +146,7 @@ class PHPSerializedWriter extends JSONWriter { writeVal(fname, val); } } - - if (pseudoSize > 0) { - writeMap(null,pseudoFields,true, false); - } - writeMapCloser(); - } - - - @Override - public void writeSolrDocumentList(String name, SolrDocumentList docs, Set fields, Map otherFields) throws IOException { - boolean includeScore=false; - if (fields!=null) { - includeScore = fields.contains("score"); - if (fields.size()==0 || (fields.size()==1 && includeScore) || fields.contains("*")) { - fields=null; // null means return all stored fields - } - } - - int sz = docs.size(); - - writeMapOpener(includeScore ? 4 : 3); - - writeKey("numFound",false); - writeLong(null,docs.getNumFound()); - - writeKey("start",false); - writeLong(null,docs.getStart()); - - if (includeScore && docs.getMaxScore() != null) { - writeKey("maxScore",false); - writeFloat(null,docs.getMaxScore()); - } - - writeKey("docs",false); - - writeArrayOpener(sz); - for (int i=0; i fields; // includes 'augment' names or null + private DocTransformer transformer; + private boolean wantsScore = false; + + + public static ReturnFields getReturnFields(SolrQueryRequest req) + { + return getReturnFields( req.getParams().get(CommonParams.FL), req ); + } + + public static ReturnFields getReturnFields(String fl, SolrQueryRequest req) + { + ReturnFields rf = new ReturnFields(); + rf.wantsScore = false; + rf.fields = new LinkedHashSet(); // order is important for CSVResponseWriter + boolean allFields = false; + + DocTransformers augmenters = new DocTransformers(); + if (fl != null) { + // TODO - this could become more efficient if widely used. + String[] flst = SolrPluginUtils.split(fl); + if (flst.length > 0 && !(flst.length==1 && flst[0].length()==0)) { + IndexSchema schema = req.getSchema(); + for (String name : flst) { + if( "*".equals( name ) ) { + allFields = true; + } + else if( SCORE.equals( name ) ) { + rf.fields.add( name ); + rf.wantsScore = true; + augmenters.addTransformer( new ScoreAugmenter( SCORE ) ); + } + else { + rf.fields.add( name ); + + // Check if it is a real score + SchemaField sf = schema.getFieldOrNull( name ); + if( sf == null ) { + // not a field name, but possibly return value + if( DOCID.equals( name ) ) { + augmenters.addTransformer( new DocIdAugmenter( DOCID ) ); + } + else if( SHARD.equals( name ) ) { + String id = "getshardid???"; + augmenters.addTransformer( new ValueAugmenter( SHARD, id ) ); + } + else if( EXPLAIN.equals( name ) ) { + augmenters.addTransformer( new ExplainAugmenter( EXPLAIN ) ); + } + else if( name.startsWith( "{!func}") ) { + // help? not sure how to parse a ValueSorce + // -- not to mention, we probably want to reuse existing ones! + augmenters.addTransformer( new ValueAugmenter( name, "TODO:"+name ) ); +// try { +// String func = name.substring( "{!func}".length() ); +// SolrParams local = null; +// FunctionQParser p = new FunctionQParser( func, local, req.getParams(), req ); +// Query q = p.parse(); +// ValueSource vs = p.parseValueSource(); +// AtomicReaderContext ctx = new AtomicReaderContext( req.getSearcher().getIndexReader() ); +// Map mmm = null; // ????? +// DocValues values = p.parseValueSource().getValues( mmm, ctx ); +// augmenters.addAugmenter( new DocValuesAugmenter( name, values ) ); +// } +// catch( Exception ex ) { +// throw new SolrException( org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST, +// "Unable to parse augmented field: "+name, ex ); +// } + } + else { + // maybe throw an exception? +// throw new SolrException( org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST, +// "Unknown Return Field: "+name ); + } + } + } + } + } + } + + // Legacy behavior? "score" == "*,score" + if( rf.fields.size() == 1 && rf.wantsScore ) { + allFields = true; + } + + if( allFields || rf.fields.isEmpty() ) { + rf.fields = null; + } + + if( augmenters.size() == 1 ) { + rf.transformer = augmenters.getTransformer(0); + } + else if( augmenters.size() > 1 ) { + rf.transformer = augmenters; + } + return rf; + } + + public Set getFieldNames() + { + return fields; + } + + public boolean getWantsScore() + { + return wantsScore; + } + + public boolean contains( String name ) + { + return fields==null || fields.contains( name ); + } + + public DocTransformer getTransformer() + { + return transformer; + } +} diff --git a/solr/src/java/org/apache/solr/response/SolrQueryResponse.java b/solr/src/java/org/apache/solr/response/SolrQueryResponse.java index bbb2f1cd530..88de3cdbc35 100644 --- a/solr/src/java/org/apache/solr/response/SolrQueryResponse.java +++ b/solr/src/java/org/apache/solr/response/SolrQueryResponse.java @@ -67,13 +67,14 @@ public class SolrQueryResponse { * @see Note on Returnable Data */ protected NamedList values = new SimpleOrderedMap(); - - /** + + +/** * Container for storing information that should be logged by Solr before returning. */ protected NamedList toLog = new SimpleOrderedMap(); - protected Set defaultReturnFields; + protected ReturnFields returnFields; // error if this is set... protected Exception err; @@ -111,18 +112,19 @@ public class SolrQueryResponse { * Sets the document field names of fields to return by default when * returning DocLists */ - public void setReturnFields(Set fields) { - defaultReturnFields=fields; + public void setReturnFields(ReturnFields fields) { + returnFields=fields; } - // TODO: should this be represented as a String[] such - // that order can be maintained if needed? /** * Gets the document field names of fields to return by default when * returning DocLists */ - public Set getReturnFields() { - return defaultReturnFields; + public ReturnFields getReturnFields() { + if( returnFields == null ) { + returnFields = new ReturnFields(); // by default return everything + } + return returnFields; } diff --git a/solr/src/java/org/apache/solr/response/TextResponseWriter.java b/solr/src/java/org/apache/solr/response/TextResponseWriter.java index d65f1ebd54a..e06185ee8d7 100644 --- a/solr/src/java/org/apache/solr/response/TextResponseWriter.java +++ b/solr/src/java/org/apache/solr/response/TextResponseWriter.java @@ -17,18 +17,23 @@ package org.apache.solr.response; -import org.apache.lucene.document.Document; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.FastWriter; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.search.DocList; import java.io.IOException; import java.io.Writer; import java.util.*; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Fieldable; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.util.FastWriter; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.transform.DocTransformer; +import org.apache.solr.response.transform.TransformContext; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.DocList; + /** Base class for text-oriented response writers. * * @version $Id$ @@ -49,7 +54,7 @@ public abstract class TextResponseWriter { protected final SolrQueryResponse rsp; // the default set of fields to return for each document - protected Set returnFields; + protected ReturnFields returnFields; protected int level; protected boolean doIndent; @@ -114,6 +119,15 @@ public abstract class TextResponseWriter { } else if (val instanceof String) { writeStr(name, val.toString(), true); // micro-optimization... using toString() avoids a cast first + } else if (val instanceof Fieldable) { + Fieldable f = (Fieldable)val; + SchemaField sf = schema.getFieldOrNull( f.name() ); + if( sf != null ) { + sf.getType().write(this, name, f); + } + else { + writeStr(name, f.stringValue(), true); + } } else if (val instanceof Integer) { writeInt(name, val.toString()); } else if (val instanceof Boolean) { @@ -129,19 +143,25 @@ public abstract class TextResponseWriter { } else if (val instanceof Double) { writeDouble(name, ((Double)val).doubleValue()); } else if (val instanceof Document) { - writeDoc(name, (Document)val, returnFields, 0.0f, false); + SolrDocument doc = toSolrDocument( (Document)val ); + writeSolrDocument(name, doc, returnFields, 0 ); } else if (val instanceof SolrDocument) { - writeSolrDocument(name, (SolrDocument)val, returnFields, null); - } else if (val instanceof DocList) { + writeSolrDocument(name, (SolrDocument)val, returnFields, 0); + } else if (val instanceof ResultContext) { // requires access to IndexReader - writeDocList(name, (DocList)val, returnFields,null); + writeDocuments(name, (ResultContext)val, returnFields); + } else if (val instanceof DocList) { + // Should not happen normally + ResultContext ctx = new ResultContext(); + ctx.docs = (DocList)val; + writeDocuments(name, ctx, returnFields); // } // else if (val instanceof DocSet) { // how do we know what fields to read? // todo: have a DocList/DocSet wrapper that // restricts the fields to write...? } else if (val instanceof SolrDocumentList) { - writeSolrDocumentList(name, (SolrDocumentList)val, returnFields, null); + writeSolrDocumentList(name, (SolrDocumentList)val, returnFields); } else if (val instanceof Map) { writeMap(name, (Map)val, false, true); } else if (val instanceof NamedList) { @@ -162,20 +182,82 @@ public abstract class TextResponseWriter { // types of formats, including those where the name may come after the value (like // some XML formats). - public abstract void writeDoc(String name, Document doc, Set returnFields, float score, boolean includeScore) throws IOException; + public abstract void writeStartDocumentList(String name, long start, int size, long numFound, Float maxScore) throws IOException; - /** - * @since solr 1.3 - */ - public abstract void writeSolrDocument(String name, SolrDocument doc, Set returnFields, Map pseudoFields) throws IOException; - - public abstract void writeDocList(String name, DocList ids, Set fields, Map otherFields) throws IOException; - - /** - * @since solr 1.3 - */ - public abstract void writeSolrDocumentList(String name, SolrDocumentList docs, Set fields, Map otherFields) throws IOException; + public abstract void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx) throws IOException; + + public abstract void writeEndDocumentList() throws IOException; + + // Assume each SolrDocument is already transformed + public final void writeSolrDocumentList(String name, SolrDocumentList docs, ReturnFields returnFields) throws IOException + { + writeStartDocumentList(name, docs.getStart(), docs.size(), docs.getNumFound(), docs.getMaxScore() ); + for( int i=0; i vals = new ArrayList(); + vals.add( f ); + out.setField( f.name(), vals ); + } + else{ + out.setField( f.name(), f ); + } + } + else { + out.addField( f.name(), f ); + } + } + return out; + } + + public final void writeDocuments(String name, ResultContext res, ReturnFields fields ) throws IOException { + DocList ids = res.docs; + TransformContext context = new TransformContext(); + context.query = res.query; + context.wantsScores = fields.getWantsScore() && ids.hasScores(); + writeStartDocumentList(name, ids.offset(), ids.size(), ids.matches(), + context.wantsScores ? new Float(ids.maxScore()) : null ); + + DocTransformer transformer = fields.getTransformer(); + context.searcher = req.getSearcher(); + context.iterator = ids.iterator(); + if( transformer != null ) { + transformer.setContext( context ); + } + int sz = ids.size(); + Set fnames = fields.getFieldNames(); + for (int i=0; i returnFields, float score, boolean includeScore) throws IOException { - startTag("doc", name, false); - incLevel(); - - if (includeScore) { - writeFloat("score", score); - } - - - // Lucene Documents have multivalued types as multiple fields - // with the same name. - // The XML needs to represent these as - // an array. The fastest way to detect multiple fields - // with the same name is to sort them first. - - - // using global tlst here, so we shouldn't call any other - // function that uses it until we are done. - tlst.clear(); - for (Object obj : doc.getFields()) { - Fieldable ff = (Fieldable)obj; - // skip this field if it is not a field to be returned. - if (returnFields!=null && !returnFields.contains(ff.name())) { - continue; - } - tlst.add(ff); - } - Collections.sort(tlst, fieldnameComparator); - - int sz = tlst.size(); - int fidx1 = 0, fidx2 = 0; - while (fidx1 < sz) { - Fieldable f1 = (Fieldable)tlst.get(fidx1); - String fname = f1.name(); - - // find the end of fields with this name - fidx2 = fidx1+1; - while (fidx2 < sz && fname.equals(((Fieldable)tlst.get(fidx2)).name()) ) { - fidx2++; - } - - /*** - // more efficient to use getFieldType instead of - // getField since that way dynamic fields won't have - // to create a SchemaField on the fly. - FieldType ft = schema.getFieldType(fname); - ***/ - - SchemaField sf = schema.getFieldOrNull(fname); - if( sf == null ) { - sf = new SchemaField( fname, new TextField() ); - } - if (fidx1+1 == fidx2) { - // single field value - if (sf.multiValued()) { - startTag("arr",fname,false); - doIndent=false; - sf.write(this, null, f1); - writer.write(""); - doIndent=defaultIndent; - } else { - sf.write(this, f1.name(), f1); - } - } else { - // multiple fields with same name detected - - startTag("arr",fname,false); - incLevel(); - doIndent=false; - int cnt=0; - for (int i=fidx1; i"); - // doIndent=true; - doIndent=defaultIndent; - } - fidx1 = fidx2; - } - - decLevel(); - if (doIndent) indent(); - writer.write(""); - } - - @Override - public void writeSolrDocument(String name, SolrDocument doc, Set returnFields, Map pseudoFields) throws IOException { - startTag("doc", name, false); - incLevel(); - - for (String fname : doc.getFieldNames()) { - if (returnFields!=null && !returnFields.contains(fname)) { - continue; - } - Object val = doc.getFieldValue(fname); - - if (val instanceof Collection) { - writeVal(fname, val); - } else { - // single valued... figure out if we should put tags around it anyway - SchemaField sf = schema.getFieldOrNull(fname); - if (sf!=null && sf.multiValued()) { - startTag("arr",fname,false); - doIndent=false; - writeVal(fname, val); - writer.write(""); - doIndent=defaultIndent; - } else { - writeVal(fname, val); - } - } - } - - if (pseudoFields != null) { - for (Object fname : pseudoFields.keySet()) { - writeVal(fname.toString(), pseudoFields.get(fname)); - } - } - - decLevel(); - if (doIndent) indent(); - writer.write(""); - } - - - private static interface DocumentListInfo { - Float getMaxScore(); - int getCount(); - long getNumFound(); - long getStart(); - void writeDocs( boolean includeScore, Set fields ) throws IOException; - } - - private final void writeDocuments( - String name, - DocumentListInfo docs, - Set fields) throws IOException + public void writeStartDocumentList(String name, + long start, int size, long numFound, Float maxScore) throws IOException { - boolean includeScore=false; - if (fields!=null) { - includeScore = fields.contains("score"); - if (fields.size()==0 || (fields.size()==1 && includeScore) || fields.contains("*")) { - fields=null; // null means return all stored fields - } - } - - int sz=docs.getCount(); if (doIndent) indent(); writer.write(""); - return; - } else { - writer.write('>'); - } - + writer.write(">"); + incLevel(); - docs.writeDocs(includeScore, fields); - decLevel(); + } + + /** + * The SolrDocument should already have multivalued fields implemented as + * Collections -- this will not rewrite to + */ + @Override + public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx ) throws IOException { + startTag("doc", name, false); + incLevel(); + + for (String fname : doc.getFieldNames()) { + if (!returnFields.contains(fname)) { + continue; + } + + Object val = doc.getFieldValue(fname); + if( "_explain_".equals( fname ) ) { + System.out.println( val ); + } + writeVal(fname, val); + } + + decLevel(); + writer.write(""); + } + + @Override + public void writeEndDocumentList() throws IOException + { + decLevel(); if (doIndent) indent(); writer.write(""); } - @Override - public final void writeSolrDocumentList(String name, final SolrDocumentList docs, Set fields, Map otherFields) throws IOException - { - this.writeDocuments( name, new DocumentListInfo() - { - public int getCount() { - return docs.size(); - } - - public Float getMaxScore() { - return docs.getMaxScore(); - } - - public long getNumFound() { - return docs.getNumFound(); - } - - public long getStart() { - return docs.getStart(); - } - - public void writeDocs(boolean includeScore, Set fields) throws IOException { - for( SolrDocument doc : docs ) { - writeSolrDocument(null, doc, fields, null); - } - } - }, fields ); - } - - @Override - public void writeDocList(String name, final DocList ids, Set fields, Map otherFields) throws IOException - { - this.writeDocuments( name, new DocumentListInfo() - { - public int getCount() { - return ids.size(); - } - - public Float getMaxScore() { - return ids.maxScore(); - } - - public long getNumFound() { - return ids.matches(); - } - - public long getStart() { - return ids.offset(); - } - - public void writeDocs(boolean includeScore, Set fields) throws IOException { - SolrIndexSearcher searcher = req.getSearcher(); - DocIterator iterator = ids.iterator(); - int sz = ids.size(); - includeScore = includeScore && ids.hasScores(); - for (int i=0; i= 0 ) { + doc.setField( name, docid ); + } + } +} diff --git a/solr/src/java/org/apache/solr/response/transform/DocTransformer.java b/solr/src/java/org/apache/solr/response/transform/DocTransformer.java new file mode 100644 index 00000000000..d3f07daaae4 --- /dev/null +++ b/solr/src/java/org/apache/solr/response/transform/DocTransformer.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.response.transform; + +import java.io.IOException; + +import org.apache.solr.common.SolrDocument; + +/** + * New instance for each request + * + * @version $Id: JSONResponseWriter.java 1065304 2011-01-30 15:10:15Z rmuir $ + */ +public abstract class DocTransformer +{ + public void setContext( TransformContext context ) {} + public abstract void transform(SolrDocument doc, int docid) throws IOException; +} diff --git a/solr/src/java/org/apache/solr/response/transform/DocTransformers.java b/solr/src/java/org/apache/solr/response/transform/DocTransformers.java new file mode 100644 index 00000000000..c9f7f609d14 --- /dev/null +++ b/solr/src/java/org/apache/solr/response/transform/DocTransformers.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.response.transform; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.solr.common.SolrDocument; + +/** + * Transform a document before it gets sent out + * + * @version $Id: JSONResponseWriter.java 1065304 2011-01-30 15:10:15Z rmuir $ + */ +public class DocTransformers extends DocTransformer +{ + final List children = new ArrayList(); + + public void addTransformer( DocTransformer a ) { + children.add( a ); + } + + public int size() + { + return children.size(); + } + + public DocTransformer getTransformer( int idx ) + { + return children.get( idx ); + } + + @Override + public void setContext( TransformContext context ) { + for( DocTransformer a : children ) { + a.setContext( context ); + } + } + + @Override + public void transform(SolrDocument doc, int docid) throws IOException { + for( DocTransformer a : children ) { + a.transform( doc, docid ); + } + } +} diff --git a/solr/src/java/org/apache/solr/response/transform/DocValuesAugmenter.java b/solr/src/java/org/apache/solr/response/transform/DocValuesAugmenter.java new file mode 100644 index 00000000000..26c79035d58 --- /dev/null +++ b/solr/src/java/org/apache/solr/response/transform/DocValuesAugmenter.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response.transform; + +import org.apache.solr.common.SolrDocument; +import org.apache.solr.search.function.DocValues; + +/** + * Add values from a ValueSource (function query etc) + * + * @version $Id: JSONResponseWriter.java 1065304 2011-01-30 15:10:15Z rmuir $ + * @since solr 4.0 + */ +public class DocValuesAugmenter extends DocTransformer +{ + final String name; + final DocValues values; + + public DocValuesAugmenter( String name, DocValues values ) + { + this.name = name; + this.values = values; + } + + @Override + public void transform(SolrDocument doc, int docid) { + // TODO, should know what the real type is -- not always string + Object v = values.strVal( docid ); + doc.setField( name, v ); + } +} diff --git a/solr/src/java/org/apache/solr/response/transform/ExplainAugmenter.java b/solr/src/java/org/apache/solr/response/transform/ExplainAugmenter.java new file mode 100644 index 00000000000..0b354a2c1a3 --- /dev/null +++ b/solr/src/java/org/apache/solr/response/transform/ExplainAugmenter.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response.transform; + +import java.io.IOException; + +import org.apache.lucene.search.Explanation; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.util.SolrPluginUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Add query explain info directly to Document + * + * @version $Id: JSONResponseWriter.java 1065304 2011-01-30 15:10:15Z rmuir $ + * @since solr 4.0 + */ +public class ExplainAugmenter extends TransformerWithContext +{ + static final Logger log = LoggerFactory.getLogger( ExplainAugmenter.class ); + final String name; + + public ExplainAugmenter( String display ) + { + this.name = display; + } + + @Override + public void transform(SolrDocument doc, int docid) { + if( context != null && context.query != null ) { + try { + Explanation exp = context.searcher.explain(context.query, docid); + doc.setField( name, SolrPluginUtils.explanationToNamedList(exp) ); + } + catch (IOException e) { + e.printStackTrace(); + } + } + } +} diff --git a/solr/src/java/org/apache/solr/response/transform/ScoreAugmenter.java b/solr/src/java/org/apache/solr/response/transform/ScoreAugmenter.java new file mode 100644 index 00000000000..2de825aa36f --- /dev/null +++ b/solr/src/java/org/apache/solr/response/transform/ScoreAugmenter.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response.transform; + +import org.apache.solr.common.SolrDocument; + +/** + * Simple Augmenter that adds the docId + * + * @version $Id: JSONResponseWriter.java 1065304 2011-01-30 15:10:15Z rmuir $ + * @since solr 4.0 + */ +public class ScoreAugmenter extends TransformerWithContext +{ + final String name; + + public ScoreAugmenter( String display ) + { + this.name = display; + } + + @Override + public void transform(SolrDocument doc, int docid) { + if( context != null && context.wantsScores ) { + if( context.iterator != null ) { + doc.setField( name, context.iterator.score() ); + } + } + } +} diff --git a/solr/src/java/org/apache/solr/response/transform/TransformContext.java b/solr/src/java/org/apache/solr/response/transform/TransformContext.java new file mode 100644 index 00000000000..81a7401257f --- /dev/null +++ b/solr/src/java/org/apache/solr/response/transform/TransformContext.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response.transform; + +import org.apache.lucene.search.Query; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.SolrIndexSearcher; + +/** + * Environment variables for the transformed documents + * + * @version $Id: JSONResponseWriter.java 1065304 2011-01-30 15:10:15Z rmuir $ + * @since solr 4.0 + */ +public class TransformContext +{ + public Query query; + public boolean wantsScores = false; + public DocIterator iterator; + public SolrIndexSearcher searcher; +} diff --git a/solr/src/java/org/apache/solr/response/transform/TransformerWithContext.java b/solr/src/java/org/apache/solr/response/transform/TransformerWithContext.java new file mode 100644 index 00000000000..36d679febda --- /dev/null +++ b/solr/src/java/org/apache/solr/response/transform/TransformerWithContext.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response.transform; + + +/** + * @version $Id: JSONResponseWriter.java 1065304 2011-01-30 15:10:15Z rmuir $ + * @since solr 4.0 + */ +public abstract class TransformerWithContext extends DocTransformer +{ + protected TransformContext context = null; + + @Override + public void setContext( TransformContext context ) { + this.context = context; + } +} diff --git a/solr/src/java/org/apache/solr/response/transform/ValueAugmenter.java b/solr/src/java/org/apache/solr/response/transform/ValueAugmenter.java new file mode 100644 index 00000000000..b3cbc4c5467 --- /dev/null +++ b/solr/src/java/org/apache/solr/response/transform/ValueAugmenter.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response.transform; + +import org.apache.solr.common.SolrDocument; + +/** + * @version $Id: JSONResponseWriter.java 1065304 2011-01-30 15:10:15Z rmuir $ + * @since solr 4.0 + */ +public class ValueAugmenter extends DocTransformer +{ + final String name; + final Object value; + + public ValueAugmenter( String name, Object value ) + { + this.name = name; + this.value = value; + } + + @Override + public void transform(SolrDocument doc, int docid) { + doc.setField( name, value ); + } +} diff --git a/solr/src/java/org/apache/solr/util/SolrPluginUtils.java b/solr/src/java/org/apache/solr/util/SolrPluginUtils.java index 6cdf1d6c3fb..f44c0b95a6c 100644 --- a/solr/src/java/org/apache/solr/util/SolrPluginUtils.java +++ b/solr/src/java/org/apache/solr/util/SolrPluginUtils.java @@ -37,6 +37,7 @@ import org.apache.solr.handler.component.HighlightComponent; import org.apache.solr.handler.component.ResponseBuilder; import org.apache.solr.highlight.SolrHighlighter; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.ReturnFields; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; @@ -115,41 +116,6 @@ public class SolrPluginUtils { return splitList.split(value.trim(), 0); } - /** - * Assumes the standard query param of "fl" to specify the return fields - * @see #setReturnFields(String,SolrQueryResponse) - */ - public static int setReturnFields(SolrQueryRequest req, - SolrQueryResponse res) { - - return setReturnFields(req.getParams().get(CommonParams.FL), res); - } - - /** - * Given a space seperated list of field names, sets the field list on the - * SolrQueryResponse. - * - * @return bitfield of SolrIndexSearcher flags that need to be set - */ - public static int setReturnFields(String fl, - SolrQueryResponse res) { - int flags = 0; - if (fl != null) { - // TODO - this could become more efficient if widely used. - // TODO - should field order be maintained? - String[] flst = split(fl); - if (flst.length > 0 && !(flst.length==1 && flst[0].length()==0)) { - Set set = new LinkedHashSet(); - for (String fname : flst) { - if("score".equalsIgnoreCase(fname)) - flags |= SolrIndexSearcher.GET_SCORES; - set.add(fname); - } - res.setReturnFields(set); - } - } - return flags; - } /** * Pre-fetch documents into the index searcher's document cache. @@ -180,14 +146,13 @@ public class SolrPluginUtils { return; } - Set returnFields = res.getReturnFields(); - Set fieldFilter = returnFields; - - if(returnFields != null) { + ReturnFields returnFields = res.getReturnFields(); + if(returnFields.getFieldNames() != null) { + Set fieldFilter = returnFields.getFieldNames(); if (rb.doHighlights) { // copy return fields list - fieldFilter = new HashSet(returnFields); + fieldFilter = new HashSet(fieldFilter); // add highlight fields SolrHighlighter highlighter = HighlightComponent.getHighlighter(req.getCore()); diff --git a/solr/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/src/test/org/apache/solr/BasicFunctionalityTest.java index 6f81d40014e..f19d9b2b8c0 100644 --- a/solr/src/test/org/apache/solr/BasicFunctionalityTest.java +++ b/solr/src/test/org/apache/solr/BasicFunctionalityTest.java @@ -42,6 +42,7 @@ import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.response.ResultContext; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.response.XMLWriter; import org.apache.solr.schema.IndexSchema; @@ -558,7 +559,7 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { SolrQueryResponse rsp = new SolrQueryResponse(); core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp); - DocList dl = (DocList) rsp.getValues().get("response"); + DocList dl = ((ResultContext) rsp.getValues().get("response")).docs; org.apache.lucene.document.Document d = req.getSearcher().doc(dl.iterator().nextDoc()); // ensure field is not lazy assertTrue( d.getFieldable("test_hlt") instanceof Field ); @@ -580,7 +581,7 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { SolrQueryResponse rsp = new SolrQueryResponse(); core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp); - DocList dl = (DocList) rsp.getValues().get("response"); + DocList dl = ((ResultContext) rsp.getValues().get("response")).docs; DocIterator di = dl.iterator(); org.apache.lucene.document.Document d = req.getSearcher().doc(di.nextDoc()); // ensure field is lazy diff --git a/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java b/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java index f1aea4dc418..8b7b8c09f76 100644 --- a/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java +++ b/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java @@ -46,9 +46,11 @@ import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.util.XML; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.FacetParams; import org.junit.Test; @@ -218,6 +220,39 @@ abstract public class SolrExampleTests extends SolrJettyTestBase assertEquals( 1, rsp.getResults().getNumFound() ); } + + + /** + * Get empty results + */ + @Test + public void testGetEmptyResults() throws Exception + { + SolrServer server = getSolrServer(); + + // Empty the database... + server.deleteByQuery( "*:*" );// delete everything! + server.commit(); + + // Add two docs + SolrInputDocument doc = new SolrInputDocument(); + doc.addField( "id", "id1", 1.0f ); + doc.addField( "name", "doc1", 1.0f ); + doc.addField( "price", 10 ); + server.add( doc ); + + doc = new SolrInputDocument(); + doc.addField( "id", "id2", 1.0f ); + server.add( doc ); + server.commit(); + + // Make sure we get empty docs for unknown field + SolrDocumentList out = server.query( new SolrQuery( "*:*" ).set("fl", "foofoofoo" ) ).getResults(); + assertEquals( 2, out.getNumFound() ); + assertEquals( 0, out.get(0).size() ); + assertEquals( 0, out.get(1).size() ); + + } private String randomTestString(int maxLength) { // we can't just use _TestUtil.randomUnicodeString() or we might get 0xfffe etc @@ -345,6 +380,55 @@ abstract public class SolrExampleTests extends SolrJettyTestBase } + @Test + public void testAugmentFields() throws Exception + { + SolrServer server = getSolrServer(); + + // Empty the database... + server.deleteByQuery( "*:*" );// delete everything! + + // Now add something... + SolrInputDocument doc = new SolrInputDocument(); + doc.addField( "id", "111", 1.0f ); + doc.addField( "name", "doc1", 1.0f ); + doc.addField( "price", 11 ); + server.add( doc ); + + doc = new SolrInputDocument(); + doc.addField( "id", "222", 1.0f ); + doc.addField( "name", "doc2", 1.0f ); + doc.addField( "price", 22 ); + server.add( doc ); + + // Add the documents + server.commit(); + + SolrQuery query = new SolrQuery(); + query.setQuery( "*:*" ); + query.set( CommonParams.FL, "id,price,_docid_,_explain_,score" ); + query.addSortField( "price", SolrQuery.ORDER.asc ); + QueryResponse rsp = server.query( query ); + + SolrDocumentList out = rsp.getResults(); + assertEquals( 2, out.getNumFound() ); + SolrDocument out1 = out.get( 0 ); + SolrDocument out2 = out.get( 1 ); + assertEquals( "111", out1.getFieldValue( "id" ) ); + assertEquals( "222", out2.getFieldValue( "id" ) ); + assertEquals( 1.0f, out1.getFieldValue( "score" ) ); + assertEquals( 1.0f, out2.getFieldValue( "score" ) ); + + // check that the docid is one bigger + int id1 = (Integer)out1.getFieldValue( "_docid_" ); + int id2 = (Integer)out2.getFieldValue( "_docid_" ); + assertEquals( "should be one bigger ["+id1+","+id2+"]", id1, id2-1 ); + + // The score from explain should be the same as the score + NamedList explain = (NamedList)out1.getFieldValue( "_explain_" ); + assertEquals( out1.get( "score"), explain.get( "value" ) ); + } + @Test public void testContentStreamRequest() throws Exception { SolrServer server = getSolrServer(); @@ -804,6 +888,7 @@ abstract public class SolrExampleTests extends SolrJettyTestBase // Make sure it ran OK SolrQuery query = new SolrQuery("*:*"); + query.set( CommonParams.FL, "id,score,_docid_" ); QueryResponse response = server.query(query); assertEquals(0, response.getStatus()); assertEquals(10, response.getResults().getNumFound()); @@ -820,6 +905,11 @@ abstract public class SolrExampleTests extends SolrJettyTestBase @Override public void streamSolrDocument(SolrDocument doc) { cnt.incrementAndGet(); + + // Make sure the transformer works for streaming + Float score = (Float)doc.get( "score" ); + Integer docid = (Integer)doc.get( "_docid_" ); + assertEquals( "should have score", new Float(1.0), score ); } }); diff --git a/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java b/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java index f5ed5027050..3b9ecacb56c 100644 --- a/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java +++ b/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java @@ -22,7 +22,6 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.util.DateUtil; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.util.SolrPluginUtils; import org.junit.*; import java.io.StringWriter; @@ -129,19 +128,19 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 { rsp.add("response", sdl); QueryResponseWriter w = new CSVResponseWriter(); - SolrPluginUtils.setReturnFields("id,foo_s", rsp); + rsp.setReturnFields( ReturnFields.getReturnFields("id,foo_s", req) ); StringWriter buf = new StringWriter(); w.write(buf, req, rsp); assertEquals("id,foo_s\n1,hi\n2,\n", buf.toString()); // try scores - SolrPluginUtils.setReturnFields("id,score,foo_s", rsp); + rsp.setReturnFields( ReturnFields.getReturnFields("id,score,foo_s", req) ); buf = new StringWriter(); w.write(buf, req, rsp); assertEquals("id,score,foo_s\n1,2.718,hi\n2,89.83,\n", buf.toString()); // get field values from docs... should be ordered and not include score unless requested - SolrPluginUtils.setReturnFields("*", rsp); + rsp.setReturnFields( ReturnFields.getReturnFields("*", req) ); buf = new StringWriter(); w.write(buf, req, rsp); assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss\n" + @@ -151,7 +150,7 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 { // get field values and scores - just check that the scores are there... we don't guarantee where - SolrPluginUtils.setReturnFields("*,score", rsp); + rsp.setReturnFields( ReturnFields.getReturnFields("*,score", req) ); buf = new StringWriter(); w.write(buf, req, rsp); String s = buf.toString(); diff --git a/solr/src/test/org/apache/solr/search/TestRangeQuery.java b/solr/src/test/org/apache/solr/search/TestRangeQuery.java index 997d3d991b5..1a1f00b1cd6 100644 --- a/solr/src/test/org/apache/solr/search/TestRangeQuery.java +++ b/solr/src/test/org/apache/solr/search/TestRangeQuery.java @@ -19,6 +19,7 @@ package org.apache.solr.search; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.ResultContext; import org.apache.solr.response.SolrQueryResponse; import org.junit.Before; import org.junit.BeforeClass; @@ -263,7 +264,9 @@ public class TestRangeQuery extends SolrTestCaseJ4 { SolrQueryResponse qr = h.queryAndResponse(handler, req); if (last != null) { // we only test if the same docs matched since some queries will include factors like idf, etc. - sameDocs((DocSet)qr.getValues().get("response"), (DocSet)last.getValues().get("response")); + DocList rA = ((ResultContext)qr.getValues().get("response")).docs; + DocList rB = ((ResultContext)last.getValues().get("response")).docs; + sameDocs( rA, rB ); } req.close(); last = qr; diff --git a/solr/src/webapp/src/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java b/solr/src/webapp/src/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java index e0a2fba0ee7..95ce0fc44d7 100644 --- a/solr/src/webapp/src/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java +++ b/solr/src/webapp/src/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java @@ -43,7 +43,9 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.BinaryResponseWriter; +import org.apache.solr.response.ResultContext; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.response.transform.DocTransformer; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.servlet.SolrRequestParsers; @@ -164,28 +166,16 @@ public class EmbeddedSolrServer extends SolrServer new BinaryResponseWriter.Resolver( req, rsp.getReturnFields()) { @Override - public void writeDocList(DocList ids, JavaBinCodec codec) throws IOException { + public void writeResults(ResultContext ctx, JavaBinCodec codec) throws IOException { // write an empty list... SolrDocumentList docs = new SolrDocumentList(); - docs.setNumFound( ids.matches() ); - docs.setStart( ids.offset() ); - docs.setMaxScore( ids.maxScore() ); + docs.setNumFound( ctx.docs.matches() ); + docs.setStart( ctx.docs.offset() ); + docs.setMaxScore( ctx.docs.maxScore() ); codec.writeSolrDocumentList( docs ); - int sz = ids.size(); - - if(searcher == null) searcher = solrQueryRequest.getSearcher(); - if(schema == null) schema = solrQueryRequest.getSchema(); - DocIterator iterator = ids.iterator(); - for (int i = 0; i < sz; i++) { - int id = iterator.nextDoc(); - Document doc = searcher.doc(id, returnFields); - SolrDocument sdoc = getDoc(doc); - if (includeScore && ids.hasScores()) { - sdoc.addField("score", iterator.score()); - } - callback.streamSolrDocument( sdoc ); - } + // This will transform + writeResultsBody( ctx, codec ); } }; @@ -194,7 +184,7 @@ public class EmbeddedSolrServer extends SolrServer new JavaBinCodec(resolver) { @Override - public void writeSolrDocument(SolrDocument doc, Set fields) throws IOException { + public void writeSolrDocument(SolrDocument doc) throws IOException { callback.streamSolrDocument( doc ); //super.writeSolrDocument( doc, fields ); }