mirror of https://github.com/apache/lucene.git
SOLR-7414: CSVResponseWriter & XLSXResponseWriter return empty field when fl alias is combined with * selector
This commit is contained in:
Ishan Chattopadhyaya
parent
8d7619f4e6
commit
e7939d5907
|
|
@ -141,6 +141,9 @@ Bug Fixes
|
|||
* SOLR-13253: IndexSchema.getResourceLoader was being used to load non-schema things, which can be a memory leak if
|
||||
"shareSchema" and other circumstances occur. Furthermore it's reference to SolrConfig was removed. (David Smiley)
|
||||
|
||||
* SOLR-7414: CSVResponseWriter & XLSXResponseWriter return empty field when fl alias is combined with '*' selector
|
||||
(Michael Lawrence, Munendra S N, Ishan Chattopadhyaya)
|
||||
|
||||
Improvements
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -27,12 +27,9 @@ import java.util.Collection;
|
|||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.FillPatternType;
|
||||
|
|
@ -43,19 +40,13 @@ import org.apache.poi.ss.usermodel.Sheet;
|
|||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFCellStyle;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.BasicResultContext;
|
||||
import org.apache.solr.response.RawResponseWriter;
|
||||
import org.apache.solr.response.ResultContext;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.response.TextResponseWriter;
|
||||
import org.apache.solr.response.TabularResponseWriter;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.schema.StrField;
|
||||
import org.apache.solr.search.DocList;
|
||||
import org.apache.solr.search.ReturnFields;
|
||||
|
||||
public class XLSXResponseWriter extends RawResponseWriter {
|
||||
|
|
@ -94,10 +85,7 @@ public class XLSXResponseWriter extends RawResponseWriter {
|
|||
}
|
||||
}
|
||||
|
||||
class XLSXWriter extends TextResponseWriter {
|
||||
|
||||
SolrQueryRequest req;
|
||||
SolrQueryResponse rsp;
|
||||
class XLSXWriter extends TabularResponseWriter {
|
||||
|
||||
static class SerialWriteWorkbook {
|
||||
SXSSFWorkbook swb;
|
||||
|
|
@ -172,45 +160,14 @@ class XLSXWriter extends TextResponseWriter {
|
|||
|
||||
private Map<String,XLField> xlFields = new LinkedHashMap<String,XLField>();
|
||||
|
||||
public XLSXWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp){
|
||||
public XLSXWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
|
||||
super(writer, req, rsp);
|
||||
this.req = req;
|
||||
this.rsp = rsp;
|
||||
}
|
||||
|
||||
public void writeResponse(OutputStream out, LinkedHashMap<String, String> colNamesMap,
|
||||
LinkedHashMap<String, Integer> colWidthsMap) throws IOException {
|
||||
SolrParams params = req.getParams();
|
||||
|
||||
Collection<String> fields = returnFields.getRequestedFieldNames();
|
||||
Object responseObj = rsp.getValues().get("response");
|
||||
boolean returnOnlyStored = false;
|
||||
if (fields==null||returnFields.hasPatternMatching()) {
|
||||
if (responseObj instanceof SolrDocumentList) {
|
||||
// get the list of fields from the SolrDocumentList
|
||||
if(fields==null) {
|
||||
fields = new LinkedHashSet<String>();
|
||||
}
|
||||
for (SolrDocument sdoc: (SolrDocumentList)responseObj) {
|
||||
fields.addAll(sdoc.getFieldNames());
|
||||
}
|
||||
} else {
|
||||
// get the list of fields from the index
|
||||
Iterable<String> all = req.getSearcher().getFieldNames();
|
||||
if (fields == null) {
|
||||
fields = Sets.newHashSet(all);
|
||||
} else {
|
||||
Iterables.addAll(fields, all);
|
||||
}
|
||||
}
|
||||
if (returnFields.wantsScore()) {
|
||||
fields.add("score");
|
||||
} else {
|
||||
fields.remove("score");
|
||||
}
|
||||
returnOnlyStored = true;
|
||||
}
|
||||
|
||||
Collection<String> fields = getFields();
|
||||
for (String field : fields) {
|
||||
if (!returnFields.wantsField(field)) {
|
||||
continue;
|
||||
|
|
@ -222,17 +179,15 @@ class XLSXWriter extends TextResponseWriter {
|
|||
continue;
|
||||
}
|
||||
|
||||
if (shouldSkipField(field)) {
|
||||
continue;
|
||||
}
|
||||
SchemaField sf = schema.getFieldOrNull(field);
|
||||
if (sf == null) {
|
||||
FieldType ft = new StrField();
|
||||
sf = new SchemaField(field, ft);
|
||||
}
|
||||
|
||||
// Return only stored fields, unless an explicit field list is specified
|
||||
if (returnOnlyStored && sf != null && !sf.stored()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
XLField xlField = new XLField();
|
||||
xlField.name = field;
|
||||
xlField.sf = sf;
|
||||
|
|
@ -264,15 +219,7 @@ class XLSXWriter extends TextResponseWriter {
|
|||
wb.setHeaderRow();
|
||||
wb.addRow();
|
||||
|
||||
if (responseObj instanceof ResultContext) {
|
||||
writeDocuments(null, (ResultContext)responseObj );
|
||||
}
|
||||
else if (responseObj instanceof DocList) {
|
||||
ResultContext ctx = new BasicResultContext((DocList)responseObj, returnFields, null, null, req);
|
||||
writeDocuments(null, ctx );
|
||||
} else if (responseObj instanceof SolrDocumentList) {
|
||||
writeSolrDocumentList(null, (SolrDocumentList)responseObj, returnFields );
|
||||
}
|
||||
writeResponse(rsp.getResponse());
|
||||
|
||||
wb.flush(out);
|
||||
wb = null;
|
||||
|
|
@ -283,23 +230,6 @@ class XLSXWriter extends TextResponseWriter {
|
|||
super.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeNamedList(String name, NamedList val) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeStartDocumentList(String name,
|
||||
long start, int size, long numFound, Float maxScore) throws IOException
|
||||
{
|
||||
// nothing
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeEndDocumentList() throws IOException
|
||||
{
|
||||
// nothing
|
||||
}
|
||||
|
||||
//NOTE: a document cannot currently contain another document
|
||||
List tmpList;
|
||||
@Override
|
||||
|
|
@ -346,10 +276,6 @@ class XLSXWriter extends TextResponseWriter {
|
|||
wb.writeCell(val);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeArray(String name, Iterator val) throws IOException {
|
||||
StringBuffer output = new StringBuffer();
|
||||
|
|
@ -403,11 +329,6 @@ class XLSXWriter extends TextResponseWriter {
|
|||
wb.writeCell(val);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeDate(String name, Date val) throws IOException {
|
||||
writeDate(name, val.toInstant().toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeDate(String name, String val) throws IOException {
|
||||
wb.writeCell(val);
|
||||
|
|
|
|||
|
|
@ -330,6 +330,12 @@
|
|||
</analyzer>
|
||||
</fieldType>
|
||||
|
||||
<!-- Point Fields -->
|
||||
<fieldType name="pint" class="solr.IntPointField" docValues="true"/>
|
||||
<fieldType name="plong" class="solr.LongPointField" docValues="true"/>
|
||||
<fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
|
||||
<fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
|
||||
<fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
|
||||
|
||||
<field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
|
||||
<field name="name" type="nametext" indexed="true" stored="true"/>
|
||||
|
|
@ -462,6 +468,9 @@
|
|||
<!-- ignored because not stored or indexed -->
|
||||
<dynamicField name="ignored_*" type="text" indexed="false" stored="false"/>
|
||||
|
||||
<dynamicField name="*_ii" type="pint" indexed="false" stored="false" useDocValuesAsStored="true"/>
|
||||
<dynamicField name="*_iis" type="pint" indexed="false" stored="false" useDocValuesAsStored="true"/>
|
||||
<dynamicField name="*_ff" type="pfloat" indexed="false" stored="false" useDocValuesAsStored="false"/>
|
||||
|
||||
<uniqueKey>id</uniqueKey>
|
||||
|
||||
|
|
|
|||
|
|
@ -18,23 +18,24 @@ package org.apache.solr.handler.extraction;
|
|||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.time.Instant;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFSheet;
|
||||
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.QueryResponseWriter;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.response.RawResponseWriter;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.search.SolrReturnFields;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
|
|
@ -64,6 +65,7 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {
|
|||
assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output", "shouldbeunstored","foo"));
|
||||
assertU(adoc("id","3", "shouldbeunstored","foo"));
|
||||
assertU(adoc("id","4", "foo_s1","foo"));
|
||||
assertU(adoc("id","5", "pubyear_ii", "123", "store_iis", "12", "price_ff", "1.3"));
|
||||
assertU(commit());
|
||||
}
|
||||
|
||||
|
|
@ -73,7 +75,7 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testStructuredDataViaBaseWriters() throws IOException, Exception {
|
||||
public void testStructuredDataViaBaseWriters() throws Exception {
|
||||
SolrQueryResponse rsp = new SolrQueryResponse();
|
||||
// Don't send a ContentStream back, this will fall back to the configured base writer.
|
||||
// But abuse the CONTENT key to ensure writer is also checking type
|
||||
|
|
@ -220,19 +222,85 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {
|
|||
//assertions specific to multiple pseudofields functions like abs, div, exists, etc.. (SOLR-5423)
|
||||
String funcText = getStringFromSheet(getWSResultForQuery(req("df", "text", "q","*", "wt","xlsx", "fl","XXX:id,YYY:exists(foo_s1)")));
|
||||
String[] funcLines = funcText.split("\n");
|
||||
assertEquals(5, funcLines.length);
|
||||
assertEquals(6, funcLines.length);
|
||||
assertEquals("XXX,YYY", funcLines[0] );
|
||||
assertEquals("1,false", funcLines[1] );
|
||||
assertEquals("3,false", funcLines[3] );
|
||||
|
||||
//assertions specific to single function without alias (SOLR-5423)
|
||||
String singleFuncText = getStringFromSheet(
|
||||
getWSResultForQuery(req("df", "text", "q","*", "wt","xlsx", "fl","exists(foo_s1),XXX:id")));
|
||||
String[] singleFuncLines = singleFuncText.split("\n");
|
||||
assertEquals(6, singleFuncLines.length);
|
||||
assertEquals("exists(foo_s1),XXX", singleFuncLines[0] );
|
||||
assertEquals("false,1", singleFuncLines[1] );
|
||||
assertEquals("false,3", singleFuncLines[3] );
|
||||
|
||||
// pseudo-fields with * in fl
|
||||
txt = getStringFromSheet(
|
||||
getWSResultForQuery(req("df", "text", "q","id:4", "wt","xlsx", "fl","*,YYY:[docid],FOO:foo_s1")));
|
||||
lines = txt.split("\n");
|
||||
assertEquals(2, lines.length);
|
||||
assertEquals(sortHeader("foo_i,foo_l,FOO,foo_s,pubyear_ii,store_iis," +
|
||||
"v2_ss,multiDefault,timestamp,foo_dt1,foo_b,YYY,foo_d,id,foo_f,v_ss,foo_s1,intDefault"), sortHeader(lines[0]));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testForDVEnabledFields() throws Exception {
|
||||
// for dv enabled and useDocValueAsStored=true
|
||||
// returns pubyear_ii, store_iis but not price_ff
|
||||
String singleFuncText = getStringFromSheet(
|
||||
getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx")));
|
||||
String sortedHeader = sortHeader("foo_i,foo_l,foo_s,pubyear_ii,store_iis," +
|
||||
"v2_ss,multiDefault,timestamp,foo_dt1,foo_b,foo_d,id,foo_f,v_ss,foo_s1,intDefault");
|
||||
String[] singleFuncLines = singleFuncText.split("\n");
|
||||
assertEquals(2, singleFuncLines.length);
|
||||
assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));
|
||||
List<String> actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))
|
||||
.filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))
|
||||
.collect(Collectors.toList());
|
||||
assertTrue(actualVal.containsAll(Arrays.asList("5", "123", "12")));
|
||||
|
||||
// explicit fl=*
|
||||
singleFuncText = getStringFromSheet(
|
||||
getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx", "fl", "*")));
|
||||
singleFuncLines = singleFuncText.split("\n");
|
||||
assertEquals(2, singleFuncLines.length);
|
||||
assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));
|
||||
actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))
|
||||
.filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))
|
||||
.collect(Collectors.toList());
|
||||
assertTrue(actualVal.containsAll(Arrays.asList("5", "123", "12")));
|
||||
|
||||
// explicit price_ff
|
||||
singleFuncText = getStringFromSheet(
|
||||
getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx", "fl", "price_ff")));
|
||||
singleFuncLines = singleFuncText.split("\n");
|
||||
assertEquals(2, singleFuncLines.length);
|
||||
assertEquals("price_ff", singleFuncLines[0]);
|
||||
assertEquals("1.3", singleFuncLines[1]);
|
||||
|
||||
// explicit price_ff with fl=*
|
||||
singleFuncText = getStringFromSheet(
|
||||
getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx", "csv.header","true", "fl", "*,price_ff")));
|
||||
sortedHeader = sortHeader("foo_i,foo_l,foo_b,foo_s,pubyear_ii,store_iis," +
|
||||
"v2_ss,multiDefault,timestamp,foo_dt1,id,foo_d,foo_f,v_ss,foo_s1,intDefault,price_ff");
|
||||
singleFuncLines = singleFuncText.split("\n");
|
||||
assertEquals(2, singleFuncLines.length);
|
||||
assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));
|
||||
actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))
|
||||
.filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))
|
||||
.collect(Collectors.toList());
|
||||
assertTrue(actualVal.containsAll(Arrays.asList("5", "123", "12", "1.3")));
|
||||
}
|
||||
|
||||
// returns first worksheet as XLSXResponseWriter only returns one sheet
|
||||
private XSSFSheet getWSResultForQuery(SolrQueryRequest req) throws IOException, Exception {
|
||||
private XSSFSheet getWSResultForQuery(SolrQueryRequest req) throws Exception {
|
||||
SolrQueryResponse rsp = h.queryAndResponse("", req);
|
||||
return getWSResultForQuery(req, rsp);
|
||||
}
|
||||
|
||||
private XSSFSheet getWSResultForQuery(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, Exception {
|
||||
private XSSFSheet getWSResultForQuery(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
|
||||
ByteArrayOutputStream xmlBout = new ByteArrayOutputStream();
|
||||
writerXlsx.write(xmlBout, req, rsp);
|
||||
XSSFWorkbook output = new XSSFWorkbook(new ByteArrayInputStream(xmlBout.toByteArray()));
|
||||
|
|
@ -254,4 +322,13 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {
|
|||
}
|
||||
return output.toString();
|
||||
}
|
||||
|
||||
/*
|
||||
* Utility method to sort a comma separated list of strings, for easier comparison regardless of platform
|
||||
*/
|
||||
private String sortHeader(String input) {
|
||||
String[] output = input.trim().split(",");
|
||||
Arrays.sort(output);
|
||||
return Arrays.toString(output);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,18 +22,12 @@ import java.io.Writer;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.FastWriter;
|
||||
|
|
@ -44,11 +38,10 @@ import org.apache.solr.request.SolrQueryRequest;
|
|||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.schema.StrField;
|
||||
import org.apache.solr.search.DocList;
|
||||
import org.apache.solr.search.ReturnFields;
|
||||
|
||||
/**
|
||||
*
|
||||
* Response writer for csv data
|
||||
*/
|
||||
|
||||
public class CSVResponseWriter implements QueryResponseWriter {
|
||||
|
|
@ -75,7 +68,7 @@ public class CSVResponseWriter implements QueryResponseWriter {
|
|||
}
|
||||
|
||||
|
||||
class CSVWriter extends TextResponseWriter {
|
||||
class CSVWriter extends TabularResponseWriter {
|
||||
static String SEPARATOR = "separator";
|
||||
static String ENCAPSULATOR = "encapsulator";
|
||||
static String ESCAPE = "escape";
|
||||
|
|
@ -245,35 +238,7 @@ class CSVWriter extends TextResponseWriter {
|
|||
// encapsulator will already be disabled if it wasn't specified
|
||||
}
|
||||
|
||||
Collection<String> fields = returnFields.getRequestedFieldNames();
|
||||
Object responseObj = rsp.getResponse();
|
||||
boolean returnStoredOrDocValStored = false;
|
||||
if (fields==null||returnFields.hasPatternMatching()) {
|
||||
if (responseObj instanceof SolrDocumentList) {
|
||||
// get the list of fields from the SolrDocumentList
|
||||
if(fields==null) {
|
||||
fields = new LinkedHashSet<>();
|
||||
}
|
||||
for (SolrDocument sdoc: (SolrDocumentList)responseObj) {
|
||||
fields.addAll(sdoc.getFieldNames());
|
||||
}
|
||||
} else {
|
||||
// get the list of fields from the index
|
||||
Iterable<String> all = req.getSearcher().getFieldNames();
|
||||
if (fields == null) {
|
||||
fields = Sets.newHashSet(all);
|
||||
} else {
|
||||
Iterables.addAll(fields, all);
|
||||
}
|
||||
}
|
||||
if (returnFields.wantsScore()) {
|
||||
fields.add("score");
|
||||
} else {
|
||||
fields.remove("score");
|
||||
}
|
||||
returnStoredOrDocValStored = true;
|
||||
}
|
||||
|
||||
Collection<String> fields = getFields();
|
||||
CSVSharedBufPrinter csvPrinterMV = new CSVSharedBufPrinter(mvWriter, mvStrategy);
|
||||
|
||||
for (String field : fields) {
|
||||
|
|
@ -287,18 +252,16 @@ class CSVWriter extends TextResponseWriter {
|
|||
continue;
|
||||
}
|
||||
|
||||
if (shouldSkipField(field)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
SchemaField sf = schema.getFieldOrNull(field);
|
||||
if (sf == null) {
|
||||
FieldType ft = new StrField();
|
||||
sf = new SchemaField(field, ft);
|
||||
}
|
||||
|
||||
// Return stored fields or useDocValuesAsStored=true fields,
|
||||
// unless an explicit field list is specified
|
||||
if (returnStoredOrDocValStored && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// check for per-field overrides
|
||||
sep = params.get("f." + field + '.' + CSV_SEPARATOR);
|
||||
encapsulator = params.get("f." + field + '.' + CSV_ENCAPSULATOR);
|
||||
|
|
@ -350,17 +313,7 @@ class CSVWriter extends TextResponseWriter {
|
|||
printer.println();
|
||||
}
|
||||
|
||||
if (responseObj instanceof ResultContext) {
|
||||
writeDocuments(null, (ResultContext)responseObj );
|
||||
}
|
||||
else if (responseObj instanceof DocList) {
|
||||
|
||||
ResultContext ctx = new BasicResultContext((DocList)responseObj, returnFields, null, null, req);
|
||||
writeDocuments(null, ctx );
|
||||
} else if (responseObj instanceof SolrDocumentList) {
|
||||
writeSolrDocumentList(null, (SolrDocumentList)responseObj, returnFields );
|
||||
}
|
||||
|
||||
writeResponse(rsp.getResponse());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -369,23 +322,6 @@ class CSVWriter extends TextResponseWriter {
|
|||
super.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeNamedList(String name, NamedList val) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeStartDocumentList(String name,
|
||||
long start, int size, long numFound, Float maxScore) throws IOException
|
||||
{
|
||||
// nothing
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeEndDocumentList() throws IOException
|
||||
{
|
||||
// nothing
|
||||
}
|
||||
|
||||
//NOTE: a document cannot currently contain another document
|
||||
List tmpList;
|
||||
@Override
|
||||
|
|
@ -456,14 +392,6 @@ class CSVWriter extends TextResponseWriter {
|
|||
printer.print(val, needsEscaping);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeArray(String name, Iterator val) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeNull(String name) throws IOException {
|
||||
printer.print(NullValue);
|
||||
|
|
@ -494,11 +422,6 @@ class CSVWriter extends TextResponseWriter {
|
|||
printer.print(val, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeDate(String name, Date val) throws IOException {
|
||||
writeDate(name, val.toInstant().toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeDate(String name, String val) throws IOException {
|
||||
printer.print(val, false);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,149 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.response;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.search.DocList;
|
||||
|
||||
/**
|
||||
* Base response writer for table-oriented data
|
||||
*/
|
||||
public abstract class TabularResponseWriter extends TextResponseWriter {
|
||||
|
||||
private boolean returnStoredOrDocValStored;
|
||||
|
||||
public TabularResponseWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse resp) {
|
||||
super(writer, req, resp);
|
||||
// fl=* or globs specified in fl
|
||||
returnStoredOrDocValStored = ((returnFields.getRequestedFieldNames() == null) ||
|
||||
returnFields.hasPatternMatching());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns fields to be returned in the response
|
||||
*/
|
||||
public Collection<String> getFields() {
|
||||
Collection<String> fields = returnFields.getRequestedFieldNames();
|
||||
Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
|
||||
Object responseObj = rsp.getResponse();
|
||||
if (fields==null||returnFields.hasPatternMatching()) {
|
||||
if (responseObj instanceof SolrDocumentList) {
|
||||
// get the list of fields from the SolrDocumentList
|
||||
if(fields==null) {
|
||||
fields = new LinkedHashSet<>();
|
||||
}
|
||||
for (SolrDocument sdoc: (SolrDocumentList)responseObj) {
|
||||
fields.addAll(sdoc.getFieldNames());
|
||||
}
|
||||
} else {
|
||||
// get the list of fields from the index
|
||||
Iterable<String> all = req.getSearcher().getFieldNames();
|
||||
if (fields == null) {
|
||||
fields = Sets.newHashSet(all);
|
||||
} else {
|
||||
Iterables.addAll(fields, all);
|
||||
}
|
||||
}
|
||||
|
||||
if (explicitReqFields != null) {
|
||||
// add explicit requested fields
|
||||
Iterables.addAll(fields, explicitReqFields);
|
||||
}
|
||||
|
||||
if (returnFields.wantsScore()) {
|
||||
fields.add("score");
|
||||
} else {
|
||||
fields.remove("score");
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if field needs to be skipped else false
|
||||
* @param field name of the field
|
||||
* @return boolean value
|
||||
*/
|
||||
public boolean shouldSkipField(String field) {
|
||||
Set<String> explicitReqFields = returnFields.getExplicitlyRequestedFieldNames();
|
||||
SchemaField sf = schema.getFieldOrNull(field);
|
||||
|
||||
// Return stored fields or useDocValuesAsStored=true fields,
|
||||
// unless an explicit field list is specified
|
||||
return (returnStoredOrDocValStored && !(explicitReqFields != null && explicitReqFields.contains(field)) &&
|
||||
sf!= null && !sf.stored() && !(sf.hasDocValues() && sf.useDocValuesAsStored()));
|
||||
}
|
||||
|
||||
public void writeResponse(Object responseObj) throws IOException {
|
||||
if (responseObj instanceof ResultContext) {
|
||||
writeDocuments(null, (ResultContext)responseObj );
|
||||
}
|
||||
else if (responseObj instanceof DocList) {
|
||||
ResultContext ctx = new BasicResultContext((DocList)responseObj, returnFields, null, null, req);
|
||||
writeDocuments(null, ctx );
|
||||
} else if (responseObj instanceof SolrDocumentList) {
|
||||
writeSolrDocumentList(null, (SolrDocumentList)responseObj, returnFields );
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeNamedList(String name, NamedList val) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeStartDocumentList(String name,
|
||||
long start, int size, long numFound, Float maxScore) throws IOException
|
||||
{
|
||||
// nothing
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeEndDocumentList() throws IOException
|
||||
{
|
||||
// nothing
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeArray(String name, Iterator val) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeDate(String name, Date val) throws IOException {
|
||||
writeDate(name, val.toInstant().toString());
|
||||
}
|
||||
}
|
||||
|
|
@ -59,6 +59,13 @@ public abstract class ReturnFields {
|
|||
*/
|
||||
public abstract Set<String> getRequestedFieldNames();
|
||||
|
||||
/**
|
||||
* The explicitly requested field names (includes pseudo fields)
|
||||
* <p>
|
||||
* @return Set of explicitly requested field names or <code>null</code> (no explict)
|
||||
*/
|
||||
public abstract Set<String> getExplicitlyRequestedFieldNames();
|
||||
|
||||
/**
|
||||
* Get the fields which have been renamed
|
||||
* @return a mapping of renamed fields
|
||||
|
|
|
|||
|
|
@ -16,6 +16,15 @@
|
|||
*/
|
||||
package org.apache.solr.search;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.lucene.queries.function.FunctionQuery;
|
||||
import org.apache.lucene.queries.function.ValueSource;
|
||||
|
|
@ -35,15 +44,6 @@ import org.apache.solr.response.transform.TransformerFactory;
|
|||
import org.apache.solr.response.transform.ValueSourceAugmenter;
|
||||
import org.apache.solr.search.SolrDocumentFetcher.RetrieveFieldsOptimizer;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* The default implementation of return fields parsing for Solr.
|
||||
*/
|
||||
|
|
@ -448,7 +448,15 @@ public class SolrReturnFields extends ReturnFields {
|
|||
|
||||
@Override
|
||||
public Set<String> getRequestedFieldNames() {
|
||||
if(_wantsAllFields || reqFieldNames==null || reqFieldNames.isEmpty()) {
|
||||
if(_wantsAllFields || reqFieldNames == null || reqFieldNames.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
return reqFieldNames;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getExplicitlyRequestedFieldNames() {
|
||||
if (reqFieldNames == null || reqFieldNames.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
return reqFieldNames;
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
|
|||
public static void createIndex() {
|
||||
assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt","2000-01-02T03:04:05Z"));
|
||||
assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output", "shouldbeunstored","foo"));
|
||||
assertU(adoc("id","3", "shouldbeunstored","foo"));
|
||||
assertU(adoc("id","3", "shouldbeunstored","foo", "foo_l", "1"));
|
||||
assertU(adoc("id","4", "amount_c", "1.50,EUR"));
|
||||
assertU(adoc("id","5", "store", "12.434,-134.1"));
|
||||
assertU(adoc("id","6", "pubyear_ii", "123", "store_iis", "12", "price_ff", "1.3"));
|
||||
|
|
@ -246,12 +246,19 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
|
|||
assertEquals("exists(shouldbeunstored),XXX", singleFuncLines[0] );
|
||||
assertEquals("false,1", singleFuncLines[1] );
|
||||
assertEquals("true,3", singleFuncLines[3] );
|
||||
|
||||
// pseudo-fields with * in fl
|
||||
txt = h.query(req("q","id:4", "wt","csv", "csv.header","true", "fl","*,YYY:[docid],FOO:amount_c"));
|
||||
lines = txt.split("\n");
|
||||
assertEquals(2, lines.length);
|
||||
assertEquals(sortHeader("foo_i,foo_l,FOO,foo_s,store,store_iis," +
|
||||
"v2_ss,pubyear_ii,foo_dt,foo_b,YYY,foo_d,id,amount_c,foo_f,v_ss"), sortHeader(lines[0]));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testForDVEnabledFields() throws Exception {
|
||||
// for dv enabled and useDocValueAsStored=true
|
||||
// returns pubyear_i, store_iis but not price_ff
|
||||
// returns pubyear_ii, store_iis but not price_ff
|
||||
String singleFuncText = h.query(req("q","id:6", "wt","csv", "csv.header","true"));
|
||||
String sortedHeader = sortHeader("amount_c,store,v_ss,foo_b,v2_ss,foo_f,foo_i,foo_d,foo_s,foo_dt,id,foo_l," +
|
||||
"pubyear_ii,store_iis");
|
||||
|
|
@ -283,6 +290,19 @@ public class TestCSVResponseWriter extends SolrTestCaseJ4 {
|
|||
assertEquals(2, singleFuncLines.length);
|
||||
assertEquals("price_ff", singleFuncLines[0]);
|
||||
assertEquals("1.3", singleFuncLines[1]);
|
||||
|
||||
// explicit price_ff with fl=*
|
||||
singleFuncText = h.query(req("q","id:6", "wt","csv", "csv.header","true", "fl", "*,price_ff"));
|
||||
sortedHeader = sortHeader("amount_c,store,v_ss,foo_b,v2_ss,foo_f,foo_i,foo_d,foo_s,foo_dt,id,foo_l," +
|
||||
"pubyear_ii,store_iis,price_ff");
|
||||
singleFuncLines = singleFuncText.split("\n");
|
||||
assertEquals(2, singleFuncLines.length);
|
||||
assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));
|
||||
actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))
|
||||
.filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))
|
||||
.collect(Collectors.toList());
|
||||
assertEquals(4, actualVal.size());
|
||||
assertTrue(actualVal.containsAll(Arrays.asList("6", "123", "12", "1.3")));
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue