mirror of https://github.com/apache/lucene.git
SOLR-1925: CSVResponseWriter and commons-csv update
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@979807 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley
parent
bed729c561
commit
6e30869fc9
|
|
@ -211,6 +211,9 @@ New Features
|
|||
Note that autoGeneratePhraseQueries="true" tends to not work well for non whitespace
|
||||
delimited languages. (yonik)
|
||||
|
||||
* SOLR-1925: Add CSVResponseWriter (use wt=csv) that returns the list of documents
|
||||
in CSV format. (Chris Mattmann, yonik)
|
||||
|
||||
|
||||
Optimizations
|
||||
----------------------
|
||||
|
|
|
|||
|
|
@ -1062,6 +1062,7 @@
|
|||
<queryResponseWriter name="php" class="solr.PHPResponseWriter"/>
|
||||
<queryResponseWriter name="phps" class="solr.PHPSerializedResponseWriter"/>
|
||||
<queryResponseWriter name="velocity" class="solr.VelocityResponseWriter"/>
|
||||
<queryResponseWriter name="csv" class="solr.CSVResponseWriter"/>
|
||||
|
||||
Custom response writers can be declared as needed...
|
||||
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
AnyObjectId[f80348dfa0b59f0840c25d1b8c25d1490d1eaf51] was removed in git history.
|
||||
Apache SVN contains full history.
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
AnyObjectId[8439e6f1a8b1d82943f84688b8086869255eda86] was removed in git history.
|
||||
Apache SVN contains full history.
|
||||
|
|
@ -16,6 +16,8 @@ package org.apache.solr.common.util;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.text.DateFormat;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
|
|
@ -196,5 +198,66 @@ public class DateUtil {
|
|||
}
|
||||
}
|
||||
|
||||
/** Formats the date and returns the calendar instance that was used (which may be reused) */
|
||||
public static Calendar formatDate(Date date, Calendar cal, Appendable out) throws IOException {
|
||||
// using a stringBuilder for numbers can be nice since
|
||||
// a temporary string isn't used (it's added directly to the
|
||||
// builder's buffer.
|
||||
|
||||
StringBuilder sb = out instanceof StringBuilder ? (StringBuilder)out : new StringBuilder();
|
||||
if (cal==null) cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.US);
|
||||
cal.setTime(date);
|
||||
|
||||
int i = cal.get(Calendar.YEAR);
|
||||
sb.append(i);
|
||||
sb.append('-');
|
||||
i = cal.get(Calendar.MONTH) + 1; // 0 based, so add 1
|
||||
if (i<10) sb.append('0');
|
||||
sb.append(i);
|
||||
sb.append('-');
|
||||
i=cal.get(Calendar.DAY_OF_MONTH);
|
||||
if (i<10) sb.append('0');
|
||||
sb.append(i);
|
||||
sb.append('T');
|
||||
i=cal.get(Calendar.HOUR_OF_DAY); // 24 hour time format
|
||||
if (i<10) sb.append('0');
|
||||
sb.append(i);
|
||||
sb.append(':');
|
||||
i=cal.get(Calendar.MINUTE);
|
||||
if (i<10) sb.append('0');
|
||||
sb.append(i);
|
||||
sb.append(':');
|
||||
i=cal.get(Calendar.SECOND);
|
||||
if (i<10) sb.append('0');
|
||||
sb.append(i);
|
||||
i=cal.get(Calendar.MILLISECOND);
|
||||
if (i != 0) {
|
||||
sb.append('.');
|
||||
if (i<100) sb.append('0');
|
||||
if (i<10) sb.append('0');
|
||||
sb.append(i);
|
||||
|
||||
// handle canonical format specifying fractional
|
||||
// seconds shall not end in '0'. Given the slowness of
|
||||
// integer div/mod, simply checking the last character
|
||||
// is probably the fastest way to check.
|
||||
int lastIdx = sb.length()-1;
|
||||
if (sb.charAt(lastIdx)=='0') {
|
||||
lastIdx--;
|
||||
if (sb.charAt(lastIdx)=='0') {
|
||||
lastIdx--;
|
||||
}
|
||||
sb.setLength(lastIdx+1);
|
||||
}
|
||||
|
||||
}
|
||||
sb.append('Z');
|
||||
|
||||
if (out != sb)
|
||||
out.append(sb);
|
||||
|
||||
return cal;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -27,9 +27,9 @@ public class FastWriter extends Writer {
|
|||
// use default BUFSIZE of BufferedWriter so if we wrap that
|
||||
// it won't cause double buffering.
|
||||
private static final int BUFSIZE = 8192;
|
||||
private final Writer sink;
|
||||
private final char[] buf;
|
||||
private int pos;
|
||||
protected final Writer sink;
|
||||
protected final char[] buf;
|
||||
protected int pos;
|
||||
|
||||
public FastWriter(Writer w) {
|
||||
this(w, new char[BUFSIZE], 0);
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import org.apache.solr.handler.component.*;
|
|||
import org.apache.solr.highlight.DefaultSolrHighlighter;
|
||||
import org.apache.solr.highlight.SolrHighlighter;
|
||||
import org.apache.solr.request.*;
|
||||
import org.apache.solr.response.*;
|
||||
import org.apache.solr.response.BinaryResponseWriter;
|
||||
import org.apache.solr.response.JSONResponseWriter;
|
||||
import org.apache.solr.response.PHPResponseWriter;
|
||||
|
|
@ -43,7 +44,6 @@ import org.apache.solr.response.QueryResponseWriter;
|
|||
import org.apache.solr.response.RawResponseWriter;
|
||||
import org.apache.solr.response.RubyResponseWriter;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.response.VelocityResponseWriter;
|
||||
import org.apache.solr.response.XMLResponseWriter;
|
||||
import org.apache.solr.schema.IndexSchema;
|
||||
import org.apache.solr.search.QParserPlugin;
|
||||
|
|
@ -1408,6 +1408,7 @@ public final class SolrCore implements SolrInfoMBean {
|
|||
m.put("raw", new RawResponseWriter());
|
||||
m.put("javabin", new BinaryResponseWriter());
|
||||
m.put("velocity", new VelocityResponseWriter());
|
||||
m.put("csv", new CSVResponseWriter());
|
||||
DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -70,18 +70,18 @@ public class CSVRequestHandler extends ContentStreamHandlerBase {
|
|||
|
||||
|
||||
abstract class CSVLoader extends ContentStreamLoader {
|
||||
static String SEPARATOR="separator";
|
||||
static String FIELDNAMES="fieldnames";
|
||||
static String HEADER="header";
|
||||
static String SKIP="skip";
|
||||
static String SKIPLINES="skipLines";
|
||||
static String MAP="map";
|
||||
static String TRIM="trim";
|
||||
static String EMPTY="keepEmpty";
|
||||
static String SPLIT="split";
|
||||
static String ENCAPSULATOR="encapsulator";
|
||||
static String ESCAPE="escape";
|
||||
static String OVERWRITE="overwrite";
|
||||
public static final String SEPARATOR="separator";
|
||||
public static final String FIELDNAMES="fieldnames";
|
||||
public static final String HEADER="header";
|
||||
public static final String SKIP="skip";
|
||||
public static final String SKIPLINES="skipLines";
|
||||
public static final String MAP="map";
|
||||
public static final String TRIM="trim";
|
||||
public static final String EMPTY="keepEmpty";
|
||||
public static final String SPLIT="split";
|
||||
public static final String ENCAPSULATOR="encapsulator";
|
||||
public static final String ESCAPE="escape";
|
||||
public static final String OVERWRITE="overwrite";
|
||||
|
||||
private static Pattern colonSplit = Pattern.compile(":");
|
||||
private static Pattern commaSplit = Pattern.compile(",");
|
||||
|
|
@ -219,7 +219,7 @@ abstract class CSVLoader extends ContentStreamLoader {
|
|||
|
||||
// if only encapsulator or escape is set, disable the other escaping mechanism
|
||||
if (encapsulator == null && escape != null) {
|
||||
strategy.setEncapsulator((char)-2); // TODO: add CSVStrategy.ENCAPSULATOR_DISABLED
|
||||
strategy.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED);
|
||||
strategy.setEscape(escape.charAt(0));
|
||||
} else {
|
||||
if (encapsulator != null) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,536 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.response;
|
||||
|
||||
import org.apache.commons.csv.CSVPrinter;
|
||||
import org.apache.commons.csv.CSVStrategy;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Fieldable;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.DateUtil;
|
||||
import org.apache.solr.common.util.FastWriter;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.schema.StrField;
|
||||
import org.apache.solr.search.DocIterator;
|
||||
import org.apache.solr.search.DocList;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
|
||||
import java.io.CharArrayWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* @version $Id$
|
||||
*/
|
||||
|
||||
public class CSVResponseWriter implements QueryResponseWriter {
|
||||
|
||||
public void init(NamedList n) {
|
||||
}
|
||||
|
||||
public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
|
||||
CSVWriter w = new CSVWriter(writer, req, rsp);
|
||||
try {
|
||||
w.writeResponse();
|
||||
} finally {
|
||||
w.close();
|
||||
}
|
||||
}
|
||||
|
||||
public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {
|
||||
// using the text/plain allows this to be viewed in the browser easily
|
||||
return CONTENT_TYPE_TEXT_UTF8;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class CSVWriter extends TextResponseWriter {
|
||||
static String SEPARATOR = "separator";
|
||||
static String ENCAPSULATOR = "encapsulator";
|
||||
static String ESCAPE = "escape";
|
||||
|
||||
static String CSV = "csv.";
|
||||
static String CSV_SEPARATOR = CSV + SEPARATOR;
|
||||
static String CSV_ENCAPSULATOR = CSV + ENCAPSULATOR;
|
||||
static String CSV_ESCAPE = CSV + ESCAPE;
|
||||
|
||||
static String MV = CSV+"mv.";
|
||||
static String MV_SEPARATOR = MV + SEPARATOR;
|
||||
static String MV_ENCAPSULATOR = MV + ENCAPSULATOR;
|
||||
static String MV_ESCAPE = MV + ESCAPE;
|
||||
|
||||
static String CSV_NULL = CSV + "null";
|
||||
static String CSV_HEADER = CSV + "header";
|
||||
static String CSV_NEWLINE = CSV + "newline";
|
||||
|
||||
char[] sharedCSVBuf = new char[8192];
|
||||
|
||||
// prevent each instance from creating it's own buffer
|
||||
class CSVSharedBufPrinter extends CSVPrinter {
|
||||
public CSVSharedBufPrinter(Writer out, CSVStrategy strategy) {
|
||||
super(out, strategy);
|
||||
super.buf = sharedCSVBuf;
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
super.newLine = true;
|
||||
// update our shared buf in case a new bigger one was allocated
|
||||
sharedCSVBuf = super.buf;
|
||||
}
|
||||
}
|
||||
|
||||
// allows access to internal buf w/o copying it
|
||||
static class OpenCharArrayWriter extends CharArrayWriter {
|
||||
public char[] getInternalBuf() { return buf; }
|
||||
}
|
||||
|
||||
// Writes all data to a char array,
|
||||
// allows access to internal buffer, and allows fast resetting.
|
||||
static class ResettableFastWriter extends FastWriter {
|
||||
OpenCharArrayWriter cw = new OpenCharArrayWriter();
|
||||
char[] result;
|
||||
int resultLen;
|
||||
|
||||
public ResettableFastWriter() {
|
||||
super(new OpenCharArrayWriter());
|
||||
cw = (OpenCharArrayWriter)sink;
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
cw.reset();
|
||||
pos=0;
|
||||
}
|
||||
|
||||
public void freeze() throws IOException {
|
||||
if (cw.size() > 0) {
|
||||
flush();
|
||||
result = cw.getInternalBuf();
|
||||
resultLen = cw.size();
|
||||
} else {
|
||||
result = buf;
|
||||
resultLen = pos;
|
||||
}
|
||||
}
|
||||
|
||||
public int getFrozenSize() { return resultLen; }
|
||||
public char[] getFrozenBuf() { return result; }
|
||||
}
|
||||
|
||||
|
||||
static class CSVField {
|
||||
String name;
|
||||
SchemaField sf;
|
||||
CSVSharedBufPrinter mvPrinter; // printer used to encode multiple values in a single CSV value
|
||||
|
||||
// used to collect values
|
||||
List<Fieldable> values = new ArrayList<Fieldable>(1); // low starting amount in case there are many fields
|
||||
int tmp;
|
||||
}
|
||||
|
||||
int pass;
|
||||
Map<String,CSVField> csvFields = new LinkedHashMap<String,CSVField>();
|
||||
|
||||
Calendar cal; // for formatting date objects
|
||||
|
||||
CSVStrategy strategy; // strategy for encoding the fields of documents
|
||||
CSVPrinter printer;
|
||||
ResettableFastWriter mvWriter = new ResettableFastWriter(); // writer used for multi-valued fields
|
||||
|
||||
String NullValue;
|
||||
boolean returnScore = false;
|
||||
|
||||
|
||||
public CSVWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
|
||||
super(writer, req, rsp);
|
||||
}
|
||||
|
||||
public void writeResponse() throws IOException {
|
||||
SolrParams params = req.getParams();
|
||||
|
||||
strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true);
|
||||
CSVStrategy strat = strategy;
|
||||
|
||||
String sep = params.get(CSV_SEPARATOR);
|
||||
if (sep!=null) {
|
||||
if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid separator:'"+sep+"'");
|
||||
strat.setDelimiter(sep.charAt(0));
|
||||
}
|
||||
|
||||
String nl = params.get(CSV_NEWLINE);
|
||||
if (nl!=null) {
|
||||
if (nl.length()==0) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid newline:'"+nl+"'");
|
||||
strat.setPrinterNewline(nl);
|
||||
}
|
||||
|
||||
String encapsulator = params.get(CSV_ENCAPSULATOR);
|
||||
String escape = params.get(CSV_ESCAPE);
|
||||
if (encapsulator!=null) {
|
||||
if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid encapsulator:'"+encapsulator+"'");
|
||||
strat.setEncapsulator(encapsulator.charAt(0));
|
||||
}
|
||||
|
||||
if (escape!=null) {
|
||||
if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid escape:'"+escape+"'");
|
||||
strat.setEscape(escape.charAt(0));
|
||||
if (encapsulator == null) {
|
||||
strat.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED);
|
||||
}
|
||||
}
|
||||
|
||||
if (strat.getEscape() == '\\') {
|
||||
// If the escape is the standard backslash, then also enable
|
||||
// unicode escapes (it's harmless since 'u' would not otherwise
|
||||
// be escaped.
|
||||
strat.setUnicodeEscapeInterpretation(true);
|
||||
}
|
||||
|
||||
printer = new CSVPrinter(writer, strategy);
|
||||
|
||||
|
||||
CSVStrategy mvStrategy = new CSVStrategy(strategy.getDelimiter(), CSVStrategy.ENCAPSULATOR_DISABLED, CSVStrategy.COMMENTS_DISABLED, '\\', false, false, false, false);
|
||||
strat = mvStrategy;
|
||||
|
||||
sep = params.get(MV_SEPARATOR);
|
||||
if (sep!=null) {
|
||||
if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv separator:'"+sep+"'");
|
||||
strat.setDelimiter(sep.charAt(0));
|
||||
}
|
||||
|
||||
encapsulator = params.get(MV_ENCAPSULATOR);
|
||||
escape = params.get(MV_ESCAPE);
|
||||
|
||||
if (encapsulator!=null) {
|
||||
if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv encapsulator:'"+encapsulator+"'");
|
||||
strat.setEncapsulator(encapsulator.charAt(0));
|
||||
if (escape == null) {
|
||||
strat.setEscape(CSVStrategy.ESCAPE_DISABLED);
|
||||
}
|
||||
}
|
||||
|
||||
escape = params.get(MV_ESCAPE);
|
||||
if (escape!=null) {
|
||||
if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv escape:'"+escape+"'");
|
||||
strat.setEscape(escape.charAt(0));
|
||||
// encapsulator will already be disabled if it wasn't specified
|
||||
}
|
||||
|
||||
returnScore = returnFields != null && returnFields.contains("score");
|
||||
boolean needListOfFields = returnFields==null || returnFields.size()==0 || (returnFields.size()==1 && returnScore) || returnFields.contains("*");
|
||||
Collection<String> fields = returnFields;
|
||||
|
||||
Object responseObj = rsp.getValues().get("response");
|
||||
if (needListOfFields) {
|
||||
if (responseObj instanceof SolrDocumentList) {
|
||||
// get the list of fields from the SolrDocumentList
|
||||
fields = new LinkedHashSet<String>();
|
||||
for (SolrDocument sdoc: (SolrDocumentList)responseObj) {
|
||||
fields.addAll(sdoc.getFieldNames());
|
||||
}
|
||||
} else {
|
||||
// get the list of fields from the index
|
||||
fields = req.getSearcher().getFieldNames();
|
||||
}
|
||||
if (returnScore) {
|
||||
fields.add("score");
|
||||
} else {
|
||||
fields.remove("score");
|
||||
}
|
||||
}
|
||||
|
||||
CSVSharedBufPrinter csvPrinterMV = new CSVSharedBufPrinter(mvWriter, mvStrategy);
|
||||
|
||||
for (String field : fields) {
|
||||
if (field.equals("score")) {
|
||||
CSVField csvField = new CSVField();
|
||||
csvField.name = "score";
|
||||
csvFields.put("score", csvField);
|
||||
continue;
|
||||
}
|
||||
|
||||
SchemaField sf = schema.getFieldOrNull(field);
|
||||
if (sf == null) {
|
||||
FieldType ft = new StrField();
|
||||
sf = new SchemaField(field, ft);
|
||||
}
|
||||
|
||||
// if we got the list of fields from the index, only list stored fields
|
||||
if (returnFields==null && sf != null && !sf.stored()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// check for per-field overrides
|
||||
sep = params.get("f." + field + '.' + CSV_SEPARATOR);
|
||||
encapsulator = params.get("f." + field + '.' + CSV_ENCAPSULATOR);
|
||||
escape = params.get("f." + field + '.' + CSV_ESCAPE);
|
||||
|
||||
CSVSharedBufPrinter csvPrinter = csvPrinterMV;
|
||||
if (sep != null || encapsulator != null || escape != null) {
|
||||
// create a new strategy + printer if there were any per-field overrides
|
||||
strat = (CSVStrategy)mvStrategy.clone();
|
||||
if (sep!=null) {
|
||||
if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv separator:'"+sep+"'");
|
||||
strat.setDelimiter(sep.charAt(0));
|
||||
}
|
||||
if (encapsulator!=null) {
|
||||
if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv encapsulator:'"+encapsulator+"'");
|
||||
strat.setEncapsulator(encapsulator.charAt(0));
|
||||
if (escape == null) {
|
||||
strat.setEscape(CSVStrategy.ESCAPE_DISABLED);
|
||||
}
|
||||
}
|
||||
if (escape!=null) {
|
||||
if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv escape:'"+escape+"'");
|
||||
strat.setEscape(escape.charAt(0));
|
||||
if (encapsulator == null) {
|
||||
strat.setEncapsulator(CSVStrategy.ENCAPSULATOR_DISABLED);
|
||||
}
|
||||
}
|
||||
csvPrinter = new CSVSharedBufPrinter(mvWriter, strat);
|
||||
}
|
||||
|
||||
|
||||
CSVField csvField = new CSVField();
|
||||
csvField.name = field;
|
||||
csvField.sf = sf;
|
||||
csvField.mvPrinter = csvPrinter;
|
||||
csvFields.put(field, csvField);
|
||||
}
|
||||
|
||||
NullValue = params.get(CSV_NULL, "");
|
||||
|
||||
if (params.getBool(CSV_HEADER, true)) {
|
||||
for (CSVField csvField : csvFields.values()) {
|
||||
printer.print(csvField.name);
|
||||
}
|
||||
printer.println();
|
||||
}
|
||||
|
||||
|
||||
if (responseObj instanceof DocList) {
|
||||
writeDocList(null, (DocList)responseObj, null, null);
|
||||
} else if (responseObj instanceof SolrDocumentList) {
|
||||
writeSolrDocumentList(null, (SolrDocumentList)responseObj, null, null);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (printer != null) printer.flush();
|
||||
super.close();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeNamedList(String name, NamedList val) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeDoc(String name, Document doc, Set<String> returnFields, float score, boolean includeScore) throws IOException {
|
||||
pass++;
|
||||
|
||||
for (Fieldable field: doc.getFields()) {
|
||||
CSVField csvField = csvFields.get(field.name());
|
||||
if (csvField == null) continue;
|
||||
if (csvField.tmp != pass) {
|
||||
csvField.tmp = pass;
|
||||
csvField.values.clear();
|
||||
}
|
||||
csvField.values.add(field);
|
||||
}
|
||||
|
||||
for (CSVField csvField : csvFields.values()) {
|
||||
if (csvField.name.equals("score")) {
|
||||
writeFloat("score", score);
|
||||
continue;
|
||||
}
|
||||
if (csvField.tmp != pass) {
|
||||
writeNull(csvField.name);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (csvField.sf.multiValued() || csvField.values.size() > 1) {
|
||||
mvWriter.reset();
|
||||
csvField.mvPrinter.reset();
|
||||
// switch the printer to use the multi-valued one
|
||||
CSVPrinter tmp = printer;
|
||||
printer = csvField.mvPrinter;
|
||||
for (Fieldable fval : csvField.values) {
|
||||
csvField.sf.getType().write(this, csvField.name, fval);
|
||||
}
|
||||
printer = tmp; // restore the original printer
|
||||
|
||||
mvWriter.freeze();
|
||||
printer.print(mvWriter.getFrozenBuf(), 0, mvWriter.getFrozenSize(), true);
|
||||
} else {
|
||||
assert csvField.values.size() == 1;
|
||||
csvField.sf.getType().write(this,csvField.name,csvField.values.get(0));
|
||||
}
|
||||
}
|
||||
|
||||
printer.println();
|
||||
}
|
||||
|
||||
//NOTE: a document cannot currently contain another document
|
||||
List tmpList;
|
||||
@Override
|
||||
public void writeSolrDocument(String name, SolrDocument doc, Set<String> returnFields, Map pseudoFields) throws IOException {
|
||||
if (tmpList == null) {
|
||||
tmpList = new ArrayList(1);
|
||||
tmpList.add(null);
|
||||
}
|
||||
|
||||
for (CSVField csvField : csvFields.values()) {
|
||||
Object val = doc.getFieldValue(csvField.name);
|
||||
int nVals = val instanceof Collection ? ((Collection)val).size() : (val==null ? 0 : 1);
|
||||
if (nVals == 0) {
|
||||
writeNull(csvField.name);
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((csvField.sf != null && csvField.sf.multiValued()) || nVals > 1) {
|
||||
Collection values;
|
||||
// normalize to a collection
|
||||
if (val instanceof Collection) {
|
||||
values = (Collection)val;
|
||||
} else {
|
||||
tmpList.set(0, val);
|
||||
values = tmpList;
|
||||
}
|
||||
|
||||
mvWriter.reset();
|
||||
csvField.mvPrinter.reset();
|
||||
// switch the printer to use the multi-valued one
|
||||
CSVPrinter tmp = printer;
|
||||
printer = csvField.mvPrinter;
|
||||
for (Object fval : values) {
|
||||
writeVal(csvField.name, fval);
|
||||
}
|
||||
printer = tmp; // restore the original printer
|
||||
|
||||
mvWriter.freeze();
|
||||
printer.print(mvWriter.getFrozenBuf(), 0, mvWriter.getFrozenSize(), true);
|
||||
|
||||
} else {
|
||||
// normalize to first value
|
||||
if (val instanceof Collection) {
|
||||
Collection values = (Collection)val;
|
||||
val = values.iterator().next();
|
||||
}
|
||||
writeVal(csvField.name, val);
|
||||
}
|
||||
}
|
||||
|
||||
printer.println();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeDocList(String name, DocList ids, Set<String> fields, Map otherFields) throws IOException {
|
||||
int sz=ids.size();
|
||||
SolrIndexSearcher searcher = req.getSearcher();
|
||||
DocIterator iterator = ids.iterator();
|
||||
for (int i=0; i<sz; i++) {
|
||||
int id = iterator.nextDoc();
|
||||
Document doc = searcher.doc(id, fields);
|
||||
writeDoc(null, doc, fields, (returnScore ? iterator.score() : 0.0f), returnScore);
|
||||
}
|
||||
}
|
||||
|
||||
Map scoreMap = new HashMap(1);
|
||||
@Override
|
||||
public void writeSolrDocumentList(String name, SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException {
|
||||
for (SolrDocument doc : docs) {
|
||||
writeSolrDocument(name, doc, fields, otherFields);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeStr(String name, String val, boolean needsEscaping) throws IOException {
|
||||
printer.print(val, needsEscaping);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeArray(String name, Object[] val) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeArray(String name, Iterator val) throws IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeNull(String name) throws IOException {
|
||||
printer.print(NullValue);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeInt(String name, String val) throws IOException {
|
||||
printer.print(val, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeLong(String name, String val) throws IOException {
|
||||
printer.print(val, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeBool(String name, String val) throws IOException {
|
||||
printer.print(val, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeFloat(String name, String val) throws IOException {
|
||||
printer.print(val, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeDouble(String name, String val) throws IOException {
|
||||
printer.print(val, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeDate(String name, Date val) throws IOException {
|
||||
StringBuilder sb = new StringBuilder(25);
|
||||
cal = DateUtil.formatDate(val, cal, sb);
|
||||
writeDate(name, sb.toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeDate(String name, String val) throws IOException {
|
||||
printer.print(val, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeShort(String name, String val) throws IOException {
|
||||
printer.print(val, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeByte(String name, String val) throws IOException {
|
||||
printer.print(val, false);
|
||||
}
|
||||
}
|
||||
|
|
@ -205,7 +205,7 @@ public class SolrPluginUtils {
|
|||
// TODO - should field order be maintained?
|
||||
String[] flst = split(fl);
|
||||
if (flst.length > 0 && !(flst.length==1 && flst[0].length()==0)) {
|
||||
Set<String> set = new HashSet<String>();
|
||||
Set<String> set = new LinkedHashSet<String>();
|
||||
for (String fname : flst) {
|
||||
if("score".equalsIgnoreCase(fname))
|
||||
flags |= SolrIndexSearcher.GET_SCORES;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,165 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.response;
|
||||
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.util.DateUtil;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.util.SolrPluginUtils;
|
||||
import org.junit.*;
|
||||
|
||||
import java.io.StringWriter;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class TestCSVResponseWriter extends SolrTestCaseJ4 {
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
initCore("solrconfig.xml","schema12.xml");
|
||||
createIndex();
|
||||
}
|
||||
|
||||
public static void createIndex() {
|
||||
assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt","2000-01-02T03:04:05Z"));
|
||||
assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output"));
|
||||
assertU(commit());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testCSVOutput() throws Exception {
|
||||
// test our basic types,and that fields come back in the requested order
|
||||
assertEquals("id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt\n1,hi,-1,12345678987654321,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n"
|
||||
, h.query(req("q","id:1", "wt","csv", "fl","id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt")));
|
||||
|
||||
// test retrieving score, csv.header
|
||||
assertEquals("1,0.0,hi\n"
|
||||
, h.query(req("q","id:1^0", "wt","csv", "csv.header","false", "fl","id,score,foo_s")));
|
||||
|
||||
// test multivalued
|
||||
assertEquals("2,\"hi,there\"\n"
|
||||
, h.query(req("q","id:2", "wt","csv", "csv.header","false", "fl","id,v_ss")));
|
||||
|
||||
// test separator change
|
||||
assertEquals("2|\"hi|there\"\n"
|
||||
, h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.separator","|", "fl","id,v_ss")));
|
||||
|
||||
// test mv separator change
|
||||
assertEquals("2,hi|there\n"
|
||||
, h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "fl","id,v_ss")));
|
||||
|
||||
// test mv separator change for a single field
|
||||
assertEquals("2,hi|there,nice:output\n"
|
||||
, h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "f.v2_ss.csv.separator",":", "fl","id,v_ss,v2_ss")));
|
||||
|
||||
// test retrieving fields from index
|
||||
String result = h.query(req("q","*:*", "wt","csv", "csv.header","true", "fl","*,score"));
|
||||
for (String field : "id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss,score".split(",")) {
|
||||
assertTrue(result.indexOf(field) >= 0);
|
||||
}
|
||||
|
||||
// test null values
|
||||
assertEquals("2,,hi|there\n"
|
||||
, h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "fl","id,foo_s,v_ss")));
|
||||
|
||||
// test alternate null value
|
||||
assertEquals("2,NULL,hi|there\n"
|
||||
, h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "csv.null","NULL","fl","id,foo_s,v_ss")));
|
||||
|
||||
// test alternate newline
|
||||
assertEquals("2,\"hi,there\"\r\n"
|
||||
, h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.newline","\r\n", "fl","id,v_ss")));
|
||||
|
||||
// test alternate encapsulator
|
||||
assertEquals("2,'hi,there'\n"
|
||||
, h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.encapsulator","'", "fl","id,v_ss")));
|
||||
|
||||
// test using escape instead of encapsulator
|
||||
assertEquals("2,hi\\,there\n"
|
||||
, h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.escape","\\", "fl","id,v_ss")));
|
||||
|
||||
// test multiple lines
|
||||
assertEquals("1,,hi\n2,\"hi,there\",\n"
|
||||
, h.query(req("q","id:[1 TO 2]", "wt","csv", "csv.header","false", "fl","id,v_ss,foo_s")));
|
||||
|
||||
|
||||
// now test SolrDocumentList
|
||||
SolrDocument d = new SolrDocument();
|
||||
SolrDocument d1 = d;
|
||||
d.addField("id","1");
|
||||
d.addField("foo_i",-1);
|
||||
d.addField("foo_s","hi");
|
||||
d.addField("foo_l","12345678987654321L");
|
||||
d.addField("foo_b",false);
|
||||
d.addField("foo_f",1.414f);
|
||||
d.addField("foo_d",-1.0E300);
|
||||
d.addField("foo_dt", DateUtil.parseDate("2000-01-02T03:04:05Z"));
|
||||
d.addField("score", "2.718");
|
||||
|
||||
d = new SolrDocument();
|
||||
SolrDocument d2 = d;
|
||||
d.addField("id","2");
|
||||
d.addField("v_ss","hi");
|
||||
d.addField("v_ss","there");
|
||||
d.addField("v2_ss","nice");
|
||||
d.addField("v2_ss","output");
|
||||
d.addField("score", "89.83");
|
||||
|
||||
SolrDocumentList sdl = new SolrDocumentList();
|
||||
sdl.add(d1);
|
||||
sdl.add(d2);
|
||||
|
||||
SolrQueryRequest req = req("q","*:*");
|
||||
SolrQueryResponse rsp = new SolrQueryResponse();
|
||||
rsp.add("response", sdl);
|
||||
QueryResponseWriter w = new CSVResponseWriter();
|
||||
|
||||
SolrPluginUtils.setReturnFields("id,foo_s", rsp);
|
||||
StringWriter buf = new StringWriter();
|
||||
w.write(buf, req, rsp);
|
||||
assertEquals("id,foo_s\n1,hi\n2,\n", buf.toString());
|
||||
|
||||
// try scores
|
||||
SolrPluginUtils.setReturnFields("id,score,foo_s", rsp);
|
||||
buf = new StringWriter();
|
||||
w.write(buf, req, rsp);
|
||||
assertEquals("id,score,foo_s\n1,2.718,hi\n2,89.83,\n", buf.toString());
|
||||
|
||||
// get field values from docs... should be ordered and not include score unless requested
|
||||
SolrPluginUtils.setReturnFields("*", rsp);
|
||||
buf = new StringWriter();
|
||||
w.write(buf, req, rsp);
|
||||
assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss\n" +
|
||||
"1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z,,\n" +
|
||||
"2,,,,,,,,\"hi,there\",\"nice,output\"\n",
|
||||
buf.toString());
|
||||
|
||||
|
||||
// get field values and scores - just check that the scores are there... we don't guarantee where
|
||||
SolrPluginUtils.setReturnFields("*,score", rsp);
|
||||
buf = new StringWriter();
|
||||
w.write(buf, req, rsp);
|
||||
String s = buf.toString();
|
||||
assertTrue(s.indexOf("score") >=0 && s.indexOf("2.718") > 0 && s.indexOf("89.83") > 0 );
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue