From 6e30869fc9ca74d88dec20b8f76468530f1ddaf1 Mon Sep 17 00:00:00 2001 From: Yonik Seeley Date: Tue, 27 Jul 2010 19:06:39 +0000 Subject: [PATCH] SOLR-1925: CSVResponseWriter and commons-csv update git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@979807 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 5 +- solr/example/solr/conf/solrconfig.xml | 1 + solr/lib/commons-csv-1.0-SNAPSHOT-r609327.jar | 2 - solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar | 2 + .../org/apache/solr/common/util/DateUtil.java | 63 ++ .../apache/solr/common/util/FastWriter.java | 6 +- .../java/org/apache/solr/core/SolrCore.java | 3 +- .../solr/handler/CSVRequestHandler.java | 26 +- .../solr/response/CSVResponseWriter.java | 536 ++++++++++++++++++ .../org/apache/solr/util/SolrPluginUtils.java | 2 +- .../solr/response/TestCSVResponseWriter.java | 165 ++++++ 11 files changed, 790 insertions(+), 21 deletions(-) delete mode 100755 solr/lib/commons-csv-1.0-SNAPSHOT-r609327.jar create mode 100644 solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar create mode 100755 solr/src/java/org/apache/solr/response/CSVResponseWriter.java create mode 100644 solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 605b4709660..daa10104119 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -210,7 +210,10 @@ New Features will cause the parser to generate text:"pdp 11" rather than (text:PDP OR text:11). Note that autoGeneratePhraseQueries="true" tends to not work well for non whitespace delimited languages. (yonik) - + +* SOLR-1925: Add CSVResponseWriter (use wt=csv) that returns the list of documents + in CSV format. (Chris Mattmann, yonik) + Optimizations ---------------------- diff --git a/solr/example/solr/conf/solrconfig.xml b/solr/example/solr/conf/solrconfig.xml index 42e5631cb06..27ca190e909 100755 --- a/solr/example/solr/conf/solrconfig.xml +++ b/solr/example/solr/conf/solrconfig.xml @@ -1062,6 +1062,7 @@ + Custom response writers can be declared as needed... diff --git a/solr/lib/commons-csv-1.0-SNAPSHOT-r609327.jar b/solr/lib/commons-csv-1.0-SNAPSHOT-r609327.jar deleted file mode 100755 index 46f905e1ac7..00000000000 --- a/solr/lib/commons-csv-1.0-SNAPSHOT-r609327.jar +++ /dev/null @@ -1,2 +0,0 @@ -AnyObjectId[f80348dfa0b59f0840c25d1b8c25d1490d1eaf51] was removed in git history. -Apache SVN contains full history. \ No newline at end of file diff --git a/solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar b/solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar new file mode 100644 index 00000000000..c048f95eac3 --- /dev/null +++ b/solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar @@ -0,0 +1,2 @@ +AnyObjectId[8439e6f1a8b1d82943f84688b8086869255eda86] was removed in git history. +Apache SVN contains full history. \ No newline at end of file diff --git a/solr/src/common/org/apache/solr/common/util/DateUtil.java b/solr/src/common/org/apache/solr/common/util/DateUtil.java index ded022606de..414d7b27b02 100644 --- a/solr/src/common/org/apache/solr/common/util/DateUtil.java +++ b/solr/src/common/org/apache/solr/common/util/DateUtil.java @@ -16,6 +16,8 @@ package org.apache.solr.common.util; * limitations under the License. */ +import java.io.IOException; +import java.io.Writer; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -196,5 +198,66 @@ public class DateUtil { } } + /** Formats the date and returns the calendar instance that was used (which may be reused) */ + public static Calendar formatDate(Date date, Calendar cal, Appendable out) throws IOException { + // using a stringBuilder for numbers can be nice since + // a temporary string isn't used (it's added directly to the + // builder's buffer. + + StringBuilder sb = out instanceof StringBuilder ? (StringBuilder)out : new StringBuilder(); + if (cal==null) cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.US); + cal.setTime(date); + + int i = cal.get(Calendar.YEAR); + sb.append(i); + sb.append('-'); + i = cal.get(Calendar.MONTH) + 1; // 0 based, so add 1 + if (i<10) sb.append('0'); + sb.append(i); + sb.append('-'); + i=cal.get(Calendar.DAY_OF_MONTH); + if (i<10) sb.append('0'); + sb.append(i); + sb.append('T'); + i=cal.get(Calendar.HOUR_OF_DAY); // 24 hour time format + if (i<10) sb.append('0'); + sb.append(i); + sb.append(':'); + i=cal.get(Calendar.MINUTE); + if (i<10) sb.append('0'); + sb.append(i); + sb.append(':'); + i=cal.get(Calendar.SECOND); + if (i<10) sb.append('0'); + sb.append(i); + i=cal.get(Calendar.MILLISECOND); + if (i != 0) { + sb.append('.'); + if (i<100) sb.append('0'); + if (i<10) sb.append('0'); + sb.append(i); + + // handle canonical format specifying fractional + // seconds shall not end in '0'. Given the slowness of + // integer div/mod, simply checking the last character + // is probably the fastest way to check. + int lastIdx = sb.length()-1; + if (sb.charAt(lastIdx)=='0') { + lastIdx--; + if (sb.charAt(lastIdx)=='0') { + lastIdx--; + } + sb.setLength(lastIdx+1); + } + + } + sb.append('Z'); + + if (out != sb) + out.append(sb); + + return cal; + } + } \ No newline at end of file diff --git a/solr/src/common/org/apache/solr/common/util/FastWriter.java b/solr/src/common/org/apache/solr/common/util/FastWriter.java index 45f06e4f2f6..090c3ab2b87 100755 --- a/solr/src/common/org/apache/solr/common/util/FastWriter.java +++ b/solr/src/common/org/apache/solr/common/util/FastWriter.java @@ -27,9 +27,9 @@ public class FastWriter extends Writer { // use default BUFSIZE of BufferedWriter so if we wrap that // it won't cause double buffering. private static final int BUFSIZE = 8192; - private final Writer sink; - private final char[] buf; - private int pos; + protected final Writer sink; + protected final char[] buf; + protected int pos; public FastWriter(Writer w) { this(w, new char[BUFSIZE], 0); diff --git a/solr/src/java/org/apache/solr/core/SolrCore.java b/solr/src/java/org/apache/solr/core/SolrCore.java index 1f657fecb5d..378a59aa3d7 100644 --- a/solr/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/src/java/org/apache/solr/core/SolrCore.java @@ -34,6 +34,7 @@ import org.apache.solr.handler.component.*; import org.apache.solr.highlight.DefaultSolrHighlighter; import org.apache.solr.highlight.SolrHighlighter; import org.apache.solr.request.*; +import org.apache.solr.response.*; import org.apache.solr.response.BinaryResponseWriter; import org.apache.solr.response.JSONResponseWriter; import org.apache.solr.response.PHPResponseWriter; @@ -43,7 +44,6 @@ import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.RawResponseWriter; import org.apache.solr.response.RubyResponseWriter; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.response.VelocityResponseWriter; import org.apache.solr.response.XMLResponseWriter; import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.QParserPlugin; @@ -1408,6 +1408,7 @@ public final class SolrCore implements SolrInfoMBean { m.put("raw", new RawResponseWriter()); m.put("javabin", new BinaryResponseWriter()); m.put("velocity", new VelocityResponseWriter()); + m.put("csv", new CSVResponseWriter()); DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m); } diff --git a/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java b/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java index 14e5e903d50..2bef11cf70f 100755 --- a/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java +++ b/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java @@ -70,18 +70,18 @@ public class CSVRequestHandler extends ContentStreamHandlerBase { abstract class CSVLoader extends ContentStreamLoader { - static String SEPARATOR="separator"; - static String FIELDNAMES="fieldnames"; - static String HEADER="header"; - static String SKIP="skip"; - static String SKIPLINES="skipLines"; - static String MAP="map"; - static String TRIM="trim"; - static String EMPTY="keepEmpty"; - static String SPLIT="split"; - static String ENCAPSULATOR="encapsulator"; - static String ESCAPE="escape"; - static String OVERWRITE="overwrite"; + public static final String SEPARATOR="separator"; + public static final String FIELDNAMES="fieldnames"; + public static final String HEADER="header"; + public static final String SKIP="skip"; + public static final String SKIPLINES="skipLines"; + public static final String MAP="map"; + public static final String TRIM="trim"; + public static final String EMPTY="keepEmpty"; + public static final String SPLIT="split"; + public static final String ENCAPSULATOR="encapsulator"; + public static final String ESCAPE="escape"; + public static final String OVERWRITE="overwrite"; private static Pattern colonSplit = Pattern.compile(":"); private static Pattern commaSplit = Pattern.compile(","); @@ -219,7 +219,7 @@ abstract class CSVLoader extends ContentStreamLoader { // if only encapsulator or escape is set, disable the other escaping mechanism if (encapsulator == null && escape != null) { - strategy.setEncapsulator((char)-2); // TODO: add CSVStrategy.ENCAPSULATOR_DISABLED + strategy.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED); strategy.setEscape(escape.charAt(0)); } else { if (encapsulator != null) { diff --git a/solr/src/java/org/apache/solr/response/CSVResponseWriter.java b/solr/src/java/org/apache/solr/response/CSVResponseWriter.java new file mode 100755 index 00000000000..c6dda92e21c --- /dev/null +++ b/solr/src/java/org/apache/solr/response/CSVResponseWriter.java @@ -0,0 +1,536 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.response; + +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVStrategy; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Fieldable; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.DateUtil; +import org.apache.solr.common.util.FastWriter; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.StrField; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrIndexSearcher; + +import java.io.CharArrayWriter; +import java.io.IOException; +import java.io.Writer; +import java.util.*; + +/** + * @version $Id$ + */ + +public class CSVResponseWriter implements QueryResponseWriter { + + public void init(NamedList n) { + } + + public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { + CSVWriter w = new CSVWriter(writer, req, rsp); + try { + w.writeResponse(); + } finally { + w.close(); + } + } + + public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { + // using the text/plain allows this to be viewed in the browser easily + return CONTENT_TYPE_TEXT_UTF8; + } +} + + +class CSVWriter extends TextResponseWriter { + static String SEPARATOR = "separator"; + static String ENCAPSULATOR = "encapsulator"; + static String ESCAPE = "escape"; + + static String CSV = "csv."; + static String CSV_SEPARATOR = CSV + SEPARATOR; + static String CSV_ENCAPSULATOR = CSV + ENCAPSULATOR; + static String CSV_ESCAPE = CSV + ESCAPE; + + static String MV = CSV+"mv."; + static String MV_SEPARATOR = MV + SEPARATOR; + static String MV_ENCAPSULATOR = MV + ENCAPSULATOR; + static String MV_ESCAPE = MV + ESCAPE; + + static String CSV_NULL = CSV + "null"; + static String CSV_HEADER = CSV + "header"; + static String CSV_NEWLINE = CSV + "newline"; + + char[] sharedCSVBuf = new char[8192]; + + // prevent each instance from creating it's own buffer + class CSVSharedBufPrinter extends CSVPrinter { + public CSVSharedBufPrinter(Writer out, CSVStrategy strategy) { + super(out, strategy); + super.buf = sharedCSVBuf; + } + + public void reset() { + super.newLine = true; + // update our shared buf in case a new bigger one was allocated + sharedCSVBuf = super.buf; + } + } + + // allows access to internal buf w/o copying it + static class OpenCharArrayWriter extends CharArrayWriter { + public char[] getInternalBuf() { return buf; } + } + + // Writes all data to a char array, + // allows access to internal buffer, and allows fast resetting. + static class ResettableFastWriter extends FastWriter { + OpenCharArrayWriter cw = new OpenCharArrayWriter(); + char[] result; + int resultLen; + + public ResettableFastWriter() { + super(new OpenCharArrayWriter()); + cw = (OpenCharArrayWriter)sink; + } + + public void reset() { + cw.reset(); + pos=0; + } + + public void freeze() throws IOException { + if (cw.size() > 0) { + flush(); + result = cw.getInternalBuf(); + resultLen = cw.size(); + } else { + result = buf; + resultLen = pos; + } + } + + public int getFrozenSize() { return resultLen; } + public char[] getFrozenBuf() { return result; } + } + + + static class CSVField { + String name; + SchemaField sf; + CSVSharedBufPrinter mvPrinter; // printer used to encode multiple values in a single CSV value + + // used to collect values + List values = new ArrayList(1); // low starting amount in case there are many fields + int tmp; + } + + int pass; + Map csvFields = new LinkedHashMap(); + + Calendar cal; // for formatting date objects + + CSVStrategy strategy; // strategy for encoding the fields of documents + CSVPrinter printer; + ResettableFastWriter mvWriter = new ResettableFastWriter(); // writer used for multi-valued fields + + String NullValue; + boolean returnScore = false; + + + public CSVWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) { + super(writer, req, rsp); + } + + public void writeResponse() throws IOException { + SolrParams params = req.getParams(); + + strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true); + CSVStrategy strat = strategy; + + String sep = params.get(CSV_SEPARATOR); + if (sep!=null) { + if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid separator:'"+sep+"'"); + strat.setDelimiter(sep.charAt(0)); + } + + String nl = params.get(CSV_NEWLINE); + if (nl!=null) { + if (nl.length()==0) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid newline:'"+nl+"'"); + strat.setPrinterNewline(nl); + } + + String encapsulator = params.get(CSV_ENCAPSULATOR); + String escape = params.get(CSV_ESCAPE); + if (encapsulator!=null) { + if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid encapsulator:'"+encapsulator+"'"); + strat.setEncapsulator(encapsulator.charAt(0)); + } + + if (escape!=null) { + if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid escape:'"+escape+"'"); + strat.setEscape(escape.charAt(0)); + if (encapsulator == null) { + strat.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED); + } + } + + if (strat.getEscape() == '\\') { + // If the escape is the standard backslash, then also enable + // unicode escapes (it's harmless since 'u' would not otherwise + // be escaped. + strat.setUnicodeEscapeInterpretation(true); + } + + printer = new CSVPrinter(writer, strategy); + + + CSVStrategy mvStrategy = new CSVStrategy(strategy.getDelimiter(), CSVStrategy.ENCAPSULATOR_DISABLED, CSVStrategy.COMMENTS_DISABLED, '\\', false, false, false, false); + strat = mvStrategy; + + sep = params.get(MV_SEPARATOR); + if (sep!=null) { + if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv separator:'"+sep+"'"); + strat.setDelimiter(sep.charAt(0)); + } + + encapsulator = params.get(MV_ENCAPSULATOR); + escape = params.get(MV_ESCAPE); + + if (encapsulator!=null) { + if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv encapsulator:'"+encapsulator+"'"); + strat.setEncapsulator(encapsulator.charAt(0)); + if (escape == null) { + strat.setEscape(CSVStrategy.ESCAPE_DISABLED); + } + } + + escape = params.get(MV_ESCAPE); + if (escape!=null) { + if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv escape:'"+escape+"'"); + strat.setEscape(escape.charAt(0)); + // encapsulator will already be disabled if it wasn't specified + } + + returnScore = returnFields != null && returnFields.contains("score"); + boolean needListOfFields = returnFields==null || returnFields.size()==0 || (returnFields.size()==1 && returnScore) || returnFields.contains("*"); + Collection fields = returnFields; + + Object responseObj = rsp.getValues().get("response"); + if (needListOfFields) { + if (responseObj instanceof SolrDocumentList) { + // get the list of fields from the SolrDocumentList + fields = new LinkedHashSet(); + for (SolrDocument sdoc: (SolrDocumentList)responseObj) { + fields.addAll(sdoc.getFieldNames()); + } + } else { + // get the list of fields from the index + fields = req.getSearcher().getFieldNames(); + } + if (returnScore) { + fields.add("score"); + } else { + fields.remove("score"); + } + } + + CSVSharedBufPrinter csvPrinterMV = new CSVSharedBufPrinter(mvWriter, mvStrategy); + + for (String field : fields) { + if (field.equals("score")) { + CSVField csvField = new CSVField(); + csvField.name = "score"; + csvFields.put("score", csvField); + continue; + } + + SchemaField sf = schema.getFieldOrNull(field); + if (sf == null) { + FieldType ft = new StrField(); + sf = new SchemaField(field, ft); + } + + // if we got the list of fields from the index, only list stored fields + if (returnFields==null && sf != null && !sf.stored()) { + continue; + } + + // check for per-field overrides + sep = params.get("f." + field + '.' + CSV_SEPARATOR); + encapsulator = params.get("f." + field + '.' + CSV_ENCAPSULATOR); + escape = params.get("f." + field + '.' + CSV_ESCAPE); + + CSVSharedBufPrinter csvPrinter = csvPrinterMV; + if (sep != null || encapsulator != null || escape != null) { + // create a new strategy + printer if there were any per-field overrides + strat = (CSVStrategy)mvStrategy.clone(); + if (sep!=null) { + if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv separator:'"+sep+"'"); + strat.setDelimiter(sep.charAt(0)); + } + if (encapsulator!=null) { + if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv encapsulator:'"+encapsulator+"'"); + strat.setEncapsulator(encapsulator.charAt(0)); + if (escape == null) { + strat.setEscape(CSVStrategy.ESCAPE_DISABLED); + } + } + if (escape!=null) { + if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv escape:'"+escape+"'"); + strat.setEscape(escape.charAt(0)); + if (encapsulator == null) { + strat.setEncapsulator(CSVStrategy.ENCAPSULATOR_DISABLED); + } + } + csvPrinter = new CSVSharedBufPrinter(mvWriter, strat); + } + + + CSVField csvField = new CSVField(); + csvField.name = field; + csvField.sf = sf; + csvField.mvPrinter = csvPrinter; + csvFields.put(field, csvField); + } + + NullValue = params.get(CSV_NULL, ""); + + if (params.getBool(CSV_HEADER, true)) { + for (CSVField csvField : csvFields.values()) { + printer.print(csvField.name); + } + printer.println(); + } + + + if (responseObj instanceof DocList) { + writeDocList(null, (DocList)responseObj, null, null); + } else if (responseObj instanceof SolrDocumentList) { + writeSolrDocumentList(null, (SolrDocumentList)responseObj, null, null); + } + + } + + @Override + public void close() throws IOException { + if (printer != null) printer.flush(); + super.close(); + } + + @Override + public void writeNamedList(String name, NamedList val) throws IOException { + } + + @Override + public void writeDoc(String name, Document doc, Set returnFields, float score, boolean includeScore) throws IOException { + pass++; + + for (Fieldable field: doc.getFields()) { + CSVField csvField = csvFields.get(field.name()); + if (csvField == null) continue; + if (csvField.tmp != pass) { + csvField.tmp = pass; + csvField.values.clear(); + } + csvField.values.add(field); + } + + for (CSVField csvField : csvFields.values()) { + if (csvField.name.equals("score")) { + writeFloat("score", score); + continue; + } + if (csvField.tmp != pass) { + writeNull(csvField.name); + continue; + } + + if (csvField.sf.multiValued() || csvField.values.size() > 1) { + mvWriter.reset(); + csvField.mvPrinter.reset(); + // switch the printer to use the multi-valued one + CSVPrinter tmp = printer; + printer = csvField.mvPrinter; + for (Fieldable fval : csvField.values) { + csvField.sf.getType().write(this, csvField.name, fval); + } + printer = tmp; // restore the original printer + + mvWriter.freeze(); + printer.print(mvWriter.getFrozenBuf(), 0, mvWriter.getFrozenSize(), true); + } else { + assert csvField.values.size() == 1; + csvField.sf.getType().write(this,csvField.name,csvField.values.get(0)); + } + } + + printer.println(); + } + + //NOTE: a document cannot currently contain another document + List tmpList; + @Override + public void writeSolrDocument(String name, SolrDocument doc, Set returnFields, Map pseudoFields) throws IOException { + if (tmpList == null) { + tmpList = new ArrayList(1); + tmpList.add(null); + } + + for (CSVField csvField : csvFields.values()) { + Object val = doc.getFieldValue(csvField.name); + int nVals = val instanceof Collection ? ((Collection)val).size() : (val==null ? 0 : 1); + if (nVals == 0) { + writeNull(csvField.name); + continue; + } + + if ((csvField.sf != null && csvField.sf.multiValued()) || nVals > 1) { + Collection values; + // normalize to a collection + if (val instanceof Collection) { + values = (Collection)val; + } else { + tmpList.set(0, val); + values = tmpList; + } + + mvWriter.reset(); + csvField.mvPrinter.reset(); + // switch the printer to use the multi-valued one + CSVPrinter tmp = printer; + printer = csvField.mvPrinter; + for (Object fval : values) { + writeVal(csvField.name, fval); + } + printer = tmp; // restore the original printer + + mvWriter.freeze(); + printer.print(mvWriter.getFrozenBuf(), 0, mvWriter.getFrozenSize(), true); + + } else { + // normalize to first value + if (val instanceof Collection) { + Collection values = (Collection)val; + val = values.iterator().next(); + } + writeVal(csvField.name, val); + } + } + + printer.println(); + } + + @Override + public void writeDocList(String name, DocList ids, Set fields, Map otherFields) throws IOException { + int sz=ids.size(); + SolrIndexSearcher searcher = req.getSearcher(); + DocIterator iterator = ids.iterator(); + for (int i=0; i fields, Map otherFields) throws IOException { + for (SolrDocument doc : docs) { + writeSolrDocument(name, doc, fields, otherFields); + } + } + + @Override + public void writeStr(String name, String val, boolean needsEscaping) throws IOException { + printer.print(val, needsEscaping); + } + + @Override + public void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException { + } + + @Override + public void writeArray(String name, Object[] val) throws IOException { + } + + @Override + public void writeArray(String name, Iterator val) throws IOException { + } + + @Override + public void writeNull(String name) throws IOException { + printer.print(NullValue); + } + + @Override + public void writeInt(String name, String val) throws IOException { + printer.print(val, false); + } + + @Override + public void writeLong(String name, String val) throws IOException { + printer.print(val, false); + } + + @Override + public void writeBool(String name, String val) throws IOException { + printer.print(val, false); + } + + @Override + public void writeFloat(String name, String val) throws IOException { + printer.print(val, false); + } + + @Override + public void writeDouble(String name, String val) throws IOException { + printer.print(val, false); + } + + @Override + public void writeDate(String name, Date val) throws IOException { + StringBuilder sb = new StringBuilder(25); + cal = DateUtil.formatDate(val, cal, sb); + writeDate(name, sb.toString()); + } + + @Override + public void writeDate(String name, String val) throws IOException { + printer.print(val, false); + } + + @Override + public void writeShort(String name, String val) throws IOException { + printer.print(val, false); + } + + @Override + public void writeByte(String name, String val) throws IOException { + printer.print(val, false); + } +} diff --git a/solr/src/java/org/apache/solr/util/SolrPluginUtils.java b/solr/src/java/org/apache/solr/util/SolrPluginUtils.java index 88cb045bc25..048efa6c3ac 100644 --- a/solr/src/java/org/apache/solr/util/SolrPluginUtils.java +++ b/solr/src/java/org/apache/solr/util/SolrPluginUtils.java @@ -205,7 +205,7 @@ public class SolrPluginUtils { // TODO - should field order be maintained? String[] flst = split(fl); if (flst.length > 0 && !(flst.length==1 && flst[0].length()==0)) { - Set set = new HashSet(); + Set set = new LinkedHashSet(); for (String fname : flst) { if("score".equalsIgnoreCase(fname)) flags |= SolrIndexSearcher.GET_SCORES; diff --git a/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java b/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java new file mode 100644 index 00000000000..c85bad44c3f --- /dev/null +++ b/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.response; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.util.DateUtil; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.util.SolrPluginUtils; +import org.junit.*; + +import java.io.StringWriter; + +import static org.junit.Assert.*; + +import static org.junit.Assert.assertEquals; + +public class TestCSVResponseWriter extends SolrTestCaseJ4 { + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml","schema12.xml"); + createIndex(); + } + + public static void createIndex() { + assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt","2000-01-02T03:04:05Z")); + assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output")); + assertU(commit()); + } + + + @Test + public void testCSVOutput() throws Exception { + // test our basic types,and that fields come back in the requested order + assertEquals("id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt\n1,hi,-1,12345678987654321,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n" + , h.query(req("q","id:1", "wt","csv", "fl","id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt"))); + + // test retrieving score, csv.header + assertEquals("1,0.0,hi\n" + , h.query(req("q","id:1^0", "wt","csv", "csv.header","false", "fl","id,score,foo_s"))); + + // test multivalued + assertEquals("2,\"hi,there\"\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "fl","id,v_ss"))); + + // test separator change + assertEquals("2|\"hi|there\"\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.separator","|", "fl","id,v_ss"))); + + // test mv separator change + assertEquals("2,hi|there\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "fl","id,v_ss"))); + + // test mv separator change for a single field + assertEquals("2,hi|there,nice:output\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "f.v2_ss.csv.separator",":", "fl","id,v_ss,v2_ss"))); + + // test retrieving fields from index + String result = h.query(req("q","*:*", "wt","csv", "csv.header","true", "fl","*,score")); + for (String field : "id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss,score".split(",")) { + assertTrue(result.indexOf(field) >= 0); + } + + // test null values + assertEquals("2,,hi|there\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "fl","id,foo_s,v_ss"))); + + // test alternate null value + assertEquals("2,NULL,hi|there\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "csv.null","NULL","fl","id,foo_s,v_ss"))); + + // test alternate newline + assertEquals("2,\"hi,there\"\r\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.newline","\r\n", "fl","id,v_ss"))); + + // test alternate encapsulator + assertEquals("2,'hi,there'\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.encapsulator","'", "fl","id,v_ss"))); + + // test using escape instead of encapsulator + assertEquals("2,hi\\,there\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.escape","\\", "fl","id,v_ss"))); + + // test multiple lines + assertEquals("1,,hi\n2,\"hi,there\",\n" + , h.query(req("q","id:[1 TO 2]", "wt","csv", "csv.header","false", "fl","id,v_ss,foo_s"))); + + + // now test SolrDocumentList + SolrDocument d = new SolrDocument(); + SolrDocument d1 = d; + d.addField("id","1"); + d.addField("foo_i",-1); + d.addField("foo_s","hi"); + d.addField("foo_l","12345678987654321L"); + d.addField("foo_b",false); + d.addField("foo_f",1.414f); + d.addField("foo_d",-1.0E300); + d.addField("foo_dt", DateUtil.parseDate("2000-01-02T03:04:05Z")); + d.addField("score", "2.718"); + + d = new SolrDocument(); + SolrDocument d2 = d; + d.addField("id","2"); + d.addField("v_ss","hi"); + d.addField("v_ss","there"); + d.addField("v2_ss","nice"); + d.addField("v2_ss","output"); + d.addField("score", "89.83"); + + SolrDocumentList sdl = new SolrDocumentList(); + sdl.add(d1); + sdl.add(d2); + + SolrQueryRequest req = req("q","*:*"); + SolrQueryResponse rsp = new SolrQueryResponse(); + rsp.add("response", sdl); + QueryResponseWriter w = new CSVResponseWriter(); + + SolrPluginUtils.setReturnFields("id,foo_s", rsp); + StringWriter buf = new StringWriter(); + w.write(buf, req, rsp); + assertEquals("id,foo_s\n1,hi\n2,\n", buf.toString()); + + // try scores + SolrPluginUtils.setReturnFields("id,score,foo_s", rsp); + buf = new StringWriter(); + w.write(buf, req, rsp); + assertEquals("id,score,foo_s\n1,2.718,hi\n2,89.83,\n", buf.toString()); + + // get field values from docs... should be ordered and not include score unless requested + SolrPluginUtils.setReturnFields("*", rsp); + buf = new StringWriter(); + w.write(buf, req, rsp); + assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss\n" + + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z,,\n" + + "2,,,,,,,,\"hi,there\",\"nice,output\"\n", + buf.toString()); + + + // get field values and scores - just check that the scores are there... we don't guarantee where + SolrPluginUtils.setReturnFields("*,score", rsp); + buf = new StringWriter(); + w.write(buf, req, rsp); + String s = buf.toString(); + assertTrue(s.indexOf("score") >=0 && s.indexOf("2.718") > 0 && s.indexOf("89.83") > 0 ); + } + +} \ No newline at end of file