SOLR-9717: Refactor '/export' to not hardcode the JSON output and to use an API

This commit is contained in:
Noble Paul 2016-11-08 13:45:59 +05:30
parent 0f0526f9b5
commit e6dfb76c98
15 changed files with 626 additions and 285 deletions

View File

@ -87,6 +87,8 @@ Other Changes
* SOLR-9720: Refactor Responsewriters to remove dependencies on TupleStream,
Tuple, Explanation (noble)
* SOLR-9717: Refactor '/export' to not hardcode the JSON output and to use an API (noble)
================== 6.3.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

View File

@ -110,7 +110,6 @@ import org.apache.solr.response.RubyResponseWriter;
import org.apache.solr.response.SchemaXmlResponseWriter;
import org.apache.solr.response.SmileResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.response.SortingResponseWriter;
import org.apache.solr.response.XMLResponseWriter;
import org.apache.solr.response.transform.TransformerFactory;
import org.apache.solr.rest.ManagedResourceStorage;
@ -2314,7 +2313,6 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
m.put("raw", new RawResponseWriter());
m.put(CommonParams.JAVABIN, new BinaryResponseWriter());
m.put("csv", new CSVResponseWriter());
m.put("xsort", new SortingResponseWriter());
m.put("schema.xml", new SchemaXmlResponseWriter());
m.put("smile", new SmileResponseWriter());
m.put(ReplicationHandler.FILE_STREAM, getFileStreamWriter());
@ -2332,12 +2330,21 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
@Override
public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse response) throws IOException {
RawWriter rawWriter = (RawWriter) response.getValues().get(ReplicationHandler.FILE_STREAM);
if(rawWriter!=null) rawWriter.write(out);
if (rawWriter != null) {
rawWriter.write(out);
if (rawWriter instanceof Closeable) ((Closeable) rawWriter).close();
}
}
@Override
public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {
return BinaryResponseParser.BINARY_CONTENT_TYPE;
RawWriter rawWriter = (RawWriter) response.getValues().get(ReplicationHandler.FILE_STREAM);
if (rawWriter != null) {
return rawWriter.getContentType();
} else {
return BinaryResponseParser.BINARY_CONTENT_TYPE;
}
}
};
}
@ -2347,6 +2354,9 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
}
public interface RawWriter {
default String getContentType() {
return BinaryResponseParser.BINARY_CONTENT_TYPE;
}
void write(OutputStream os) throws IOException ;
}

View File

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler;
import java.util.HashMap;
import java.util.Map;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.handler.component.SearchHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import static org.apache.solr.common.params.CommonParams.JSON;
public class ExportHandler extends SearchHandler {
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
try {
super.handleRequestBody(req, rsp);
} catch (Exception e) {
rsp.setException(e);
}
String wt = req.getParams().get(CommonParams.WT, JSON);
if("xsort".equals(wt)) wt = JSON;
Map<String, String> map = new HashMap<>(1);
map.put(CommonParams.WT, ReplicationHandler.FILE_STREAM);
req.setParams(SolrParams.wrapDefaults(new MapSolrParams(map),req.getParams()));
rsp.add(ReplicationHandler.FILE_STREAM, new ExportWriter(req, rsp, wt));
}
}

View File

@ -14,17 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.response;
package org.apache.solr.handler;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.List;
import java.util.ArrayList;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
@ -40,11 +44,18 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LongValues;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.MapWriter.EntryWriter;
import org.apache.solr.common.PushWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.JSONResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.BoolField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
@ -61,24 +72,65 @@ import org.apache.solr.search.SyntaxError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.apache.solr.common.util.Utils.makeMap;
public class SortingResponseWriter implements QueryResponseWriter {
public class ExportWriter implements SolrCore.RawWriter, Closeable {
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private OutputStreamWriter respWriter;
final SolrQueryRequest req;
final SolrQueryResponse res;
FieldWriter[] fieldWriters;
int totalHits = 0;
FixedBitSet[] sets = null;
PushWriter writer;
private String wt;
ExportWriter(SolrQueryRequest req, SolrQueryResponse res, String wt) {
this.req = req;
this.res = res;
this.wt = wt;
public void init(NamedList args) {
/* NOOP */
}
public String getContentType(SolrQueryRequest req, SolrQueryResponse res) {
return "application/json";
@Override
public String getContentType() {
if ("javabin".equals(wt)) {
return BinaryResponseParser.BINARY_CONTENT_TYPE;
} else return "json";
}
public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse res) throws IOException {
Exception e1 = res.getException();
if(e1 != null) {
if(!(e1 instanceof IgnoreException)) {
writeException(e1, writer, false);
@Override
public void close() throws IOException {
if (writer != null) writer.close();
if (respWriter != null) {
respWriter.flush();
respWriter.close();
}
}
protected void writeException(Exception e, PushWriter w, boolean log) throws IOException {
w.writeMap(mw -> {
mw.put("responseHeader", singletonMap("status", 400))
.put("response", makeMap(
"numFound", 0,
"docs", singletonList(singletonMap("EXCEPTION", e.getMessage()))));
});
if (log) {
SolrException.log(logger, e);
}
}
public void write(OutputStream os) throws IOException {
respWriter = new OutputStreamWriter(os, StandardCharsets.UTF_8);
writer = JSONResponseWriter.getPushWriter(respWriter, req, res);
Exception exception = res.getException();
if (exception != null) {
if (!(exception instanceof IgnoreException)) {
writeException(exception, writer, false);
}
return;
}
@ -113,8 +165,6 @@ public class SortingResponseWriter implements QueryResponseWriter {
// You'll have to uncomment the if below to hit the null pointer exception.
// This is such an unusual case (i.e. an empty index) that catching this concdition here is probably OK.
// This came to light in the very artifical case of indexing a single doc to Cloud.
int totalHits = 0;
FixedBitSet[] sets = null;
if (req.getContext().get("totalHits") != null) {
totalHits = ((Integer)req.getContext().get("totalHits")).intValue();
sets = (FixedBitSet[]) req.getContext().get("export");
@ -145,8 +195,6 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
}
FieldWriter[] fieldWriters = null;
try {
fieldWriters = getFieldWriters(fields, req.getSearcher());
} catch (Exception e) {
@ -154,9 +202,17 @@ public class SortingResponseWriter implements QueryResponseWriter {
return;
}
writer.write("{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":"+totalHits+", \"docs\":[");
writer.writeMap(m -> {
m.put("responseHeader", singletonMap("status", 0));
m.put("response", (MapWriter) mw -> {
mw.put("numFound", totalHits);
mw.put("docs", (IteratorWriter) iw -> writeDocs(req, iw, sort));
});
});
}
protected void writeDocs(SolrQueryRequest req, IteratorWriter.ItemWriter writer, Sort sort) throws IOException {
//Write the data.
List<LeafReaderContext> leaves = req.getSearcher().getTopReaderContext().leaves();
SortDoc sortDoc = getSortDoc(req.getSearcher(), sort.getSort());
@ -165,7 +221,6 @@ public class SortingResponseWriter implements QueryResponseWriter {
SortQueue queue = new SortQueue(queueSize, sortDoc);
SortDoc[] outDocs = new SortDoc[queueSize];
boolean commaNeeded = false;
while(count < totalHits) {
//long begin = System.nanoTime();
queue.reset();
@ -199,12 +254,10 @@ public class SortingResponseWriter implements QueryResponseWriter {
try {
for(int i=outDocsIndex; i>=0; --i) {
SortDoc s = outDocs[i];
if(commaNeeded){writer.write(',');}
writer.write('{');
writeDoc(s, leaves, fieldWriters, sets, writer);
writer.write('}');
commaNeeded = true;
s.reset();
writer.add((MapWriter) ew -> {
writeDoc(s, leaves, ew);
s.reset();
});
}
} catch(Throwable e) {
Throwable ex = e;
@ -224,54 +277,24 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
}
}
//System.out.println("Sort Time 2:"+Long.toString(total/1000000));
writer.write("]}}");
writer.flush();
}
public static class IgnoreException extends IOException {
public void printStackTrace(PrintWriter pw) {
pw.print("Early Client Disconnect");
}
public String getMessage() {
return "Early Client Disconnect";
}
}
protected void writeDoc(SortDoc sortDoc,
List<LeafReaderContext> leaves,
FieldWriter[] fieldWriters,
FixedBitSet[] sets,
Writer out) throws IOException{
EntryWriter ew) throws IOException {
int ord = sortDoc.ord;
FixedBitSet set = sets[ord];
set.clear(sortDoc.docId);
LeafReaderContext context = leaves.get(ord);
int fieldIndex = 0;
for(FieldWriter fieldWriter : fieldWriters) {
if(fieldWriter.write(sortDoc.docId, context.reader(), out, fieldIndex)){
for (FieldWriter fieldWriter : fieldWriters) {
if (fieldWriter.write(sortDoc.docId, context.reader(), ew, fieldIndex)) {
++fieldIndex;
}
}
}
protected void writeException(Exception e, Writer out, boolean log) throws IOException{
out.write("{\"responseHeader\": {\"status\": 400}, \"response\":{\"numFound\":0, \"docs\":[");
out.write("{\"EXCEPTION\":\"");
writeStr(e.getMessage(), out);
out.write("\"}");
out.write("]}}");
out.flush();
if(log) {
SolrException.log(logger, e);
}
}
protected FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searcher) throws IOException {
IndexSchema schema = searcher.getSchema();
FieldWriter[] writers = new FieldWriter[fields.length];
@ -291,50 +314,49 @@ public class SortingResponseWriter implements QueryResponseWriter {
boolean multiValued = schemaField.multiValued();
FieldType fieldType = schemaField.getType();
if(fieldType instanceof TrieIntField) {
if(multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, true);
if (fieldType instanceof TrieIntField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new IntFieldWriter(field);
}
} else if (fieldType instanceof TrieLongField) {
if(multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, true);
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new LongFieldWriter(field);
}
} else if (fieldType instanceof TrieFloatField) {
if(multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, true);
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new FloatFieldWriter(field);
}
} else if(fieldType instanceof TrieDoubleField) {
if(multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, true);
} else if (fieldType instanceof TrieDoubleField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new DoubleFieldWriter(field);
}
} else if(fieldType instanceof StrField) {
if(multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, false);
} else if (fieldType instanceof StrField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
} else {
writers[i] = new StringFieldWriter(field, fieldType);
}
} else if (fieldType instanceof TrieDateField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, false);
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
} else {
writers[i] = new DateFieldWriter(field);
}
} else if(fieldType instanceof BoolField) {
if(multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, true);
} else if (fieldType instanceof BoolField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new BoolFieldWriter(field, fieldType);
}
}
else {
} else {
throw new IOException("Export fields must either be one of the following types: int,float,long,double,string,date,boolean");
}
}
@ -398,8 +420,8 @@ public class SortingResponseWriter implements QueryResponseWriter {
// _and_ since "F" happens to sort before "T" (thus false sorts "less" than true)
// we can just use the existing StringValue here.
LeafReader reader = searcher.getSlowAtomicReader();
SortedDocValues vals = reader.getSortedDocValues(field);
if(reverse) {
SortedDocValues vals = reader.getSortedDocValues(field);
if (reverse) {
sortValues[i] = new StringValue(vals, field, new IntDesc());
} else {
sortValues[i] = new StringValue(vals, field, new IntAsc());
@ -439,8 +461,8 @@ public class SortingResponseWriter implements QueryResponseWriter {
private void populate() {
Object[] heap = getHeapArray();
cache = new SortDoc[heap.length];
for(int i=1; i<heap.length; i++) {
cache[i] = heap[i] = proto.copy();
for (int i = 1; i < heap.length; i++) {
cache[i] = heap[i] = proto.copy();
}
size = maxSize;
}
@ -470,7 +492,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
public void setNextReader(LeafReaderContext context) throws IOException {
this.ord = context.ord;
for(SortValue value : sortValues) {
for (SortValue value : sortValues) {
value.setNextReader(context);
}
}
@ -872,7 +894,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
public void setCurrentValue(int docId) {
currentValue = (int)vals.get(docId);
currentValue = (int) vals.get(docId);
}
public int compareTo(SortValue o) {
@ -1029,7 +1051,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
public void setCurrentValue(int docId) {
currentValue = Float.intBitsToFloat((int)vals.get(docId));
currentValue = Float.intBitsToFloat((int) vals.get(docId));
}
public void setCurrentValue(SortValue sv) {
@ -1234,7 +1256,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
protected abstract class FieldWriter {
public abstract boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException;
public abstract boolean write(int docId, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException;
}
class IntFieldWriter extends FieldWriter {
@ -1244,17 +1266,10 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.field = field;
}
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
int val = (int)vals.get(docId);
if(fieldIndex>0) {
out.write(',');
}
out.write('"');
out.write(this.field);
out.write('"');
out.write(':');
out.write(Integer.toString(val));
int val = (int) vals.get(docId);
ew.put(this.field, val);
return true;
}
}
@ -1262,57 +1277,42 @@ public class SortingResponseWriter implements QueryResponseWriter {
class MultiFieldWriter extends FieldWriter {
private String field;
private FieldType fieldType;
private SchemaField schemaField;
private boolean numeric;
private CharsRefBuilder cref = new CharsRefBuilder();
public MultiFieldWriter(String field, FieldType fieldType, boolean numeric) {
public MultiFieldWriter(String field, FieldType fieldType, SchemaField schemaField, boolean numeric) {
this.field = field;
this.fieldType = fieldType;
this.schemaField = schemaField;
this.numeric = numeric;
}
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
public boolean write(int docId, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException {
SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
vals.setDocument(docId);
List<Long> ords = new ArrayList();
long o = -1;
while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
ords.add(o);
}
long first = vals.nextOrd();
if (first == SortedSetDocValues.NO_MORE_ORDS) return false;
out.put(this.field,
(IteratorWriter) w -> {
boolean isFirst = true;
for (; ; ) {
long o;
if (isFirst) {
isFirst = false;
o = first;
} else o = vals.nextOrd();
if (o == SortedSetDocValues.NO_MORE_ORDS) break;
BytesRef ref = vals.lookupOrd(o);
fieldType.indexedToReadable(ref, cref);
IndexableField f = fieldType.createField(schemaField, cref.toString(), 1.0f);
if (f == null) w.add(cref.toString());
else w.add(fieldType.toObject(f));
}
});
if(ords.size()== 0) {
return false;
}
if(fieldIndex>0) {
out.write(',');
}
out.write('"');
out.write(this.field);
out.write('"');
out.write(':');
out.write('[');
int v = 0;
for(long ord : ords) {
BytesRef ref = vals.lookupOrd(ord);
fieldType.indexedToReadable(ref, cref);
if(v > 0) {
out.write(',');
}
if(!numeric) {
out.write('"');
}
writeStr(cref.toString(), out);
if(!numeric) {
out.write('"');
}
++v;
}
out.write("]");
return true;
}
}
@ -1323,17 +1323,11 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.field = field;
}
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
long val = vals.get(docId);
if(fieldIndex > 0) {
out.write(',');
}
out.write('"');
out.write(this.field);
out.write('"');
out.write(':');
out.write(Long.toString(val));
ew.put(field, val);
return true;
}
}
@ -1345,20 +1339,10 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.field = field;
}
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
long val = vals.get(docId);
if (fieldIndex > 0) {
out.write(',');
}
out.write('"');
out.write(this.field);
out.write('"');
out.write(':');
out.write('"');
writeStr(new Date(val).toInstant().toString(), out);
out.write('"');
ew.put(this.field, new Date(val));
return true;
}
}
@ -1373,26 +1357,16 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.fieldType = fieldType;
}
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
SortedDocValues vals = DocValues.getSorted(reader, this.field);
int ord = vals.getOrd(docId);
if(ord == -1) {
if (ord == -1) {
return false;
}
BytesRef ref = vals.lookupOrd(ord);
fieldType.indexedToReadable(ref, cref);
if (fieldIndex > 0) {
out.write(',');
}
out.write('"');
out.write(this.field);
out.write('"');
out.write(':');
//out.write('"');
writeStr(cref.toString(), out);
//out.write('"');
ew.put(this.field, "true".equals(cref.toString()));
return true;
}
}
@ -1404,17 +1378,10 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.field = field;
}
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
int val = (int)vals.get(docId);
if(fieldIndex > 0) {
out.write(',');
}
out.write('"');
out.write(this.field);
out.write('"');
out.write(':');
out.write(Float.toString(Float.intBitsToFloat(val)));
int val = (int) vals.get(docId);
ew.put(this.field, Float.intBitsToFloat(val));
return true;
}
}
@ -1426,17 +1393,10 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.field = field;
}
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
if(fieldIndex > 0) {
out.write(',');
}
long val = vals.get(docId);
out.write('"');
out.write(this.field);
out.write('"');
out.write(':');
out.write(Double.toString(Double.longBitsToDouble(val)));
ew.put(this.field, Double.longBitsToDouble(val));
return true;
}
}
@ -1451,73 +1411,19 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.fieldType = fieldType;
}
public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
SortedDocValues vals = DocValues.getSorted(reader, this.field);
int ord = vals.getOrd(docId);
if(ord == -1) {
if (ord == -1) {
return false;
}
BytesRef ref = vals.lookupOrd(ord);
fieldType.indexedToReadable(ref, cref);
if(fieldIndex > 0) {
out.write(',');
}
out.write('"');
out.write(this.field);
out.write('"');
out.write(":");
out.write('"');
writeStr(cref.toString(), out);
out.write('"');
ew.put(this.field, cref.toString());
return true;
}
}
private void writeStr(String val, Writer writer) throws IOException {
for (int i=0; i<val.length(); i++) {
char ch = val.charAt(i);
if ((ch > '#' && ch != '\\' && ch < '\u2028') || ch == ' ') { // fast path
writer.write(ch);
continue;
}
switch(ch) {
case '"':
case '\\':
writer.write('\\');
writer.write(ch);
break;
case '\r': writer.write('\\'); writer.write('r'); break;
case '\n': writer.write('\\'); writer.write('n'); break;
case '\t': writer.write('\\'); writer.write('t'); break;
case '\b': writer.write('\\'); writer.write('b'); break;
case '\f': writer.write('\\'); writer.write('f'); break;
case '\u2028': // fallthrough
case '\u2029':
unicodeEscape(writer,ch);
break;
// case '/':
default: {
if (ch <= 0x1F) {
unicodeEscape(writer,ch);
} else {
writer.write(ch);
}
}
}
}
}
private static char[] hexdigits = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
protected static void unicodeEscape(Appendable out, int ch) throws IOException {
out.append('\\');
out.append('u');
out.append(hexdigits[(ch>>>12) ]);
out.append(hexdigits[(ch>>>8) & 0xf]);
out.append(hexdigits[(ch>>>4) & 0xf]);
out.append(hexdigits[(ch) & 0xf]);
}
public abstract class PriorityQueue<T> {
protected int size = 0;
protected final int maxSize;
@ -1715,4 +1621,15 @@ public class SortingResponseWriter implements QueryResponseWriter {
return (Object[]) heap;
}
}
public class IgnoreException extends IOException {
public void printStackTrace(PrintWriter pw) {
pw.print("Early Client Disconnect");
}
public String getMessage() {
return "Early Client Disconnect";
}
}
}

View File

@ -24,7 +24,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter.EntryWriter;
import org.apache.solr.common.PushWriter;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
@ -74,6 +78,11 @@ public class JSONResponseWriter implements QueryResponseWriter {
public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {
return contentType;
}
public static PushWriter getPushWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
return new JSONWriter(writer, req, rsp);
}
}
class JSONWriter extends TextResponseWriter {
@ -507,6 +516,53 @@ class JSONWriter extends TextResponseWriter {
}
}
@Override
public void writeIterator(IteratorWriter val) throws IOException {
writeArrayOpener(-1);
incLevel();
val.writeIter(new IteratorWriter.ItemWriter() {
boolean first = true;
@Override
public IteratorWriter.ItemWriter add(Object o) throws IOException {
if (!first) {
JSONWriter.this.indent();
JSONWriter.this.writeArraySeparator();
}
JSONWriter.this.writeVal(null, o);
first = false;
return this;
}
});
decLevel();
writeArrayCloser();
}
@Override
public void writeMap(MapWriter val)
throws IOException {
writeMapOpener(-1);
incLevel();
val.writeMap(new EntryWriter() {
boolean isFirst = true;
@Override
public EntryWriter put(String k, Object v) throws IOException {
if (isFirst) {
isFirst = false;
} else {
JSONWriter.this.writeMapSeparator();
}
if (doIndent) JSONWriter.this.indent();
JSONWriter.this.writeKey(k, true);
JSONWriter.this.writeVal(k, v);
return this;
}
});
decLevel();
writeMapCloser();
}
@Override
public void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException {

View File

@ -31,9 +31,12 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.EnumFieldValue;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.PushWriter;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.NamedList;
@ -48,7 +51,7 @@ import org.apache.solr.util.FastWriter;
*
*
*/
public abstract class TextResponseWriter {
public abstract class TextResponseWriter implements PushWriter {
// indent up to 40 spaces
static final char[] indentChars = new char[81];
@ -138,19 +141,19 @@ public abstract class TextResponseWriter {
writeStr(name, f.stringValue(), true);
}
} else if (val instanceof Number) {
writeNumber(name, (Number)val);
writeNumber(name, (Number) val);
} else if (val instanceof Boolean) {
writeBool(name, (Boolean)val);
writeBool(name, (Boolean) val);
} else if (val instanceof Date) {
writeDate(name,(Date)val);
writeDate(name, (Date) val);
} else if (val instanceof Document) {
SolrDocument doc = DocsStreamer.getDoc((Document) val, schema);
writeSolrDocument(name, doc,returnFields, 0 );
writeSolrDocument(name, doc, returnFields, 0);
} else if (val instanceof SolrDocument) {
writeSolrDocument(name, (SolrDocument)val,returnFields, 0);
writeSolrDocument(name, (SolrDocument) val, returnFields, 0);
} else if (val instanceof ResultContext) {
// requires access to IndexReader
writeDocuments(name, (ResultContext)val);
writeDocuments(name, (ResultContext) val);
} else if (val instanceof DocList) {
// Should not happen normally
ResultContext ctx = new BasicResultContext((DocList)val, returnFields, null, null, req);
@ -168,6 +171,8 @@ public abstract class TextResponseWriter {
writeNamedList(name, (NamedList)val);
} else if (val instanceof Path) {
writeStr(name, ((Path) val).toAbsolutePath().toString(), true);
} else if (val instanceof IteratorWriter) {
writeIterator((IteratorWriter) val);
} else if (val instanceof Iterable) {
writeArray(name,((Iterable)val).iterator());
} else if (val instanceof Object[]) {
@ -184,6 +189,8 @@ public abstract class TextResponseWriter {
writeStr(name, val.toString(), true);
} else if (val instanceof WriteableValue) {
((WriteableValue)val).write(name, this);
} else if (val instanceof MapWriter) {
writeMap((MapWriter) val);
} else if (val instanceof MapSerializable) {
//todo find a better way to reuse the map more efficiently
writeMap(name, ((MapSerializable) val).toMap(new LinkedHashMap<>()), false, true);
@ -192,6 +199,15 @@ public abstract class TextResponseWriter {
writeStr(name, val.getClass().getName() + ':' + val.toString(), true);
}
}
@Override
public void writeMap(MapWriter mw) throws IOException {
//todo
}
@Override
public void writeIterator(IteratorWriter iw) throws IOException {
/*todo*/
}
protected void writeBool(String name , Boolean val) throws IOException {
writeBool(name, val.toString());

View File

@ -92,14 +92,16 @@
"useParams":"_ADMIN_FILE"
},
"/export": {
"class": "solr.SearchHandler",
"class": "solr.ExportHandler",
"useParams":"_EXPORT",
"components": [
"query"
],
"defaults": {
"wt": "json"
},
"invariants": {
"rq": "{!xport}",
"wt": "xsort",
"distrib": false
}
},

View File

@ -97,7 +97,7 @@ public class SolrCoreTest extends SolrTestCaseJ4 {
++ihCount; assertEquals(pathToClassMap.get("/admin/system"), "solr.SystemInfoHandler");
++ihCount; assertEquals(pathToClassMap.get("/admin/threads"), "solr.ThreadDumpHandler");
++ihCount; assertEquals(pathToClassMap.get("/config"), "solr.SolrConfigHandler");
++ihCount; assertEquals(pathToClassMap.get("/export"), "solr.SearchHandler");
++ihCount; assertEquals(pathToClassMap.get("/export"), "solr.ExportHandler");
++ihCount; assertEquals(pathToClassMap.get("/terms"), "solr.SearchHandler");
++ihCount; assertEquals(pathToClassMap.get("/get"), "solr.RealTimeGetHandler");
++ihCount; assertEquals(pathToClassMap.get(ReplicationHandler.PATH), "solr.ReplicationHandler");

View File

@ -184,12 +184,15 @@ public class JSONWriterTest extends SolrTestCaseJ4 {
methodsExpectedNotOverriden.add("writeArrayOpener");
methodsExpectedNotOverriden.add("writeArraySeparator");
methodsExpectedNotOverriden.add("writeArrayCloser");
methodsExpectedNotOverriden.add("public void org.apache.solr.response.JSONWriter.writeMap(org.apache.solr.common.MapWriter) throws java.io.IOException");
methodsExpectedNotOverriden.add("public void org.apache.solr.response.JSONWriter.writeIterator(org.apache.solr.common.IteratorWriter) throws java.io.IOException");
final Class<?> subClass = ArrayOfNamedValuePairJSONWriter.class;
final Class<?> superClass = subClass.getSuperclass();
for (final Method superClassMethod : superClass.getDeclaredMethods()) {
final String methodName = superClassMethod.getName();
final String methodFullName = superClassMethod.toString();
if (!methodName.startsWith("write")) continue;
final int modifiers = superClassMethod.getModifiers();
@ -197,7 +200,8 @@ public class JSONWriterTest extends SolrTestCaseJ4 {
if (Modifier.isStatic(modifiers)) continue;
if (Modifier.isPrivate(modifiers)) continue;
final boolean expectOverriden = !methodsExpectedNotOverriden.contains(methodName);
final boolean expectOverriden = !methodsExpectedNotOverriden.contains(methodName)
&& !methodsExpectedNotOverriden.contains(methodFullName);
try {
final Method subClassMethod = subClass.getDeclaredMethod(
@ -215,7 +219,7 @@ public class JSONWriterTest extends SolrTestCaseJ4 {
if (expectOverriden) {
fail(subClass + " needs to override '" + superClassMethod + "'");
} else {
assertTrue(methodName+" not found in remaining "+methodsExpectedNotOverriden, methodsExpectedNotOverriden.remove(methodName));
assertTrue(methodName+" not found in remaining "+methodsExpectedNotOverriden, methodsExpectedNotOverriden.remove(methodName)|| methodsExpectedNotOverriden.remove(methodFullName));
}
}
}

View File

@ -17,11 +17,12 @@
package org.apache.solr.response;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.util.Utils;
import org.junit.*;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@SuppressCodecs({"Lucene3x", "Lucene40","Lucene41","Lucene42","Lucene45"})
public class TestSortingResponseWriter extends SolrTestCaseJ4 {
public class TestExportWriter extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeClass() throws Exception {
System.setProperty("export.test", "true");
@ -109,67 +110,71 @@ public class TestSortingResponseWriter extends SolrTestCaseJ4 {
//Test single value DocValue output
String s = h.query(req("q", "id:1", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":1,\"stringdv\":\"hello world\",\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":1,\"stringdv\":\"hello world\",\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
//Test null value string:
s = h.query(req("q", "id:7", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":7,\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":7,\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
//Test multiValue docValues output
s = h.query(req("q", "id:1", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"intdv_m\":[100,250],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[\"Everton\",\"liverpool\",\"manchester \\\"city\\\"\"]}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"intdv_m\":[100,250],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[\"Everton\",\"liverpool\",\"manchester \\\"city\\\"\"]}]}}");
//Test multiValues docValues output with nulls
s = h.query(req("q", "id:7", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434]}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434]}]}}");
//Test single sort param is working
s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv desc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}}");
s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv asc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}}");
// Test sort on String will null value. Null value should sort last on desc and first on asc.
s = h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv desc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":7}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":7}]}}");
s = h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv asc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":7},{\"intdv\":1}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":7},{\"intdv\":1}]}}");
//Test multi-sort params
s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}}");
s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "floatdv desc,intdv asc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}}");
//Test three sort fields
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,stringdv asc,intdv desc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
//Test three sort fields
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,stringdv desc,intdv asc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
//Test four sort fields
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,floatdv desc,floatdv asc,intdv desc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":1},{\"intdv\":2}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":1},{\"intdv\":2}]}}");
s = h.query(req("q", "intdv:[2 TO 1000]", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":7},{\"intdv\":2}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":7},{\"intdv\":2}]}}");
s = h.query(req("q", "stringdv:blah", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":0, \"docs\":[]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":0, \"docs\":[]}}");
s = h.query(req("q", "id:8", "qt", "/export", "fl", "stringdv", "sort", "intdv asc"));
assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"stringdv\":\"chello \\\"world\\\"\"}]}}");
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"stringdv\":\"chello \\\"world\\\"\"}]}}");
}
private void assertJsonEquals(String actual, String expected) {
assertEquals(Utils.toJSONString(Utils.fromJSONString(expected)), Utils.toJSONString(Utils.fromJSONString(actual)));
}
@Test

View File

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.response;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.lang.invoke.MethodHandles;
import java.util.Map;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.PushWriter;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.Utils;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.singletonMap;
public class TestPushWriter extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public void testStandardResponse() throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
OutputStreamWriter osw = new OutputStreamWriter(baos, UTF_8);
PushWriter pw = new JSONWriter(osw,
new LocalSolrQueryRequest(null, new ModifiableSolrParams()), new SolrQueryResponse());
writeData(pw);
osw.flush();
log.info(new String(baos.toByteArray(), "UTF-8"));
Object m = Utils.fromJSON(baos.toByteArray());
checkValues((Map) m);
}
protected void checkValues(Map m) {
assertEquals(0, ((Number)Utils.getObjectByPath(m, true, "responseHeader/status")).intValue());
assertEquals(10, ((Number)Utils.getObjectByPath(m, true, "response/numFound")).intValue());
assertEquals(1, ((Number)Utils.getObjectByPath(m, true, "response/docs[0]/id")).intValue());
assertEquals(2, ((Number)Utils.getObjectByPath(m, true, "response/docs[1]/id")).intValue());
assertEquals(3, ((Number)Utils.getObjectByPath(m, true, "response/docs[2]/id")).intValue());
}
protected void writeData(PushWriter pw) throws IOException {
pw.writeMap(m -> {
m.put("responseHeader", singletonMap("status", 0))
.put("response", (MapWriter) m1 -> {
m1.put("numFound", 10)
.put("docs", (IteratorWriter) w -> {
w.add((MapWriter) m3 -> m3.put("id", 1))
.add(singletonMap("id", 2))
.add(singletonMap("id", 3));
}); }); });
pw.close();
}
}

View File

@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common;
import java.io.IOException;
/**
* Interface to help do push writing to an array
*/
public interface IteratorWriter {
/**
* @param w after this method returns , the EntryWriter Object is invalid
* Do not hold a reference to this object
*/
void writeIter(ItemWriter w) throws IOException;
interface ItemWriter {
ItemWriter add(Object o) throws IOException;
default ItemWriter add(int v) throws IOException {
add((Integer) v);
return this;
}
default ItemWriter add(long v) throws IOException {
add((Long) v);
return this;
}
default ItemWriter add(float v) throws IOException {
add((Float) v);
return this;
}
default ItemWriter add(double v) throws IOException {
add((Double) v);
return this;
}
default ItemWriter add(boolean v) throws IOException {
add((Boolean) v);
return this;
}
}
}

View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common;
import java.io.IOException;
import java.util.Map;
/**
* Use this class if the Map size is not known
*/
public interface MapWriter extends MapSerializable {
@Override
default Map toMap(Map<String, Object> map) {
try {
writeMap(new EntryWriter() {
@Override
public EntryWriter put(String k, Object v) throws IOException {
map.put(k, v);
return this;
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
return map;
}
void writeMap(EntryWriter ew) throws IOException;
interface EntryWriter {
/**Writes a key value into the map
* @param k The key
* @param v The value can be any supported object
*/
EntryWriter put(String k, Object v) throws IOException;
default EntryWriter put(String k, int v) throws IOException {
put(k, (Integer) v);
return this;
}
default EntryWriter put(String k, long v) throws IOException {
put(k, (Long) v);
return this;
}
default EntryWriter put(String k, float v) throws IOException {
put(k, (Float) v);
return this;
}
default EntryWriter put(String k, double v) throws IOException{
put(k, (Double) v);
return this;
}
default EntryWriter put(String k, boolean v) throws IOException{
put(k, (Boolean) v);
return this;
}
}
}

View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common;
import java.io.Closeable;
import java.io.IOException;
/**This is an interface to stream data out using a push API
*
*/
public interface PushWriter extends Closeable {
/**Write a Map. The map is opened in the beginning of the method
* and closed at the end. All map entries MUST be written before this
* method returns
*/
void writeMap(MapWriter mw) throws IOException;
/**Write an array. The array is opened at the beginning of this method
* and closed at the end. All array entries must be returned before this
* method returns
*
*/
void writeIterator(IteratorWriter iw) throws IOException;
}

View File

@ -16,6 +16,7 @@
*/
package org.apache.solr.util;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.nio.file.Path;
@ -30,12 +31,22 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.NamedList.NamedListEntry;
import org.apache.solr.core.*;
import org.apache.solr.core.CloudConfig;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.CorePropertiesLocator;
import org.apache.solr.core.CoresLocator;
import org.apache.solr.core.NodeConfig;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.core.SolrXmlConfig;
import org.apache.solr.handler.UpdateRequestHandler;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.BinaryQueryResponseWriter;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.IndexSchema;
@ -311,10 +322,18 @@ public class TestHarness extends BaseTestHarness {
if (rsp.getException() != null) {
throw rsp.getException();
}
StringWriter sw = new StringWriter(32000);
QueryResponseWriter responseWriter = core.getQueryResponseWriter(req);
responseWriter.write(sw,req,rsp);
return sw.toString();
if (responseWriter instanceof BinaryQueryResponseWriter) {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(32000);
BinaryQueryResponseWriter writer = (BinaryQueryResponseWriter) responseWriter;
writer.write(byteArrayOutputStream, req, rsp);
return new String(byteArrayOutputStream.toByteArray(), "UTF-8");
} else {
StringWriter sw = new StringWriter(32000);
responseWriter.write(sw,req,rsp);
return sw.toString();
}
} finally {
req.close();
SolrRequestInfo.clearRequestInfo();