Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-9720: Refactor Responsewriters to remove dependencies on TupleStream, Tuple, Explanation

This commit is contained in:
Noble Paul 2016-11-03 14:04:46 +05:30
parent 940a337105
commit 1f595a20a2
12 changed files with 176 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -115,6 +115,9 @@ Other Changes
* SOLR-9621: Remove several Guava & Apache Commons calls in favor of java 8 alternatives.
(Michael Braun via David Smiley)
* SOLR-9720: Refactor Responsewriters to remove dependencies on TupleStream,
Tuple, Explanation (noble)
================== 6.3.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

13
solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java
View File

@ -109,7 +109,6 @@ class JSONWriter extends TextResponseWriter {
if(wrapperFunction!=null) {
writer.write(wrapperFunction + "(");
}
if(req.getParams().getBool(CommonParams.OMIT_HEADER, false)) rsp.removeResponseHeader();
writeNamedList(null, rsp.getValues());
if(wrapperFunction!=null) {
writer.write(')');
@ -289,7 +288,7 @@ class JSONWriter extends TextResponseWriter {
// NamedList("a"=1,"b"=2,null=3) => ["a",1,"b",2,null,3]
protected void writeNamedListAsFlat(String name, NamedList val) throws IOException {
int sz = val.size();
writeArrayOpener(sz);
writeArrayOpener(sz*2);
incLevel();
for (int i=0; i<sz; i++) {
@ -542,9 +541,19 @@ class JSONWriter extends TextResponseWriter {
}
}
@Override
public void writeArray(String name, List l) throws IOException {
writeArrayOpener(l.size());
writeJsonIter(l.iterator());
}
@Override
public void writeArray(String name, Iterator val) throws IOException {
writeArrayOpener(-1); // no trivial way to determine array size
writeJsonIter(val);
}
protected void writeJsonIter(Iterator val) throws IOException {
incLevel();
boolean first=true;
while( val.hasNext() ) {

8
solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java
View File

@ -18,6 +18,9 @@ package org.apache.solr.response;
import java.io.Writer;
import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
@ -76,6 +79,11 @@ class PHPWriter extends JSONWriter {
writer.write("array(");
}
@Override
public void writeArray(String name, List l) throws IOException {
writeArray(name,l.iterator());
}
@Override
public void writeArrayCloser() throws IOException {
writer.write(')');

43
solr/core/src/java/org/apache/solr/response/TextResponseWriter.java
View File

@ -23,18 +23,18 @@ import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.stream.TupleStream;
import org.apache.solr.client.solrj.io.stream.expr.Explanation;
import org.apache.solr.common.EnumFieldValue;
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
@ -82,6 +82,7 @@ public abstract class TextResponseWriter {
doIndent=true;
}
returnFields = rsp.getReturnFields();
if (req.getParams().getBool(CommonParams.OMIT_HEADER, false)) rsp.removeResponseHeader();
}
/** done with this ResponseWriter... make sure any buffers are flushed to writer */
@ -165,10 +166,6 @@ public abstract class TextResponseWriter {
writeMap(name, (Map)val, false, true);
} else if (val instanceof NamedList) {
writeNamedList(name, (NamedList)val);
} else if (val instanceof TupleStream) {
writeTupleStream((TupleStream) val);
} else if (val instanceof Explanation){
writeExplanation((Explanation) val);
} else if (val instanceof Path) {
writeStr(name, ((Path) val).toAbsolutePath().toString(), true);
} else if (val instanceof Iterable) {
@ -189,7 +186,7 @@ public abstract class TextResponseWriter {
((WriteableValue)val).write(name, this);
} else if (val instanceof MapSerializable) {
//todo find a better way to reuse the map more efficiently
writeMap(name, ((MapSerializable) val).toMap(new NamedList().asShallowMap()), false, true);
writeMap(name, ((MapSerializable) val).toMap(new LinkedHashMap<>()), false, true);
} else {
// default... for debugging only
writeStr(name, val.getClass().getName() + ':' + val.toString(), true);
@ -262,7 +259,11 @@ public abstract class TextResponseWriter {
public abstract void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException;
public void writeArray(String name, Object[] val) throws IOException {
writeArray(name, Arrays.asList(val).iterator());
writeArray(name, Arrays.asList(val));
}
public void writeArray(String name, List l) throws IOException {
writeArray(name, l.iterator());
}
public abstract void writeArray(String name, Iterator val) throws IOException;
@ -304,30 +305,6 @@ public abstract class TextResponseWriter {
}
}
public void writeTupleStream(TupleStream tupleStream) throws IOException {
tupleStream.open();
tupleStream.writeStreamOpen(writer);
boolean isFirst = true;
while(true) {
Tuple tuple = tupleStream.read();
if(!isFirst) {
writer.write(",");
}
writer.write("\n");
writeMap(null, tuple.fields, false, true);
isFirst = false;
if(tuple.EOF) {
break;
}
}
tupleStream.writeStreamClose(writer);
tupleStream.close();
}
public void writeExplanation(Explanation explanation) throws IOException {
writeMap(null, explanation.toMap(), false, true);
}
/** if this form of the method is called, val is the Java string form of a double */
public abstract void writeDouble(String name, String val) throws IOException;

8
solr/core/src/resources/ImplicitPlugins.json
View File

@ -114,16 +114,20 @@
"/stream": {
"class": "solr.StreamHandler",
"useParams":"_STREAM",
"defaults": {
"wt": "json"
},
"invariants": {
"wt": "json",
"distrib": false
}
},
"/sql": {
"class": "solr.SQLHandler",
"useParams":"_SQL",
"defaults": {
"wt": "json"
},
"invariants": {
"wt": "json",
"distrib": false
}
},

10
solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java
View File

@ -22,7 +22,10 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.List;
import java.util.function.BiConsumer;
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.SolrDocument;
/**
* A simple abstraction of a record containing key/value pairs.
@ -31,7 +34,7 @@ import java.util.List;
*
**/
public class Tuple implements Cloneable {
public class Tuple implements Cloneable, MapSerializable {
/**
* When EOF field is true the Tuple marks the end of the stream.
@ -193,4 +196,9 @@ public class Tuple implements Cloneable {
public void merge(Tuple other){
fields.putAll(other.getMap());
}
@Override
public Map toMap(Map<String, Object> map) {
return fields;
}
}

3
solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JSONTupleStream.java
View File

@ -40,7 +40,7 @@ import org.noggit.ObjectBuilder;
Initial version works with the json format and only SolrDocs are handled.
*/
public class JSONTupleStream {
public class JSONTupleStream implements TupleStreamParser {
private List<String> path; // future... for more general stream handling
private Reader reader;
private JSONParser parser;
@ -71,6 +71,7 @@ public class JSONTupleStream {
/** returns the next Tuple or null */
@Override
public Map<String,Object> next() throws IOException {
if (!atDocs) {
boolean found = advanceToDocs();

43
solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java
View File

@ -17,6 +17,8 @@
package org.apache.solr.client.solrj.io.stream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.HashMap;
@ -24,7 +26,11 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.InputStreamResponseParser;
import org.apache.solr.client.solrj.io.SolrClientCache;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.comp.StreamComparator;
@ -32,9 +38,12 @@ import org.apache.solr.client.solrj.io.stream.expr.Explanation;
import org.apache.solr.client.solrj.io.stream.expr.Explanation.ExpressionType;
import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -54,7 +63,7 @@ public class SolrStream extends TupleStream {
private int workerID;
private boolean trace;
private Map<String, String> fieldMappings;
private transient JSONTupleStream jsonTupleStream;
private transient TupleStreamParser tupleStreamParser;
private transient HttpSolrClient client;
private transient SolrClientCache cache;
private String slice;
@ -115,9 +124,9 @@ public class SolrStream extends TupleStream {
}
try {
jsonTupleStream = JSONTupleStream.create(client, loadParams(params));
tupleStreamParser = constructParser(client, loadParams(params));
} catch (Exception e) {
throw new IOException(e);
throw new IOException("params " + params, e);
}
}
@ -182,8 +191,8 @@ public class SolrStream extends TupleStream {
public void close() throws IOException {
if(jsonTupleStream != null) {
jsonTupleStream.close();
if (tupleStreamParser != null) {
tupleStreamParser.close();
}
if(cache == null) {
@ -197,7 +206,7 @@ public class SolrStream extends TupleStream {
public Tuple read() throws IOException {
try {
Map fields = jsonTupleStream.next();
Map fields = tupleStreamParser.next();
if (fields == null) {
//Return the EOF tuple.
@ -257,4 +266,26 @@ public class SolrStream extends TupleStream {
return fields;
}
// temporary...
public static TupleStreamParser constructParser(SolrClient server, SolrParams requestParams) throws IOException, SolrServerException {
String p = requestParams.get("qt");
if (p != null) {
ModifiableSolrParams modifiableSolrParams = (ModifiableSolrParams) requestParams;
modifiableSolrParams.remove("qt");
}
String wt = requestParams.get(CommonParams.WT, "json");
assert CommonParams.JSON.equals(wt);
QueryRequest query = new QueryRequest(requestParams);
query.setPath(p);
query.setResponseParser(new InputStreamResponseParser(wt));
query.setMethod(SolrRequest.METHOD.POST);
NamedList<Object> genericResponse = server.request(query);
InputStream stream = (InputStream) genericResponse.get("stream");
InputStreamReader reader = new InputStreamReader(stream, "UTF-8");
return new JSONTupleStream(reader);
}
}

53
solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TupleStream.java
View File

@ -19,17 +19,21 @@ package org.apache.solr.client.solrj.io.stream;
import java.io.Closeable;
import java.io.IOException;
import java.io.Serializable;
import java.io.Writer;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.comp.StreamComparator;
import org.apache.solr.client.solrj.io.stream.expr.Explanation;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.SolrException;
public abstract class TupleStream implements Closeable, Serializable {
public abstract class TupleStream implements Closeable, Serializable, MapSerializable {
private static final long serialVersionUID = 1;
@ -38,14 +42,14 @@ public abstract class TupleStream implements Closeable, Serializable {
public TupleStream() {
}
/*
public static void writeStreamOpen(Writer out) throws IOException {
out.write("{\"docs\":[");
}
public static void writeStreamClose(Writer out) throws IOException {
out.write("]}");
}
}*/
public abstract void setStreamContext(StreamContext context);
@ -64,7 +68,46 @@ public abstract class TupleStream implements Closeable, Serializable {
public int getCost() {
return 0;
}
private boolean isOpen = false;
@Override
public Map toMap(Map<String, Object> map) {
try {
if (!isOpen) {
open();
isOpen = true;
}
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
return Collections.singletonMap("docs", new Iterator<Tuple>() {
Tuple tuple;
boolean isEOF = false;
@Override
public boolean hasNext() {
if (isEOF) return false;
if (tuple != null) return true;
try {
tuple = read();
if(tuple != null && tuple.EOF) close();
return tuple != null;
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
@Override
public Tuple next() {
Tuple tmp = tuple;
tuple = null;
isEOF = tmp == null || tmp.EOF;
return tmp;
}
});
}
public UUID getStreamNodeId(){
return streamNodeId;
}

27
solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TupleStreamParser.java Normal file
View File

@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.io.stream;
import java.io.Closeable;
import java.io.IOException;
import java.util.Map;
public interface TupleStreamParser extends Closeable {
Map<String,Object> next() throws IOException;
}

19
solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/expr/Explanation.java
View File

@ -18,14 +18,17 @@ package org.apache.solr.client.solrj.io.stream.expr;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.solr.common.MapSerializable;
/**
* Explanation containing details about a expression
*/
public class Explanation {
public class Explanation implements MapSerializable {
private String expressionNodeId;
private String expressionType;
@ -124,24 +127,24 @@ public class Explanation {
}
helpers.add(helper);
}
public Map<String,Object> toMap(){
Map<String,Object> map = new HashMap<String,Object>();
@Override
public Map toMap(Map<String, Object> map) {
if(null != expressionNodeId){ map.put("expressionNodeId",expressionNodeId); }
if(null != expressionType){ map.put("expressionType",expressionType); }
if(null != functionName){ map.put("functionName",functionName); }
if(null != implementingClass){ map.put("implementingClass",implementingClass); }
if(null != expression){ map.put("expression",expression); }
if(null != note){ map.put("note",note); }
if(null != helpers && 0 != helpers.size()){
List<Map<String,Object>> helperMaps = new ArrayList<Map<String,Object>>();
List<Map<String,Object>> helperMaps = new ArrayList<>();
for(Explanation helper : helpers){
helperMaps.add(helper.toMap());
helperMaps.add(helper.toMap(new LinkedHashMap<>()));
}
map.put("helpers", helperMaps);
}
return map;
}

7
solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/expr/StreamExplanation.java
View File

@ -17,6 +17,7 @@
package org.apache.solr.client.solrj.io.stream.expr;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -54,13 +55,13 @@ public class StreamExplanation extends Explanation {
children.add(child);
}
public Map<String,Object> toMap(){
Map<String,Object> map = super.toMap();
public Map<String,Object> toMap(Map<String,Object> map){
map = super.toMap(map);
if(null != children && 0 != children.size()){
List<Map<String,Object>> childrenMaps = new ArrayList<Map<String,Object>>();
for(Explanation child : children){
childrenMaps.add(child.toMap());
childrenMaps.add(child.toMap(new LinkedHashMap<>()));
}
map.put("children", childrenMaps);
}