From 4d6be313323cf4389c50b87cd6da5db8d7314627 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Tue, 8 Nov 2016 18:41:26 +0530 Subject: [PATCH] SOLR-9720: Use the new MapWriter and IteratorWriter interface --- .../apache/solr/client/solrj/io/Tuple.java | 19 ++++-- .../client/solrj/io/stream/TupleStream.java | 67 ++++++------------- .../apache/solr/common/IteratorWriter.java | 6 +- .../org/apache/solr/common/MapWriter.java | 17 +++-- 4 files changed, 50 insertions(+), 59 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java index aee074e8dd0..58d948dd65f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Tuple.java @@ -16,14 +16,15 @@ */ package org.apache.solr.client.solrj.io; +import java.io.IOException; import java.time.Instant; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; -import java.util.Map; import java.util.List; +import java.util.Map; -import org.apache.solr.common.MapSerializable; +import org.apache.solr.common.MapWriter; /** * A simple abstraction of a record containing key/value pairs. @@ -32,7 +33,7 @@ import org.apache.solr.common.MapSerializable; * **/ -public class Tuple implements Cloneable, MapSerializable { +public class Tuple implements Cloneable, MapWriter { /** * When EOF field is true the Tuple marks the end of the stream. @@ -90,7 +91,7 @@ public class Tuple implements Cloneable, MapSerializable { } } - // Convenience method since Booleans can be pased around as Strings. + // Convenience method since Booleans can be passed around as Strings. public Boolean getBool(Object key) { Object o = this.fields.get(key); @@ -196,7 +197,13 @@ public class Tuple implements Cloneable, MapSerializable { } @Override - public Map toMap(Map map) { - return fields; + public void writeMap(EntryWriter ew) throws IOException { + fields.forEach((k, v) -> { + try { + ew.put((String)k,v); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TupleStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TupleStream.java index 6f381ec68b5..49a806f1660 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TupleStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TupleStream.java @@ -19,21 +19,19 @@ package org.apache.solr.client.solrj.io.stream; import java.io.Closeable; import java.io.IOException; import java.io.Serializable; -import java.util.Collections; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.UUID; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; import org.apache.solr.client.solrj.io.stream.expr.Explanation; import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; -import org.apache.solr.common.MapSerializable; +import org.apache.solr.common.IteratorWriter; +import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; -public abstract class TupleStream implements Closeable, Serializable, MapSerializable { +public abstract class TupleStream implements Closeable, Serializable, MapWriter { private static final long serialVersionUID = 1; @@ -42,15 +40,6 @@ public abstract class TupleStream implements Closeable, Serializable, MapSeriali public TupleStream() { } -/* - public static void writeStreamOpen(Writer out) throws IOException { - out.write("{\"docs\":["); - } - - public static void writeStreamClose(Writer out) throws IOException { - out.write("]}"); - }*/ - public abstract void setStreamContext(StreamContext context); public abstract List children(); @@ -69,41 +58,25 @@ public abstract class TupleStream implements Closeable, Serializable, MapSeriali return 0; } - private boolean isOpen = false; - @Override - public Map toMap(Map map) { - try { - if (!isOpen) { - open(); - isOpen = true; - } - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - return Collections.singletonMap("docs", new Iterator() { - Tuple tuple; - boolean isEOF = false; - - @Override - public boolean hasNext() { - if (isEOF) return false; - if (tuple != null) return true; - try { - tuple = read(); - if(tuple != null && tuple.EOF) close(); - return tuple != null; - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + public void writeMap(EntryWriter ew) throws IOException { + open(); + ew.put("docs", (IteratorWriter) iw -> { + try { + for (; ; ) { + Tuple tuple = read(); + if (tuple != null) { + iw.add(tuple); + if (tuple.EOF) { + close(); + break; + } + } else { + break; + } } - } - - @Override - public Tuple next() { - Tuple tmp = tuple; - tuple = null; - isEOF = tmp == null || tmp.EOF; - return tmp; + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } }); } diff --git a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java index 1294784314d..0049a5bda99 100644 --- a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java +++ b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java @@ -25,12 +25,14 @@ import java.io.IOException; */ public interface IteratorWriter { /** - * @param w after this method returns , the EntryWriter Object is invalid + * @param iw after this method returns , the EntryWriter Object is invalid * Do not hold a reference to this object */ - void writeIter(ItemWriter w) throws IOException; + void writeIter(ItemWriter iw) throws IOException; interface ItemWriter { + /**The item could be any supported type + */ ItemWriter add(Object o) throws IOException; default ItemWriter add(int v) throws IOException { diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java index d5dad2a1d19..8fb9d031b27 100644 --- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java +++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java @@ -22,7 +22,9 @@ import java.io.IOException; import java.util.Map; /** - * Use this class if the Map size is not known + * Use this class to push all entries of a Map into an output. + * This avoids creating map instances and is supposed to be memory efficient. + * If the entries are primitives, unnecessary boxing is also avoided */ public interface MapWriter extends MapSerializable { @@ -45,8 +47,14 @@ public interface MapWriter extends MapSerializable { void writeMap(EntryWriter ew) throws IOException; + /** + * An interface to push one entry at a time to the output + */ interface EntryWriter { - /**Writes a key value into the map + + /** + * Writes a key value into the map + * * @param k The key * @param v The value can be any supported object */ @@ -69,11 +77,12 @@ public interface MapWriter extends MapSerializable { return this; } - default EntryWriter put(String k, double v) throws IOException{ + default EntryWriter put(String k, double v) throws IOException { put(k, (Double) v); return this; } - default EntryWriter put(String k, boolean v) throws IOException{ + + default EntryWriter put(String k, boolean v) throws IOException { put(k, (Boolean) v); return this; }