From edfcaa61610571de8fedac2147178419414284f7 Mon Sep 17 00:00:00 2001 From: Yonik Seeley Date: Tue, 15 Apr 2008 19:22:38 +0000 Subject: [PATCH] SOLR-486: Binary response format git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@648384 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 10 +- .../solrj/impl/BinaryResponseParser.java | 45 ++ .../client/solrj/impl/XMLResponseParser.java | 5 +- .../apache/solr/common/SolrDocumentList.java | 8 + .../solr/common/util/FastInputStream.java | 211 +++++++ .../solr/common/util/FastOutputStream.java | 188 ++++++ .../solr/common/util/NamedListCodec.java | 565 ++++++++++++++++++ src/java/org/apache/solr/core/SolrCore.java | 13 +- .../solr/handler/AnalysisRequestHandler.java | 8 +- .../handler/component/DebugComponent.java | 2 +- .../handler/component/FacetComponent.java | 10 +- .../handler/component/HighlightComponent.java | 4 +- .../handler/component/QueryComponent.java | 2 +- .../solr/handler/component/SearchHandler.java | 29 +- .../solr/request/BinaryResponseWriter.java | 146 +++++ .../solr/schema/SortableDoubleField.java | 2 +- .../solr/schema/SortableFloatField.java | 2 +- .../apache/solr/schema/SortableIntField.java | 2 +- .../apache/solr/schema/SortableLongField.java | 2 +- .../processor/LogUpdateProcessorFactory.java | 3 +- .../apache/solr/TestDistributedSearch.java | 23 +- .../solr/common/util/TestNamedListCodec.java | 187 ++++++ .../apache/solr/request/TestWriterPerf.java | 178 ++++++ src/test/test-files/solr/conf/schema11.xml | 1 + .../solr/servlet/SolrDispatchFilter.java | 2 +- 25 files changed, 1607 insertions(+), 41 deletions(-) create mode 100755 client/java/solrj/src/org/apache/solr/client/solrj/impl/BinaryResponseParser.java create mode 100755 src/java/org/apache/solr/common/util/FastInputStream.java create mode 100755 src/java/org/apache/solr/common/util/FastOutputStream.java create mode 100755 src/java/org/apache/solr/common/util/NamedListCodec.java create mode 100755 src/java/org/apache/solr/request/BinaryResponseWriter.java create mode 100755 src/test/org/apache/solr/common/util/TestNamedListCodec.java create mode 100755 src/test/org/apache/solr/request/TestWriterPerf.java diff --git a/CHANGES.txt b/CHANGES.txt index f259b226cb5..acf4be16708 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -239,8 +239,14 @@ New Features 42. SOLR-494: Added cool admin Ajaxed schema explorer. (Greg Ludington via ehatcher) -43. SOLR-497: Added date faceting to the QueryResponse in SolrJ and QueryResponseTest (Shalin Shekhar Mangar via gsingers) - +43. SOLR-497: Added date faceting to the QueryResponse in SolrJ + and QueryResponseTest (Shalin Shekhar Mangar via gsingers) + +44. SOLR-486: Binary response format, faster and smaller + than XML and JSON response formats (use wt=javabin). + BinaryResponseParser for utilizing the binary format via SolrJ. + (Noble Paul, yonik) + Changes in runtime behavior Optimizations diff --git a/client/java/solrj/src/org/apache/solr/client/solrj/impl/BinaryResponseParser.java b/client/java/solrj/src/org/apache/solr/client/solrj/impl/BinaryResponseParser.java new file mode 100755 index 00000000000..10ad6c68816 --- /dev/null +++ b/client/java/solrj/src/org/apache/solr/client/solrj/impl/BinaryResponseParser.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.impl; + +import org.apache.solr.client.solrj.ResponseParser; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.NamedListCodec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; + +public class BinaryResponseParser extends ResponseParser { + public String getWriterType() { + return "javabin"; + } + + public NamedList processResponse(InputStream body, String encoding) { + try { + return new NamedListCodec().unmarshal(body); + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "parsing error", e); + + } + } + + public NamedList processResponse(Reader reader) { + throw new RuntimeException("Cannot handle character stream"); + } +} diff --git a/client/java/solrj/src/org/apache/solr/client/solrj/impl/XMLResponseParser.java b/client/java/solrj/src/org/apache/solr/client/solrj/impl/XMLResponseParser.java index be0c0b8ff8b..c17a436a578 100644 --- a/client/java/solrj/src/org/apache/solr/client/solrj/impl/XMLResponseParser.java +++ b/client/java/solrj/src/org/apache/solr/client/solrj/impl/XMLResponseParser.java @@ -35,6 +35,7 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; /** * @@ -123,7 +124,7 @@ public class XMLResponseParser extends ResponseParser response = readNamedList( parser ); } else if( name.equals( "solr" ) ) { - return new NamedList(); + return new SimpleOrderedMap(); } else { throw new Exception( "really needs to be response or result. " + @@ -200,7 +201,7 @@ public class XMLResponseParser extends ResponseParser } StringBuilder builder = new StringBuilder(); - NamedList nl = new NamedList(); + NamedList nl = new SimpleOrderedMap(); KnownType type = null; String name = null; diff --git a/src/java/org/apache/solr/common/SolrDocumentList.java b/src/java/org/apache/solr/common/SolrDocumentList.java index 715518f109e..273cd80751a 100644 --- a/src/java/org/apache/solr/common/SolrDocumentList.java +++ b/src/java/org/apache/solr/common/SolrDocumentList.java @@ -56,4 +56,12 @@ public class SolrDocumentList extends ArrayList public void setStart(long start) { this.start = start; } + + public String toString() { + return "{numFound="+numFound + +",start="+start + + (maxScore!=null ? ""+maxScore : "") + +",docs="+super.toString() + +"}"; + } } diff --git a/src/java/org/apache/solr/common/util/FastInputStream.java b/src/java/org/apache/solr/common/util/FastInputStream.java new file mode 100755 index 00000000000..3abddc54797 --- /dev/null +++ b/src/java/org/apache/solr/common/util/FastInputStream.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.common.util; + +import java.io.*; + +/** Single threaded buffered InputStream + * Internal Solr use only, subject to change. + */ +public class FastInputStream extends InputStream implements DataInput { + private final InputStream in; + private final byte[] buf; + private int pos; + private int end; + + public FastInputStream(InputStream in) { + // use default BUFSIZE of BufferedOutputStream so if we wrap that + // it won't cause double buffering. + this(in, new byte[8192], 0, 0); + } + + public FastInputStream(InputStream in, byte[] tempBuffer, int start, int end) { + this.in = in; + this.buf = tempBuffer; + this.pos = start; + this.end = end; + } + + + public static FastInputStream wrap(InputStream in) { + return (in instanceof FastInputStream) ? (FastInputStream)in : new FastInputStream(in); + } + + @Override + public int read() throws IOException { + if (pos >= end) { + refill(); + if (pos >= end) return -1; + } + return buf[pos++] & 0xff; + } + + public int readUnsignedByte() throws IOException { + if (pos >= end) { + refill(); + if (pos >= end) throw new EOFException(); + } + return buf[pos++] & 0xff; + } + + public void refill() throws IOException { + // this will set end to -1 at EOF + end = in.read(buf, 0, buf.length); + pos = 0; + } + + @Override + public int available() throws IOException { + return end - pos; + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + int r=0; // number of bytes read + // first read from our buffer; + if (end-pos > 0) { + r = Math.min(end-pos, len); + System.arraycopy(buf, pos, b, off, r); + pos += r; + } + + if (r == len) return r; + + // amount left to read is >= buffer size + if (len-r >= buf.length) { + int ret = in.read(b, off+r, len-r); + if (ret==-1) return r==0 ? -1 : r; + r += ret; + return r; + } + + refill(); + + // first read from our buffer; + if (end-pos > 0) { + int toRead = Math.min(end-pos, len-r); + System.arraycopy(buf, pos, b, off+r, toRead); + pos += toRead; + r += toRead; + return r; + } + + return -1; + } + + @Override + public void close() throws IOException { + in.close(); + } + + public void readFully(byte b[]) throws IOException { + readFully(b, 0, b.length); + } + + public void readFully(byte b[], int off, int len) throws IOException { + while (len>0) { + int ret = read(b, off, len); + if (ret==-1) { + throw new EOFException(); + } + off += ret; + len -= ret; + } + } + + public int skipBytes(int n) throws IOException { + if (end-pos >= n) { + pos += n; + return n; + } + + if (end-pos<0) return -1; + + int r = end-pos; + pos = end; + + while (r < n) { + refill(); + if (end-pos <= 0) return r; + int toRead = Math.min(end-pos, n-r); + r += toRead; + pos += toRead; + } + + return r; + } + + public boolean readBoolean() throws IOException { + return readByte()==1; + } + + public byte readByte() throws IOException { + if (pos >= end) { + refill(); + if (pos >= end) throw new EOFException(); + } + return buf[pos++]; + } + + + public short readShort() throws IOException { + return (short)((readUnsignedByte() << 8) | readUnsignedByte()); + } + + public int readUnsignedShort() throws IOException { + return (readUnsignedByte() << 8) | readUnsignedByte(); + } + + public char readChar() throws IOException { + return (char)((readUnsignedByte() << 8) | readUnsignedByte()); + } + + public int readInt() throws IOException { + return ((readUnsignedByte() << 24) + |(readUnsignedByte() << 16) + |(readUnsignedByte() << 8) + | readUnsignedByte()); + } + + public long readLong() throws IOException { + return (((long)readUnsignedByte()) << 56) + | (((long)readUnsignedByte()) << 48) + | (((long)readUnsignedByte()) << 40) + | (((long)readUnsignedByte()) << 32) + | (((long)readUnsignedByte()) << 24) + | (readUnsignedByte() << 16) + | (readUnsignedByte() << 8) + | (readUnsignedByte()); + } + + public float readFloat() throws IOException { + return Float.intBitsToFloat(readInt()); + } + + public double readDouble() throws IOException { + return Double.longBitsToDouble(readLong()); + } + + public String readLine() throws IOException { + return new DataInputStream(this).readLine(); + } + + public String readUTF() throws IOException { + return new DataInputStream(this).readUTF(); + } +} diff --git a/src/java/org/apache/solr/common/util/FastOutputStream.java b/src/java/org/apache/solr/common/util/FastOutputStream.java new file mode 100755 index 00000000000..3f6275c2ee7 --- /dev/null +++ b/src/java/org/apache/solr/common/util/FastOutputStream.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.common.util; + +import java.io.*; + +/** Single threaded buffered OutputStream + * Internal Solr use only, subject to change. + */ +public class FastOutputStream extends OutputStream implements DataOutput { + private final OutputStream out; + private final byte[] buf; + private long written; // how many bytes written + private int pos; + + public FastOutputStream(OutputStream w) { + // use default BUFSIZE of BufferedOutputStream so if we wrap that + // it won't cause double buffering. + this(w, new byte[8192], 0); + } + + public FastOutputStream(OutputStream sink, byte[] tempBuffer, int start) { + this.out = sink; + this.buf = tempBuffer; + this.pos = start; + } + + + public static FastOutputStream wrap(OutputStream sink) { + return (sink instanceof FastOutputStream) ? (FastOutputStream)sink : new FastOutputStream(sink); + } + + @Override + public void write(int b) throws IOException { + write((byte)b); + } + + public void write(byte b[]) throws IOException { + } + + public void write(byte b) throws IOException { + if (pos >= buf.length) { + out.write(buf); + written += pos; + pos=0; + } + buf[pos++] = b; + } + + @Override + public void write(byte arr[], int off, int len) throws IOException { + int space = buf.length - pos; + if (len < space) { + System.arraycopy(arr, off, buf, pos, len); + pos += len; + } else if (len buffer.length + * @param len + */ + public void reserve(int len) throws IOException { + if (len > (buf.length - pos)) + flushBuffer(); + } + + ////////////////// DataOutput methods /////////////////// + public void writeBoolean(boolean v) throws IOException { + write(v ? 1:0); + } + + public void writeByte(int v) throws IOException { + write((byte)v); + } + + public void writeShort(int v) throws IOException { + write((byte)(v >>> 8)); + write((byte)v); + } + + public void writeChar(int v) throws IOException { + writeShort(v); + } + + public void writeInt(int v) throws IOException { + reserve(4); + buf[pos] = (byte)(v>>>24); + buf[pos+1] = (byte)(v>>>16); + buf[pos+2] = (byte)(v>>>8); + buf[pos+3] = (byte)(v); + pos+=4; + } + + public void writeLong(long v) throws IOException { + reserve(8); + buf[pos] = (byte)(v>>>56); + buf[pos+1] = (byte)(v>>>48); + buf[pos+2] = (byte)(v>>>40); + buf[pos+3] = (byte)(v>>>32); + buf[pos+4] = (byte)(v>>>24); + buf[pos+5] = (byte)(v>>>16); + buf[pos+6] = (byte)(v>>>8); + buf[pos+7] = (byte)(v); + pos+=8; + } + + public void writeFloat(float v) throws IOException { + writeInt(Float.floatToRawIntBits(v)); + } + + public void writeDouble(double v) throws IOException { + writeLong(Double.doubleToRawLongBits(v)); + } + + public void writeBytes(String s) throws IOException { + // non-optimized version, but this shouldn't be used anyway + for (int i=0; i>>5) { + case STR >>> 5 : return readStr(dis); + case SINT >>>5 : return readSmallInt(dis); + case SLONG >>>5 : return readSmallLong(dis); + case ARR >>> 5 : return readArray(dis); + case ORDERED_MAP >>> 5 : return readOrderedMap(dis); + case NAMED_LST >>> 5 : return readNamedList(dis); + } + + switch(tagByte){ + case NULL : return null; + case DATE : return new Date(dis.readLong()); + case INT : return dis.readInt(); + case BOOL_TRUE : return Boolean.TRUE; + case BOOL_FALSE : return Boolean.FALSE; + case FLOAT : return dis.readFloat(); + case DOUBLE : return dis.readDouble(); + case LONG : return dis.readLong(); + case BYTE : return dis.readByte(); + case SHORT : return dis.readShort(); + case MAP : return readMap(dis); + case SOLRDOC : return readSolrDocument(dis); + case SOLRDOCLST : return readSolrDocumentList(dis); + case BYTEARR : return readByteArray(dis); + } + + throw new RuntimeException("Unknown type " + tagByte); + } + + private boolean writeKnownType(Object val) throws IOException { + if (writePrimitive(val)) return true; + if (val instanceof NamedList) { + writeNamedList((NamedList) val); + return true; + } + if (val instanceof SolrDocumentList) { // SolrDocumentList is a List, so must come before List check + writeSolrDocumentList((SolrDocumentList) val); + return true; + } + if (val instanceof List) { + writeArray((List) val); + return true; + } + if (val instanceof Object[]) { + writeArray((Object[]) val); + return true; + } + if (val instanceof SolrDocument) { + writeSolrDocument((SolrDocument) val); + return true; + } + if (val instanceof Iterator) { + writeIterator((Iterator) val); + return true; + } + if (val instanceof Map) { + writeMap((Map) val); + return true; + } + return false; + } + + public void writeTag(byte tag) throws IOException { + daos.writeByte(tag); + } + + public void writeTag(byte tag, int size) throws IOException { + if ((tag & 0xe0) != 0) { + if (size < 0x1f) { + daos.writeByte(tag | size); + } else { + daos.writeByte(tag | 0x1f); + writeVInt(size-0x1f, daos); + } + } else { + daos.writeByte(tag); + writeVInt(size, daos); + } + } + + private void writeByteArray(byte[] arr, int offset, int len) throws IOException { + writeTag(BYTEARR, len); + daos.write(arr, offset, len); + } + + private byte[] readByteArray(FastInputStream dis) throws IOException { + byte[] arr = new byte[readVInt(dis)]; + dis.readFully(arr); + return arr; + } + + public void writeSolrDocument(SolrDocument doc) throws IOException { + writeTag(SOLRDOC); + writeTag(ORDERED_MAP, doc.getFieldNames().size()); + for (Map.Entry entry : doc) { + String name = entry.getKey(); + writeStr(name); + Object val = entry.getValue(); + writeVal(val); + } + } + + private SolrDocument readSolrDocument(FastInputStream dis) throws IOException { + NamedList nl = (NamedList) readVal(dis); + SolrDocument doc = new SolrDocument(); + for (int i = 0; i < nl.size(); i++) { + String name = nl.getName(i); + Object val = nl.getVal(i); + doc.setField(name, val); + } + return doc; + } + + private SolrDocumentList readSolrDocumentList(FastInputStream dis) throws IOException { + SolrDocumentList solrDocs = new SolrDocumentList(); + List list = (List) readVal(dis); + solrDocs.setNumFound((Long) list.get(0)); + solrDocs.setStart((Long)list.get(1)); + solrDocs.setMaxScore((Float)list.get(2)); + + List l = (List) readVal(dis); + solrDocs.addAll(l); + return solrDocs; + } + + private void writeSolrDocumentList(SolrDocumentList docs) + throws IOException { + writeTag(SOLRDOCLST); + List l = new ArrayList(3); + l.add(docs.getNumFound()); + l.add(docs.getStart()); + l.add(docs.getMaxScore()); + writeArray(l); + writeArray(docs); + } + + private Map readMap(FastInputStream dis) + throws IOException { + int sz = readVInt(dis); + Map m = new LinkedHashMap(); + for (int i = 0; i < sz; i++) { + Object key = readVal(dis); + Object val = readVal(dis); + m.put(key, val); + + } + return m; + } + + private void writeIterator(Iterator iter) throws IOException { + ArrayList l = new ArrayList(); + while (iter.hasNext()) l.add(iter.next()); + writeArray(l); + } + + public void writeArray(List l) throws IOException { + writeTag(ARR, l.size()); + for (int i = 0; i < l.size(); i++) { + writeVal(l.get(i)); + } + } + + public void writeArray(Object[] arr) throws IOException { + writeTag(ARR, arr.length); + for (int i = 0; i < arr.length; i++) { + Object o = arr[i]; + writeVal(o); + } + } + + private List readArray(FastInputStream dis) throws IOException { + int sz = readSize(dis); + ArrayList l = new ArrayList(sz); + for (int i = 0; i < sz; i++) { + l.add(readVal(dis)); + } + return l; + } + + /** write the string as tag+length, with length being the number of UTF-16 characters, + * followed by the string encoded in modified-UTF8 + */ + private void writeStr(String s) throws IOException { + if (s==null) { + writeTag(NULL); + } + // Can't use string serialization or toUTF()... it's limited to 64K + // plus it's bigger than it needs to be for small strings anyway + int len = s.length(); + writeTag(STR, len); + writeChars(daos, s, 0, len); + } + + + char[] charArr; + private String readStr(FastInputStream dis) throws IOException { + int sz = readSize(dis); + if (charArr==null || charArr.length < sz) { + charArr = new char[sz]; + } + readChars(dis, charArr, 0, sz); + return new String(charArr, 0, sz); + } + + private void writeInt(int val) throws IOException { + if (val>0) { + int b = SINT | (val & 0x0f); + + if (val >= 0x0f) { + b |= 0x10; + daos.writeByte(b); + writeVInt(val>>>4, daos); + } else { + daos.writeByte(b); + } + + } else { + daos.writeByte(INT); + daos.writeInt(val); + } + } + + private int readSmallInt(FastInputStream dis) throws IOException { + int v = tagByte & 0x0F; + if ((tagByte & 0x10) != 0) + v = (readVInt(dis)<<4) | v; + return v; + } + + + private void writeLong(long val) throws IOException { + if ((val & 0xff00000000000000L) == 0) { + int b = SLONG | ((int)val & 0x0f); + if (val >= 0x0f) { + b |= 0x10; + daos.writeByte(b); + writeVLong(val>>>4, daos); + } else { + daos.writeByte(b); + } + } else { + daos.writeByte(LONG); + daos.writeLong(val); + } + } + + private long readSmallLong(FastInputStream dis) throws IOException { + long v = tagByte & 0x0F; + if ((tagByte & 0x10) != 0) + v = (readVLong(dis)<<4) | v; + return v; + } + + private boolean writePrimitive(Object val) throws IOException { + if (val == null) { + daos.writeByte(NULL); + return true; + } else if (val instanceof String) { + writeStr((String)val); + return true; + } else if (val instanceof Integer) { + writeInt(((Integer)val).intValue()); + return true; + } else if (val instanceof Long) { + writeLong(((Long)val).longValue()); + return true; + } else if (val instanceof Float) { + daos.writeByte(FLOAT); + daos.writeFloat(((Float) val).floatValue()); + return true; + } else if (val instanceof Date) { + daos.writeByte(DATE); + daos.writeLong(((Date) val).getTime()); + return true; + } else if (val instanceof Boolean) { + if ((Boolean) val) daos.writeByte(BOOL_TRUE); + else daos.writeByte(BOOL_FALSE); + return true; + } else if (val instanceof Double) { + daos.writeByte(DOUBLE); + daos.writeDouble(((Double) val).doubleValue()); + return true; + } else if (val instanceof Byte) { + daos.writeByte(BYTE); + daos.writeByte(((Byte) val).intValue()); + return true; + } else if (val instanceof Short) { + daos.writeByte(SHORT); + daos.writeShort(((Short) val).intValue()); + return true; + } else if (val instanceof byte[]) { + writeByteArray((byte[])val, 0, ((byte[])val).length); + return true; + } + return false; + } + + private void writeMap( Map val) + throws IOException { + writeTag(MAP, val.size()); + for (Map.Entry entry : (Set) val.entrySet()) { + writeVal(entry.getKey()); + writeVal(entry.getValue()); + } + } + + + private int readSize(FastInputStream in) throws IOException { + int sz = tagByte & 0x1f; + if (sz == 0x1f) sz += readVInt(in); + return sz; + } + + + + /** + * Special method for variable length int (copied from lucene). Usually used for writing the length of a collection/array/map + * In most of the cases the length can be represented in one byte (length < 127) so it saves 3 bytes/object + * + * @param i + * @param out + * @throws IOException + */ + private static void writeVInt(int i, FastOutputStream out) throws IOException { + while ((i & ~0x7F) != 0) { + out.writeByte((byte) ((i & 0x7f) | 0x80)); + i >>>= 7; + } + out.writeByte((byte) i); + } + + /** + * The counterpart for the above + * + * @param in + * @return the int value + * @throws IOException + */ + public static int readVInt(FastInputStream in) throws IOException { + byte b = in.readByte(); + int i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) { + b = in.readByte(); + i |= (b & 0x7F) << shift; + } + return i; + } + + + private static void writeVLong(long i, FastOutputStream out) throws IOException { + while ((i & ~0x7F) != 0) { + out.writeByte((byte)((i & 0x7f) | 0x80)); + i >>>= 7; + } + out.writeByte((byte) i); + } + + private static long readVLong(FastInputStream in) throws IOException { + byte b = in.readByte(); + long i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) { + b = in.readByte(); + i |= (long)(b & 0x7F) << shift; + } + return i; + } + + /** Writes a sequence of UTF-8 encoded characters from a string. + * @param s the source of the characters + * @param start the first character in the sequence + * @param length the number of characters in the sequence + * @see org.apache.lucene.store.IndexInput#readChars(char[],int,int) + */ + public static void writeChars(FastOutputStream os, String s, int start, int length) + throws IOException { + final int end = start + length; + for (int i = start; i < end; i++) { + final int code = (int)s.charAt(i); + if (code >= 0x01 && code <= 0x7F) + os.write(code); + else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) { + os.write(0xC0 | (code >> 6)); + os.write(0x80 | (code & 0x3F)); + } else { + os.write(0xE0 | (code >>> 12)); + os.write(0x80 | ((code >> 6) & 0x3F)); + os.write(0x80 | (code & 0x3F)); + } + } + } + + /** Reads UTF-8 encoded characters into an array. + * @param buffer the array to read characters into + * @param start the offset in the array to start storing characters + * @param length the number of characters to read + * @see org.apache.lucene.store.IndexOutput#writeChars(String,int,int) + */ + public static void readChars(FastInputStream in, char[] buffer, int start, int length) + throws IOException { + final int end = start + length; + for (int i = start; i < end; i++) { + int b = in.read(); + if ((b & 0x80) == 0) + buffer[i] = (char)b; + else if ((b & 0xE0) != 0xE0) { + buffer[i] = (char)(((b & 0x1F) << 6) + | (in.read() & 0x3F)); + } else + buffer[i] = (char)(((b & 0x0F) << 12) + | ((in.read() & 0x3F) << 6) + | (in.read() & 0x3F)); + } + } + + + public static interface ObjectResolver{ + public Object resolve(Object o, NamedListCodec codec) throws IOException; + } + + +} diff --git a/src/java/org/apache/solr/core/SolrCore.java b/src/java/org/apache/solr/core/SolrCore.java index 3540acbd91a..a518e9d7832 100644 --- a/src/java/org/apache/solr/core/SolrCore.java +++ b/src/java/org/apache/solr/core/SolrCore.java @@ -51,15 +51,7 @@ import org.apache.solr.handler.component.QueryComponent; import org.apache.solr.handler.component.SearchComponent; import org.apache.solr.highlight.DefaultSolrHighlighter; import org.apache.solr.highlight.SolrHighlighter; -import org.apache.solr.request.JSONResponseWriter; -import org.apache.solr.request.PythonResponseWriter; -import org.apache.solr.request.QueryResponseWriter; -import org.apache.solr.request.RawResponseWriter; -import org.apache.solr.request.RubyResponseWriter; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.request.SolrQueryResponse; -import org.apache.solr.request.SolrRequestHandler; -import org.apache.solr.request.XMLResponseWriter; +import org.apache.solr.request.*; import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.QParserPlugin; @@ -1038,6 +1030,9 @@ public final class SolrCore { if (responseWriters.get("raw")==null) { responseWriters.put("raw", new RawResponseWriter()); } + if (responseWriters.get("javabin") == null) { + responseWriters.put("javabin", new BinaryResponseWriter()); + } } /** Finds a writer by name, or returns the default writer if not found. */ diff --git a/src/java/org/apache/solr/handler/AnalysisRequestHandler.java b/src/java/org/apache/solr/handler/AnalysisRequestHandler.java index b0cd1b46f1d..bdabd579f58 100644 --- a/src/java/org/apache/solr/handler/AnalysisRequestHandler.java +++ b/src/java/org/apache/solr/handler/AnalysisRequestHandler.java @@ -26,6 +26,7 @@ import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryResponse; import org.apache.solr.schema.FieldType; @@ -96,7 +97,7 @@ public class AnalysisRequestHandler extends RequestHandlerBase { NamedList processContent(XMLStreamReader parser, IndexSchema schema) throws XMLStreamException, IOException { - NamedList result = new NamedList(); + NamedList result = new SimpleOrderedMap(); while (true) { int event = parser.next(); switch (event) { @@ -111,7 +112,7 @@ public class AnalysisRequestHandler extends RequestHandlerBase { SolrInputDocument doc = readDoc(parser); SchemaField uniq = schema.getUniqueKeyField(); - NamedList>> theTokens = new NamedList>>(); + NamedList>> theTokens = new SimpleOrderedMap>>(); result.add(doc.getFieldValue(uniq.getName()).toString(), theTokens); for (String name : doc.getFieldNames()) { FieldType ft = schema.getFieldType(name); @@ -132,10 +133,11 @@ public class AnalysisRequestHandler extends RequestHandlerBase { } static NamedList> getTokens(TokenStream tstream) throws IOException { + // outer is namedList since order of tokens is important NamedList> tokens = new NamedList>(); Token t = null; while (((t = tstream.next()) != null)) { - NamedList token = new NamedList(); + NamedList token = new SimpleOrderedMap(); tokens.add("token", token); token.add("value", new String(t.termBuffer(), 0, t.termLength())); token.add("start", t.startOffset()); diff --git a/src/java/org/apache/solr/handler/component/DebugComponent.java b/src/java/org/apache/solr/handler/component/DebugComponent.java index 7ef13401a5d..5cd396a49e8 100644 --- a/src/java/org/apache/solr/handler/component/DebugComponent.java +++ b/src/java/org/apache/solr/handler/component/DebugComponent.java @@ -129,7 +129,7 @@ public class DebugComponent extends SearchComponent } } - explain = HighlightComponent.removeNulls(new NamedList(Arrays.asList(arr))); + explain = HighlightComponent.removeNulls(new SimpleOrderedMap(Arrays.asList(arr))); int idx = info.indexOf("explain",0); if (idx>=0) { info.setVal(idx, explain); diff --git a/src/java/org/apache/solr/handler/component/FacetComponent.java b/src/java/org/apache/solr/handler/component/FacetComponent.java index 151d49fa775..24c386263a2 100644 --- a/src/java/org/apache/solr/handler/component/FacetComponent.java +++ b/src/java/org/apache/solr/handler/component/FacetComponent.java @@ -323,18 +323,18 @@ System.out.println("Got " + facet_q + " , refining count: " + sfc + " += " + cou FacetInfo fi = rb._facetInfo; - NamedList facet_counts = new NamedList(); - NamedList facet_queries = new NamedList(); + NamedList facet_counts = new SimpleOrderedMap(); + NamedList facet_queries = new SimpleOrderedMap(); facet_counts.add("facet_queries",facet_queries); for (Map.Entry entry : fi.queryFacets.entrySet()) { facet_queries.add(entry.getKey(), num(entry.getValue())); } - NamedList facet_fields = new NamedList(); + NamedList facet_fields = new SimpleOrderedMap(); facet_counts.add("facet_fields", facet_fields); for (DistribFieldFacet dff : fi.topFacets.values()) { - SimpleOrderedMap fieldCounts = new SimpleOrderedMap(); + NamedList fieldCounts = new NamedList(); // order is more important for facets facet_fields.add(dff.field, fieldCounts); ShardFacetCount[] counts = dff.countSorted; @@ -355,7 +355,7 @@ System.out.println("Got " + facet_q + " , refining count: " + sfc + " += " + cou // TODO: list facets (sorted by natural order) // TODO: facet dates - facet_counts.add("facet_dates", new NamedList()); + facet_counts.add("facet_dates", new SimpleOrderedMap()); rb.rsp.add("facet_counts", facet_counts); diff --git a/src/java/org/apache/solr/handler/component/HighlightComponent.java b/src/java/org/apache/solr/handler/component/HighlightComponent.java index c348cc72faf..d5d7a371fea 100644 --- a/src/java/org/apache/solr/handler/component/HighlightComponent.java +++ b/src/java/org/apache/solr/handler/component/HighlightComponent.java @@ -126,7 +126,7 @@ public class HighlightComponent extends SearchComponent } // remove nulls in case not all docs were able to be retrieved - rb.rsp.add("highlighting", removeNulls(new NamedList(Arrays.asList(arr)))); + rb.rsp.add("highlighting", removeNulls(new SimpleOrderedMap(Arrays.asList(arr)))); } } @@ -134,7 +134,7 @@ public class HighlightComponent extends SearchComponent static NamedList removeNulls(NamedList nl) { for (int i=0; i nl; + public long getElapsedTime() { + return elapsedTime; + } + + public NamedList getResponse() { + return nl; + } + } + void submit(final ShardRequest sreq, final String shard, final ModifiableSolrParams params) { Callable task = new Callable() { public ShardResponse call() throws Exception { @@ -323,6 +337,9 @@ class HttpCommComponent { ShardResponse srsp = new ShardResponse(); srsp.req = sreq; srsp.shard = shard; + SimpleSolrResponse ssr = new SimpleSolrResponse(); + srsp.rsp = ssr; + long startTime = System.currentTimeMillis(); try { // String url = "http://" + shard + "/select"; @@ -332,11 +349,15 @@ class HttpCommComponent { params.remove("version"); SolrServer server = new CommonsHttpSolrServer(url, client); - // SolrRequest req = new SolrRequest(SolrRequest.METHOD.GET, "/select"); + // SolrRequest req = new QueryRequest(SolrRequest.METHOD.POST, "/select"); // use generic request to avoid extra processing of queries - // QueryRequest req = new QueryRequest(sreq.params); + QueryRequest req = new QueryRequest(sreq.params); + req.setMethod(SolrRequest.METHOD.POST); + req.setResponseParser(new BinaryResponseParser()); // this sets the wt param // srsp.rsp = server.request(req); - srsp.rsp = server.query(sreq.params); + // srsp.rsp = server.query(sreq.params); + + ssr.nl = server.request(req); } catch (Throwable th) { srsp.exception = th; if (th instanceof SolrException) { @@ -346,6 +367,8 @@ class HttpCommComponent { } } + ssr.elapsedTime = System.currentTimeMillis() - startTime; + return srsp; } }; diff --git a/src/java/org/apache/solr/request/BinaryResponseWriter.java b/src/java/org/apache/solr/request/BinaryResponseWriter.java new file mode 100755 index 00000000000..e458530072c --- /dev/null +++ b/src/java/org/apache/solr/request/BinaryResponseWriter.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.request; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Fieldable; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.NamedListCodec; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.TextField; +import org.apache.solr.schema.FieldType; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrIndexSearcher; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.util.*; + + +public class BinaryResponseWriter implements BinaryQueryResponseWriter { + public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse response) throws IOException { + Resolver resolver = new Resolver(req, response.getReturnFields()); + NamedListCodec codec = new NamedListCodec(resolver); + codec.marshal(response.getValues(), out); + } + + public void write(Writer writer, SolrQueryRequest request, SolrQueryResponse response) throws IOException { + throw new RuntimeException("This is a binary writer , Cannot write to a characterstream"); + } + + public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { + return "application/octet-stream"; + } + + public void init(NamedList args) { + /* NOOP */ + } + + private static class Resolver implements NamedListCodec.ObjectResolver { + private final IndexSchema schema; + private final SolrIndexSearcher searcher; + private final Set returnFields; + private final boolean includeScore; + + // transmit field values using FieldType.toObject() + // rather than the String from FieldType.toExternal() + boolean useFieldObjects = true; + + public Resolver(SolrQueryRequest req, Set returnFields) { + this.schema = req.getSchema(); + this.searcher = req.getSearcher(); + this.includeScore = returnFields!=null && returnFields.contains("score"); + + if (returnFields != null) { + if (returnFields.size() == 0 || (returnFields.size() == 1 && includeScore) || returnFields.contains("*")) { + returnFields = null; // null means return all stored fields + } + } + this.returnFields = returnFields; + } + + public Object resolve(Object o, NamedListCodec codec) throws IOException { + if (o instanceof DocList) { + writeDocList((DocList) o, codec); + return null; // null means we completely handled it + } + + if (o instanceof Document) { + return getDoc((Document) o); + } + + return o; + } + + public void writeDocList(DocList ids, NamedListCodec codec) throws IOException { + codec.writeTag(NamedListCodec.SOLRDOCLST); + List l = new ArrayList(3); + l.add((long)ids.matches()); + l.add((long)ids.offset()); + Float maxScore = null; + if (includeScore && ids.hasScores()) { + maxScore = ids.maxScore(); + } + l.add(maxScore); + codec.writeArray(l); + + int sz = ids.size(); + codec.writeTag(NamedListCodec.ARR, sz); + + DocIterator iterator = ids.iterator(); + for (int i = 0; i < sz; i++) { + int id = iterator.nextDoc(); + Document doc = searcher.doc(id, returnFields); + + SolrDocument sdoc = getDoc(doc); + + if (includeScore && ids.hasScores()) { + sdoc.addField("score", iterator.score()); + } + + codec.writeSolrDocument(sdoc); + } + } + + + public SolrDocument getDoc(Document doc) { + SolrDocument solrDoc = new SolrDocument(); + for (Fieldable f : (List)doc.getFields()) { + String fieldName = f.name(); + if (returnFields!=null && !returnFields.contains(fieldName)) continue; + FieldType ft = schema.getFieldTypeNoEx(fieldName); + Object val; + if (ft==null) { // handle fields not in the schema + if (f.isBinary()) val = f.binaryValue(); + else val = f.stringValue(); + } else { + val = useFieldObjects ? ft.toObject(f) : ft.toExternal(f); + } + solrDoc.addField(fieldName, val); + } + return solrDoc; + } + + } + +} diff --git a/src/java/org/apache/solr/schema/SortableDoubleField.java b/src/java/org/apache/solr/schema/SortableDoubleField.java index 78572669263..579c7a1f909 100644 --- a/src/java/org/apache/solr/schema/SortableDoubleField.java +++ b/src/java/org/apache/solr/schema/SortableDoubleField.java @@ -55,7 +55,7 @@ public class SortableDoubleField extends FieldType { @Override public Double toObject(Fieldable f) { - return Double.valueOf( toExternal(f) ); + return NumberUtils.SortableStr2double(f.stringValue()); } public String indexedToReadable(String indexedForm) { diff --git a/src/java/org/apache/solr/schema/SortableFloatField.java b/src/java/org/apache/solr/schema/SortableFloatField.java index 87c552df6be..43841687d3a 100644 --- a/src/java/org/apache/solr/schema/SortableFloatField.java +++ b/src/java/org/apache/solr/schema/SortableFloatField.java @@ -55,7 +55,7 @@ public class SortableFloatField extends FieldType { @Override public Float toObject(Fieldable f) { - return Float.valueOf( toExternal(f) ); + return NumberUtils.SortableStr2float(f.stringValue()); } public String indexedToReadable(String indexedForm) { diff --git a/src/java/org/apache/solr/schema/SortableIntField.java b/src/java/org/apache/solr/schema/SortableIntField.java index 1fb78726bb1..b6deecd3ab7 100644 --- a/src/java/org/apache/solr/schema/SortableIntField.java +++ b/src/java/org/apache/solr/schema/SortableIntField.java @@ -62,7 +62,7 @@ public class SortableIntField extends FieldType { @Override public Integer toObject(Fieldable f) { - return Integer.valueOf( toExternal(f) ); + return NumberUtils.SortableStr2int(f.stringValue(), 0, 3); } public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException { diff --git a/src/java/org/apache/solr/schema/SortableLongField.java b/src/java/org/apache/solr/schema/SortableLongField.java index f20444366f0..7411b96ca18 100644 --- a/src/java/org/apache/solr/schema/SortableLongField.java +++ b/src/java/org/apache/solr/schema/SortableLongField.java @@ -59,7 +59,7 @@ public class SortableLongField extends FieldType { @Override public Long toObject(Fieldable f) { - return Long.valueOf( toExternal(f) ); + return NumberUtils.SortableStr2long(f.stringValue(),0,5); } public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException { diff --git a/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java b/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java index b14d58f02ce..c6d78845794 100644 --- a/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java +++ b/src/java/org/apache/solr/update/processor/LogUpdateProcessorFactory.java @@ -25,6 +25,7 @@ import java.util.logging.Level; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.DOMUtil; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryResponse; @@ -89,7 +90,7 @@ class LogUpdateProcessor extends UpdateRequestProcessor { // TODO: make log level configurable as well, or is that overkill? // (ryan) maybe? I added it mostly to show that it *can* be configurable - this.toLog = new NamedList(); + this.toLog = new SimpleOrderedMap(); } @Override diff --git a/src/test/org/apache/solr/TestDistributedSearch.java b/src/test/org/apache/solr/TestDistributedSearch.java index fffc86baff0..213e5f3651d 100755 --- a/src/test/org/apache/solr/TestDistributedSearch.java +++ b/src/test/org/apache/solr/TestDistributedSearch.java @@ -48,7 +48,6 @@ import junit.framework.TestCase; * @since solr 1.3 */ public class TestDistributedSearch extends TestCase { - Random r = new Random(0); File testDir; @@ -388,6 +387,18 @@ public class TestDistributedSearch extends TestCase { return compare((Object[])a, (Object[])b, flags, handle); } + if (a instanceof byte[] && b instanceof byte[]) { + if (!Arrays.equals((byte[])a, (byte[])b)) { + return ":" + a + "!=" + b; + } + return null; + } + + if (a instanceof List && b instanceof List) { + return compare(((List)a).toArray(), ((List)b).toArray(), flags, handle); + + } + if (!(a.equals(b))) { return ":" + a + "!=" + b; } @@ -397,10 +408,7 @@ public class TestDistributedSearch extends TestCase { void compareResponses(QueryResponse a, QueryResponse b) { - String cmp; - System.out.println(a); - System.out.println(b); - + String cmp; cmp = compare(a.getResponse(), b.getResponse(), flags, handle); if (cmp != null) { System.out.println(a); @@ -422,7 +430,8 @@ public class TestDistributedSearch extends TestCase { public void doTest() throws Exception { del("*:*"); - index(id,1, i1, 100,t1,"now is the time for all good men"); + index(id,1, i1, 100,t1,"now is the time for all good men" + ,"foo_f", 1.414f, "foo_b", "true", "foo_d", 1.414d); index(id,2, i1, 50 ,t1,"to come to the aid of their country."); index(id,3, i1, 2 ,t1,"how now brown cow"); index(id,4, i1, -100 ,t1,"the quick fox jumped over the lazy dog"); @@ -431,7 +440,7 @@ public class TestDistributedSearch extends TestCase { index(id,7, i1, 123 ,t1,"humpty dumpy had a great fall"); index(id,8, i1, 876 ,t1,"all the kings horses and all the kings men"); index(id,9, i1, 7 ,t1,"couldn't put humpty together again"); - index(id,10, i1, 4321 ,t1,"this too shal pass"); + index(id,10, i1, 4321 ,t1,"this too shall pass"); index(id,11, i1, -987 ,t1,"An eye for eye only ends up making the whole world blind."); index(id,12, i1, 379 ,t1,"Great works are performed, not by strength, but by perseverance."); index(id,13, i1, 232 ,t1,"no eggs on wall, lesson learned", oddField, "odd man out"); diff --git a/src/test/org/apache/solr/common/util/TestNamedListCodec.java b/src/test/org/apache/solr/common/util/TestNamedListCodec.java new file mode 100755 index 00000000000..42f73bedcfe --- /dev/null +++ b/src/test/org/apache/solr/common/util/TestNamedListCodec.java @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.common.util; + +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.TestDistributedSearch; + +import java.io.ByteArrayOutputStream; +import java.io.ByteArrayInputStream; +import java.util.List; +import java.util.Random; +import java.util.ArrayList; + +import junit.framework.TestCase; + +public class TestNamedListCodec extends TestCase { + public void testSimple() throws Exception{ + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + NamedList nl = new NamedList(); + Float fval = new Float( 10.01f ); + Boolean bval = Boolean.TRUE; + String sval = "12qwaszx"; + + // Set up a simple document + NamedList r = new NamedList(); + + + nl.add("responseHeader", r); + + r.add("status",0); + r.add("QTime",63); + NamedList p = new NamedList(); + r.add("params",p); + p.add("rows",10); + p.add("start",0); + p.add("indent","on"); + p.add("q","ipod"); + + + SolrDocumentList list = new SolrDocumentList(); + nl.add("response", list ); + list.setMaxScore(1.0f); + list.setStart(10); + list.setNumFound(12); + + SolrDocument doc = new SolrDocument(); + doc.addField( "f", fval ); + doc.addField( "b", bval ); + doc.addField( "s", sval ); + doc.addField( "f", 100 ); + list.add(doc); + + doc = new SolrDocument(); + doc.addField( "f", fval ); + doc.addField( "b", bval ); + doc.addField( "s", sval ); + doc.addField( "f", 101 ); + list.add(doc); + + nl.add("zzz",doc); + + new NamedListCodec(null).marshal(nl,baos); + byte[] arr = baos.toByteArray(); + nl = new NamedListCodec().unmarshal(new ByteArrayInputStream(arr)); + + + assertEquals(3, nl.size()); + assertEquals( "ipod",((NamedList)((NamedList)nl.getVal(0)).get("params")).get("q") ); + list = (SolrDocumentList) nl.getVal(1); + assertEquals(12,list.getNumFound() ); + assertEquals(10,list.getStart() ); + assertEquals(101, ((List)list.get(1).getFieldValue("f")).get(1)); + } + + + + + int rSz(int orderOfMagnitude) { + int sz = r.nextInt(orderOfMagnitude); + switch (sz) { + case 0: return r.nextInt(10); + case 1: return r.nextInt(100); + case 2: return r.nextInt(1000); + default: return r.nextInt(10000); + } + } + + public String rStr(int sz) { + StringBuilder sb = new StringBuilder(); + for (int i=0; i lst = new ArrayList(); + for (Object o : olst) lst.add(o.toString()); + assertU(adoc(lst.toArray(new String[lst.size()]))); + } + + void makeIndex() { + index(id,1, i1, 100,t1,"now is the time for all good men", tag,"patriotic"); + index(id,2, i1, 50 ,t1,"to come to the aid of their country.", tag,"patriotic",tag,"country",tag,"nation",tag,"speeches"); + index(id,3, i1, 2 ,t1,"how now brown cow", tag,"cow",tag,"jersey"); + index(id,4, i1, -100 ,t1,"the quick fox jumped over the lazy dog",tag,"fox",tag,"dog",tag,"quick",tag,"slow",tag,"lazy"); + index(id,5, i1, 50 ,t1,"the quick fox jumped way over the lazy dog",tag,"fox",tag,"dog"); + index(id,6, i1, -60 ,t1,"humpty dumpy sat on a wall",tag,"humpty",tag,"dumpty"); + index(id,7, i1, 123 ,t1,"humpty dumpy had a great fall",tag,"accidents"); + index(id,8, i1, 876 ,t1,"all the kings horses and all the kings men",tag,"king",tag,"horses",tag,"trouble"); + index(id,9, i1, 7 ,t1,"couldn't put humpty together again",tag,"humpty",tag,"broken"); + index(id,10, i1, 4321 ,t1,"this too shall pass",tag,"1",tag,"2",tag,"infinity"); + index(id,11, i1, 33 ,t1,"An eye for eye only ends up making the whole world blind.",tag,"ouch",tag,"eye",tag,"peace",tag,"world"); + index(id,12, i1, 379 ,t1,"Great works are performed, not by strength, but by perseverance.",tag,"herculese",tag,"strong",tag,"stubborn"); + assertU(optimize()); + } + + + /** make sure to close req after you are done using the response */ + public SolrQueryResponse getResponse(SolrQueryRequest req) throws IOException, Exception { + SolrQueryResponse rsp = new SolrQueryResponse(); + h.getCore().execute(h.getCore().getRequestHandler(null),req,rsp); + if (rsp.getException() != null) { + throw rsp.getException(); + } + return rsp; + } + + + void doPerf(String writerName, SolrQueryRequest req, int encIter, int decIter) throws Exception { + SolrQueryResponse rsp = getResponse(req); + QueryResponseWriter w = h.getCore().getQueryResponseWriter(writerName); + + + ByteArrayOutputStream out=null; + + System.gc(); + long start = System.currentTimeMillis(); + for (int i=0; i + diff --git a/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java b/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java index 4beecad549c..c4d8e01d1f8 100644 --- a/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java @@ -148,7 +148,7 @@ public class SolrDispatchFilter implements Filter return mcore; } - + public void destroy() { if (multicore != null) { multicore.shutdown();