Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-885 -- NamedListCodec is renamed to JavaBinCodec and returns Object instead of NamedList. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@728622 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shalin Shekhar Mangar 2008-12-22 06:46:01 +00:00
parent 1fd7abeb57
commit 32443026f2
9 changed files with 733 additions and 669 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGES.txt
View File

@ -228,6 +228,9 @@ Other Changes
Object, so this doesn't fix any bugs, but helps protect against
future changes. (Kay Kay via hossman)
12. SOLR-885: NamedListCodec is renamed to JavaBinCodec and returns Object instead of NamedList.
(Noble Paul, yonik via shalin)
Build
----------------------
1. SOLR-776: Added in ability to sign artifacts via Ant for releases (gsingers)

702
src/common/org/apache/solr/common/util/JavaBinCodec.java Executable file
View File

@ -0,0 +1,702 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common.util;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.*;
/**
* The class is designed to optimaly serialize/deserialize a NamedList. As we know there are only a limited type of
* items this class can do it with very minimal amount of payload and code. There are 15 known types and if there is an
* object in the object tree which does not fall into these types, It must be converted to one of these. Implement an
* ObjectResolver and pass it over It is expected that this class is used on both end of the pipes. The class has one
* read method and one write method for each of the datatypes
* <p/>
* Note -- Never re-use an instance of this class for more than one marshal or unmarshall operation. Always create a new
* instance.
*/
public class JavaBinCodec {
public static final byte
NULL = 0,
BOOL_TRUE = 1,
BOOL_FALSE = 2,
BYTE = 3,
SHORT = 4,
DOUBLE = 5,
INT = 6,
LONG = 7,
FLOAT = 8,
DATE = 9,
MAP = 10,
SOLRDOC = 11,
SOLRDOCLST = 12,
BYTEARR = 13,
ITERATOR = 14,
/**
* this is a special tag signals an end. No value is associated with it
*/
END = 15,
// types that combine tag + length (or other info) in a single byte
TAG_AND_LEN = (byte) (1 << 5),
STR = (byte) (1 << 5),
SINT = (byte) (2 << 5),
SLONG = (byte) (3 << 5),
ARR = (byte) (4 << 5), //
ORDERED_MAP = (byte) (5 << 5), // SimpleOrderedMap (a NamedList subclass, and more common)
NAMED_LST = (byte) (6 << 5), // NamedList
EXTERN_STRING = (byte) (7 << 5);
private static byte VERSION = 1;
private ObjectResolver resolver;
protected FastOutputStream daos;
public JavaBinCodec() {
}
public JavaBinCodec(ObjectResolver resolver) {
this.resolver = resolver;
}
public void marshal(Object nl, OutputStream os) throws IOException {
daos = FastOutputStream.wrap(os);
try {
daos.writeByte(VERSION);
writeVal(nl);
} finally {
daos.flushBuffer();
}
}
byte version;
public Object unmarshal(InputStream is) throws IOException {
FastInputStream dis = FastInputStream.wrap(is);
version = dis.readByte();
if (version != VERSION) {
throw new RuntimeException("Invalid version or the data in not in 'javabin' format");
}
return (Object) readVal(dis);
}
public SimpleOrderedMap readOrderedMap(FastInputStream dis) throws IOException {
int sz = readSize(dis);
SimpleOrderedMap nl = new SimpleOrderedMap();
for (int i = 0; i < sz; i++) {
String name = (String) readVal(dis);
Object val = readVal(dis);
nl.add(name, val);
}
return nl;
}
public NamedList readNamedList(FastInputStream dis) throws IOException {
int sz = readSize(dis);
NamedList nl = new NamedList();
for (int i = 0; i < sz; i++) {
String name = (String) readVal(dis);
Object val = readVal(dis);
nl.add(name, val);
}
return nl;
}
public void writeNamedList(NamedList nl) throws IOException {
writeTag(nl instanceof SimpleOrderedMap ? ORDERED_MAP : NAMED_LST, nl.size());
for (int i = 0; i < nl.size(); i++) {
String name = nl.getName(i);
writeExternString(name);
Object val = nl.getVal(i);
writeVal(val);
}
}
public void writeVal(Object val) throws IOException {
if (writeKnownType(val)) {
return;
} else {
Object tmpVal = val;
if (resolver != null) {
tmpVal = resolver.resolve(val, this);
if (tmpVal == null) return; // null means the resolver took care of it fully
if (writeKnownType(tmpVal)) return;
}
}
writeVal(val.getClass().getName() + ':' + val.toString());
}
protected static final Object END_OBJ = new Object();
byte tagByte;
public Object readVal(FastInputStream dis) throws IOException {
tagByte = dis.readByte();
// if ((tagByte & 0xe0) == 0) {
// if top 3 bits are clear, this is a normal tag
// OK, try type + size in single byte
switch (tagByte >>> 5) {
case STR >>> 5:
return readStr(dis);
case SINT >>> 5:
return readSmallInt(dis);
case SLONG >>> 5:
return readSmallLong(dis);
case ARR >>> 5:
return readArray(dis);
case ORDERED_MAP >>> 5:
return readOrderedMap(dis);
case NAMED_LST >>> 5:
return readNamedList(dis);
case EXTERN_STRING >>> 5:
return readExternString(dis);
}
switch (tagByte) {
case NULL:
return null;
case DATE:
return new Date(dis.readLong());
case INT:
return dis.readInt();
case BOOL_TRUE:
return Boolean.TRUE;
case BOOL_FALSE:
return Boolean.FALSE;
case FLOAT:
return dis.readFloat();
case DOUBLE:
return dis.readDouble();
case LONG:
return dis.readLong();
case BYTE:
return dis.readByte();
case SHORT:
return dis.readShort();
case MAP:
return readMap(dis);
case SOLRDOC:
return readSolrDocument(dis);
case SOLRDOCLST:
return readSolrDocumentList(dis);
case BYTEARR:
return readByteArray(dis);
case ITERATOR:
return readIterator(dis);
case END:
return END_OBJ;
}
throw new RuntimeException("Unknown type " + tagByte);
}
public boolean writeKnownType(Object val) throws IOException {
if (writePrimitive(val)) return true;
if (val instanceof NamedList) {
writeNamedList((NamedList) val);
return true;
}
if (val instanceof SolrDocumentList) { // SolrDocumentList is a List, so must come before List check
writeSolrDocumentList((SolrDocumentList) val);
return true;
}
if (val instanceof List) {
writeArray((List) val);
return true;
}
if (val instanceof Object[]) {
writeArray((Object[]) val);
return true;
}
if (val instanceof SolrDocument) {
//this needs special treatment to know which fields are to be written
if (resolver == null) {
writeSolrDocument((SolrDocument) val);
} else {
Object retVal = resolver.resolve(val, this);
if (retVal != null) {
if (retVal instanceof SolrDocument) {
writeSolrDocument((SolrDocument) retVal);
} else {
writeVal(retVal);
}
}
}
return true;
}
if (val instanceof Map) {
writeMap((Map) val);
return true;
}
if (val instanceof Iterator) {
writeIterator((Iterator) val);
return true;
}
if (val instanceof Iterable) {
writeIterator(((Iterable) val).iterator());
return true;
}
return false;
}
public void writeTag(byte tag) throws IOException {
daos.writeByte(tag);
}
public void writeTag(byte tag, int size) throws IOException {
if ((tag & 0xe0) != 0) {
if (size < 0x1f) {
daos.writeByte(tag | size);
} else {
daos.writeByte(tag | 0x1f);
writeVInt(size - 0x1f, daos);
}
} else {
daos.writeByte(tag);
writeVInt(size, daos);
}
}
public void writeByteArray(byte[] arr, int offset, int len) throws IOException {
writeTag(BYTEARR, len);
daos.write(arr, offset, len);
}
public byte[] readByteArray(FastInputStream dis) throws IOException {
byte[] arr = new byte[readVInt(dis)];
dis.readFully(arr);
return arr;
}
public void writeSolrDocument(SolrDocument doc) throws IOException {
writeSolrDocument(doc, null);
}
public void writeSolrDocument(SolrDocument doc, Set<String> fields) throws IOException {
int count = 0;
if (fields == null) {
count = doc.getFieldNames().size();
} else {
for (Map.Entry<String, Object> entry : doc) {
if (fields.contains(entry.getKey())) count++;
}
}
writeTag(SOLRDOC);
writeTag(ORDERED_MAP, count);
for (Map.Entry<String, Object> entry : doc) {
if (fields == null || fields.contains(entry.getKey())) {
String name = entry.getKey();
writeExternString(name);
Object val = entry.getValue();
writeVal(val);
}
}
}
public SolrDocument readSolrDocument(FastInputStream dis) throws IOException {
NamedList nl = (NamedList) readVal(dis);
SolrDocument doc = new SolrDocument();
for (int i = 0; i < nl.size(); i++) {
String name = nl.getName(i);
Object val = nl.getVal(i);
doc.setField(name, val);
}
return doc;
}
public SolrDocumentList readSolrDocumentList(FastInputStream dis) throws IOException {
SolrDocumentList solrDocs = new SolrDocumentList();
List list = (List) readVal(dis);
solrDocs.setNumFound((Long) list.get(0));
solrDocs.setStart((Long) list.get(1));
solrDocs.setMaxScore((Float) list.get(2));
List l = (List) readVal(dis);
solrDocs.addAll(l);
return solrDocs;
}
public void writeSolrDocumentList(SolrDocumentList docs)
throws IOException {
writeTag(SOLRDOCLST);
List l = new ArrayList(3);
l.add(docs.getNumFound());
l.add(docs.getStart());
l.add(docs.getMaxScore());
writeArray(l);
writeArray(docs);
}
public Map readMap(FastInputStream dis)
throws IOException {
int sz = readVInt(dis);
Map m = new LinkedHashMap();
for (int i = 0; i < sz; i++) {
Object key = readVal(dis);
Object val = readVal(dis);
m.put(key, val);
}
return m;
}
public void writeIterator(Iterator iter) throws IOException {
writeTag(ITERATOR);
while (iter.hasNext()) {
writeVal(iter.next());
}
writeVal(END_OBJ);
}
public List readIterator(FastInputStream fis) throws IOException {
ArrayList l = new ArrayList();
while (true) {
Object o = readVal(fis);
if (o == END_OBJ) break;
l.add(o);
}
return l;
}
public void writeArray(List l) throws IOException {
writeTag(ARR, l.size());
for (int i = 0; i < l.size(); i++) {
writeVal(l.get(i));
}
}
public void writeArray(Object[] arr) throws IOException {
writeTag(ARR, arr.length);
for (int i = 0; i < arr.length; i++) {
Object o = arr[i];
writeVal(o);
}
}
public List readArray(FastInputStream dis) throws IOException {
int sz = readSize(dis);
ArrayList l = new ArrayList(sz);
for (int i = 0; i < sz; i++) {
l.add(readVal(dis));
}
return l;
}
/**
* write the string as tag+length, with length being the number of UTF-16 characters, followed by the string encoded
* in modified-UTF8
*/
public void writeStr(String s) throws IOException {
if (s == null) {
writeTag(NULL);
return;
}
// Can't use string serialization or toUTF()... it's limited to 64K
// plus it's bigger than it needs to be for small strings anyway
int len = s.length();
writeTag(STR, len);
writeChars(daos, s, 0, len);
}
char[] charArr;
public String readStr(FastInputStream dis) throws IOException {
int sz = readSize(dis);
if (charArr == null || charArr.length < sz) {
charArr = new char[sz];
}
readChars(dis, charArr, 0, sz);
return new String(charArr, 0, sz);
}
public void writeInt(int val) throws IOException {
if (val > 0) {
int b = SINT | (val & 0x0f);
if (val >= 0x0f) {
b |= 0x10;
daos.writeByte(b);
writeVInt(val >>> 4, daos);
} else {
daos.writeByte(b);
}
} else {
daos.writeByte(INT);
daos.writeInt(val);
}
}
public int readSmallInt(FastInputStream dis) throws IOException {
int v = tagByte & 0x0F;
if ((tagByte & 0x10) != 0)
v = (readVInt(dis) << 4) | v;
return v;
}
public void writeLong(long val) throws IOException {
if ((val & 0xff00000000000000L) == 0) {
int b = SLONG | ((int) val & 0x0f);
if (val >= 0x0f) {
b |= 0x10;
daos.writeByte(b);
writeVLong(val >>> 4, daos);
} else {
daos.writeByte(b);
}
} else {
daos.writeByte(LONG);
daos.writeLong(val);
}
}
public long readSmallLong(FastInputStream dis) throws IOException {
long v = tagByte & 0x0F;
if ((tagByte & 0x10) != 0)
v = (readVLong(dis) << 4) | v;
return v;
}
public boolean writePrimitive(Object val) throws IOException {
if (val == null) {
daos.writeByte(NULL);
return true;
} else if (val instanceof String) {
writeStr((String) val);
return true;
} else if (val instanceof Integer) {
writeInt(((Integer) val).intValue());
return true;
} else if (val instanceof Long) {
writeLong(((Long) val).longValue());
return true;
} else if (val instanceof Float) {
daos.writeByte(FLOAT);
daos.writeFloat(((Float) val).floatValue());
return true;
} else if (val instanceof Date) {
daos.writeByte(DATE);
daos.writeLong(((Date) val).getTime());
return true;
} else if (val instanceof Boolean) {
if ((Boolean) val) daos.writeByte(BOOL_TRUE);
else daos.writeByte(BOOL_FALSE);
return true;
} else if (val instanceof Double) {
daos.writeByte(DOUBLE);
daos.writeDouble(((Double) val).doubleValue());
return true;
} else if (val instanceof Byte) {
daos.writeByte(BYTE);
daos.writeByte(((Byte) val).intValue());
return true;
} else if (val instanceof Short) {
daos.writeByte(SHORT);
daos.writeShort(((Short) val).intValue());
return true;
} else if (val instanceof byte[]) {
writeByteArray((byte[]) val, 0, ((byte[]) val).length);
return true;
} else if (val == END_OBJ) {
writeTag(END);
return true;
}
return false;
}
public void writeMap(Map val)
throws IOException {
writeTag(MAP, val.size());
for (Map.Entry entry : (Set<Map.Entry>) val.entrySet()) {
Object key = entry.getKey();
if (key instanceof String) {
writeExternString((String) key);
} else {
writeVal(key);
}
writeVal(entry.getValue());
}
}
public int readSize(FastInputStream in) throws IOException {
int sz = tagByte & 0x1f;
if (sz == 0x1f) sz += readVInt(in);
return sz;
}
/**
* Special method for variable length int (copied from lucene). Usually used for writing the length of a
* collection/array/map In most of the cases the length can be represented in one byte (length < 127) so it saves 3
* bytes/object
*
* @param i
* @param out
*
* @throws IOException
*/
public static void writeVInt(int i, FastOutputStream out) throws IOException {
while ((i & ~0x7F) != 0) {
out.writeByte((byte) ((i & 0x7f) | 0x80));
i >>>= 7;
}
out.writeByte((byte) i);
}
/**
* The counterpart for the above
*
* @param in
*
* @return the int value
*
* @throws IOException
*/
public static int readVInt(FastInputStream in) throws IOException {
byte b = in.readByte();
int i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = in.readByte();
i |= (b & 0x7F) << shift;
}
return i;
}
public static void writeVLong(long i, FastOutputStream out) throws IOException {
while ((i & ~0x7F) != 0) {
out.writeByte((byte) ((i & 0x7f) | 0x80));
i >>>= 7;
}
out.writeByte((byte) i);
}
public static long readVLong(FastInputStream in) throws IOException {
byte b = in.readByte();
long i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = in.readByte();
i |= (long) (b & 0x7F) << shift;
}
return i;
}
/**
* Writes a sequence of UTF-8 encoded characters from a string.
*
* @param s the source of the characters
* @param start the first character in the sequence
* @param length the number of characters in the sequence
*
* @see org.apache.lucene.store.IndexInput#readChars(char[],int,int)
*/
public static void writeChars(FastOutputStream os, String s, int start, int length)
throws IOException {
final int end = start + length;
for (int i = start; i < end; i++) {
final int code = (int) s.charAt(i);
if (code >= 0x01 && code <= 0x7F)
os.write(code);
else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
os.write(0xC0 | (code >> 6));
os.write(0x80 | (code & 0x3F));
} else {
os.write(0xE0 | (code >>> 12));
os.write(0x80 | ((code >> 6) & 0x3F));
os.write(0x80 | (code & 0x3F));
}
}
}
/**
* Reads UTF-8 encoded characters into an array.
*
* @param buffer the array to read characters into
* @param start the offset in the array to start storing characters
* @param length the number of characters to read
*
* @see org.apache.lucene.store.IndexOutput#writeChars(String,int,int)
*/
public static void readChars(FastInputStream in, char[] buffer, int start, int length)
throws IOException {
final int end = start + length;
for (int i = start; i < end; i++) {
int b = in.read();
if ((b & 0x80) == 0)
buffer[i] = (char) b;
else if ((b & 0xE0) != 0xE0) {
buffer[i] = (char) (((b & 0x1F) << 6)
| (in.read() & 0x3F));
} else
buffer[i] = (char) (((b & 0x0F) << 12)
| ((in.read() & 0x3F) << 6)
| (in.read() & 0x3F));
}
}
private int stringsCount = 0;
private Map<String, Integer> stringsMap;
private List<String> stringsList;
public void writeExternString(String s) throws IOException {
if (s == null) {
writeTag(NULL);
return;
}
Integer idx = stringsMap == null ? null : stringsMap.get(s);
if (idx == null) idx = 0;
writeTag(EXTERN_STRING, idx);
if (idx == 0) {
writeStr(s);
if (stringsMap == null) stringsMap = new HashMap<String, Integer>();
stringsMap.put(s, ++stringsCount);
}
}
public String readExternString(FastInputStream fis) throws IOException {
int idx = readSize(fis);
if (idx != 0) {// idx != 0 is the index of the extern string
return stringsList.get(idx - 1);
} else {// idx == 0 means it has a string value
String s = (String) readVal(fis);
if (stringsList == null) stringsList = new ArrayList<String>();
stringsList.add(s);
return s;
}
}
public static interface ObjectResolver {
public Object resolve(Object o, JavaBinCodec codec) throws IOException;
}
}

641
src/common/org/apache/solr/common/util/NamedListCodec.java
View File

@ -16,644 +16,9 @@
*/
package org.apache.solr.common.util;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import java.io.*;
import java.util.*;
/**
* The class is designed to optimaly serialize/deserialize a NamedList. As we know there are only
* a limited type of items this class can do it with very minimal amount of payload and code. There are
* 15 known types and if there is an object in the object tree which does not fall into these types, It must be
* converted to one of these. Implement an ObjectResolver and pass it over
* It is expected that this class is used on both end of the pipes.
* The class has one read method and one write method for each of the datatypes
*
* Note -- Never re-use an instance of this class for more than one marshal or unmarshall operation.
* Always create a new instance.
*
* @deprecated use JavaBinCodec instead
*/
public class NamedListCodec {
public static final byte
NULL = 0,
BOOL_TRUE = 1,
BOOL_FALSE = 2,
BYTE = 3,
SHORT = 4,
DOUBLE = 5,
INT = 6,
LONG = 7,
FLOAT = 8,
DATE = 9,
MAP = 10,
SOLRDOC = 11,
SOLRDOCLST = 12,
BYTEARR = 13,
ITERATOR = 14,
/** this is a special tag signals an end. No value is associated with it*/
END = 15,
// types that combine tag + length (or other info) in a single byte
TAG_AND_LEN=(byte)(1 << 5),
STR = (byte)(1 << 5),
SINT = (byte)(2 << 5),
SLONG = (byte)(3 << 5),
ARR = (byte)(4 << 5), //
ORDERED_MAP=(byte)(5 << 5), // SimpleOrderedMap (a NamedList subclass, and more common)
NAMED_LST = (byte)(6 << 5), // NamedList
EXTERN_STRING = (byte)(7 << 5);
private static byte VERSION = 1;
private ObjectResolver resolver;
protected FastOutputStream daos;
public NamedListCodec() { }
public NamedListCodec(ObjectResolver resolver) {
this.resolver = resolver;
}
public void marshal(Object nl, OutputStream os) throws IOException {
daos = FastOutputStream.wrap(os);
try {
daos.writeByte(VERSION);
writeVal(nl);
} finally {
daos.flushBuffer();
}
}
byte version;
public Object unmarshal(InputStream is) throws IOException {
FastInputStream dis = FastInputStream.wrap(is);
version = dis.readByte();
if(version != VERSION){
throw new RuntimeException("Invalid version or the data in not in 'javabin' format");
}
return (Object)readVal(dis);
}
public SimpleOrderedMap readOrderedMap(FastInputStream dis) throws IOException {
int sz = readSize(dis);
SimpleOrderedMap nl = new SimpleOrderedMap();
for (int i = 0; i < sz; i++) {
String name = (String)readVal(dis);
Object val = readVal(dis);
nl.add(name, val);
}
return nl;
}
public NamedList readNamedList(FastInputStream dis) throws IOException {
int sz = readSize(dis);
NamedList nl = new NamedList();
for (int i = 0; i < sz; i++) {
String name = (String)readVal(dis);
Object val = readVal(dis);
nl.add(name, val);
}
return nl;
}
public void writeNamedList(NamedList nl) throws IOException {
writeTag(nl instanceof SimpleOrderedMap ? ORDERED_MAP : NAMED_LST, nl.size());
for (int i = 0; i < nl.size(); i++) {
String name = nl.getName(i);
writeExternString(name);
Object val = nl.getVal(i);
writeVal(val);
}
}
public void writeVal(Object val) throws IOException {
if (writeKnownType(val)) {
return;
} else {
Object tmpVal = val;
if(resolver !=null) {
tmpVal = resolver.resolve(val, this);
if (tmpVal == null) return; // null means the resolver took care of it fully
if(writeKnownType(tmpVal)) return;
}
}
writeVal(val.getClass().getName() + ':' + val.toString());
}
protected static final Object END_OBJ = new Object();
byte tagByte;
public Object readVal(FastInputStream dis) throws IOException {
tagByte = dis.readByte();
// if ((tagByte & 0xe0) == 0) {
// if top 3 bits are clear, this is a normal tag
// OK, try type + size in single byte
switch(tagByte>>>5) {
case STR >>> 5 : return readStr(dis);
case SINT >>>5 : return readSmallInt(dis);
case SLONG >>>5 : return readSmallLong(dis);
case ARR >>> 5 : return readArray(dis);
case ORDERED_MAP >>> 5 : return readOrderedMap(dis);
case NAMED_LST >>> 5 : return readNamedList(dis);
case EXTERN_STRING >>> 5 : return readExternString(dis);
}
switch(tagByte){
case NULL : return null;
case DATE : return new Date(dis.readLong());
case INT : return dis.readInt();
case BOOL_TRUE : return Boolean.TRUE;
case BOOL_FALSE : return Boolean.FALSE;
case FLOAT : return dis.readFloat();
case DOUBLE : return dis.readDouble();
case LONG : return dis.readLong();
case BYTE : return dis.readByte();
case SHORT : return dis.readShort();
case MAP : return readMap(dis);
case SOLRDOC : return readSolrDocument(dis);
case SOLRDOCLST : return readSolrDocumentList(dis);
case BYTEARR : return readByteArray(dis);
case ITERATOR : return readIterator(dis);
case END : return END_OBJ;
}
throw new RuntimeException("Unknown type " + tagByte);
}
public boolean writeKnownType(Object val) throws IOException {
if (writePrimitive(val)) return true;
if (val instanceof NamedList) {
writeNamedList((NamedList) val);
return true;
}
if (val instanceof SolrDocumentList) { // SolrDocumentList is a List, so must come before List check
writeSolrDocumentList((SolrDocumentList) val);
return true;
}
if (val instanceof List) {
writeArray((List) val);
return true;
}
if (val instanceof Object[]) {
writeArray((Object[]) val);
return true;
}
if (val instanceof SolrDocument) {
//this needs special treatment to know which fields are to be written
if(resolver == null){
writeSolrDocument((SolrDocument) val);
}else {
Object retVal = resolver.resolve(val, this);
if(retVal != null) {
if (retVal instanceof SolrDocument) {
writeSolrDocument((SolrDocument) retVal);
} else {
writeVal(retVal);
}
}
}
return true;
}
if (val instanceof Map) {
writeMap((Map) val);
return true;
}
if (val instanceof Iterator) {
writeIterator((Iterator) val);
return true;
}
if (val instanceof Iterable) {
writeIterator(((Iterable)val).iterator());
return true;
}
return false;
}
public void writeTag(byte tag) throws IOException {
daos.writeByte(tag);
}
public void writeTag(byte tag, int size) throws IOException {
if ((tag & 0xe0) != 0) {
if (size < 0x1f) {
daos.writeByte(tag | size);
} else {
daos.writeByte(tag | 0x1f);
writeVInt(size-0x1f, daos);
}
} else {
daos.writeByte(tag);
writeVInt(size, daos);
}
}
public void writeByteArray(byte[] arr, int offset, int len) throws IOException {
writeTag(BYTEARR, len);
daos.write(arr, offset, len);
}
public byte[] readByteArray(FastInputStream dis) throws IOException {
byte[] arr = new byte[readVInt(dis)];
dis.readFully(arr);
return arr;
}
public void writeSolrDocument(SolrDocument doc) throws IOException {
writeSolrDocument(doc, null);
}
public void writeSolrDocument(SolrDocument doc, Set<String> fields) throws IOException {
int count = 0;
if (fields == null) {
count = doc.getFieldNames().size();
} else {
for (Map.Entry<String, Object> entry : doc) {
if (fields.contains(entry.getKey())) count++;
}
}
writeTag(SOLRDOC);
writeTag(ORDERED_MAP, count);
for (Map.Entry<String, Object> entry : doc) {
if (fields == null || fields.contains(entry.getKey())) {
String name = entry.getKey();
writeExternString(name);
Object val = entry.getValue();
writeVal(val);
}
}
}
public SolrDocument readSolrDocument(FastInputStream dis) throws IOException {
NamedList nl = (NamedList) readVal(dis);
SolrDocument doc = new SolrDocument();
for (int i = 0; i < nl.size(); i++) {
String name = nl.getName(i);
Object val = nl.getVal(i);
doc.setField(name, val);
}
return doc;
}
public SolrDocumentList readSolrDocumentList(FastInputStream dis) throws IOException {
SolrDocumentList solrDocs = new SolrDocumentList();
List list = (List) readVal(dis);
solrDocs.setNumFound((Long) list.get(0));
solrDocs.setStart((Long)list.get(1));
solrDocs.setMaxScore((Float)list.get(2));
List l = (List) readVal(dis);
solrDocs.addAll(l);
return solrDocs;
}
public void writeSolrDocumentList(SolrDocumentList docs)
throws IOException {
writeTag(SOLRDOCLST);
List l = new ArrayList(3);
l.add(docs.getNumFound());
l.add(docs.getStart());
l.add(docs.getMaxScore());
writeArray(l);
writeArray(docs);
}
public Map readMap(FastInputStream dis)
throws IOException {
int sz = readVInt(dis);
Map m = new LinkedHashMap();
for (int i = 0; i < sz; i++) {
Object key = readVal(dis);
Object val = readVal(dis);
m.put(key, val);
}
return m;
}
public void writeIterator(Iterator iter) throws IOException {
writeTag(ITERATOR);
while (iter.hasNext()) {
writeVal(iter.next());
}
writeVal(END_OBJ);
}
public List readIterator(FastInputStream fis) throws IOException {
ArrayList l = new ArrayList();
while(true){
Object o = readVal(fis);
if(o == END_OBJ) break;
l.add(o);
}
return l;
}
public void writeArray(List l) throws IOException {
writeTag(ARR, l.size());
for (int i = 0; i < l.size(); i++) {
writeVal(l.get(i));
}
}
public void writeArray(Object[] arr) throws IOException {
writeTag(ARR, arr.length);
for (int i = 0; i < arr.length; i++) {
Object o = arr[i];
writeVal(o);
}
}
public List readArray(FastInputStream dis) throws IOException {
int sz = readSize(dis);
ArrayList l = new ArrayList(sz);
for (int i = 0; i < sz; i++) {
l.add(readVal(dis));
}
return l;
}
/** write the string as tag+length, with length being the number of UTF-16 characters,
* followed by the string encoded in modified-UTF8
*/
public void writeStr(String s) throws IOException {
if (s==null) {
writeTag(NULL);
return;
}
// Can't use string serialization or toUTF()... it's limited to 64K
// plus it's bigger than it needs to be for small strings anyway
int len = s.length();
writeTag(STR, len);
writeChars(daos, s, 0, len);
}
char[] charArr;
public String readStr(FastInputStream dis) throws IOException {
int sz = readSize(dis);
if (charArr==null || charArr.length < sz) {
charArr = new char[sz];
}
readChars(dis, charArr, 0, sz);
return new String(charArr, 0, sz);
}
public void writeInt(int val) throws IOException {
if (val>0) {
int b = SINT | (val & 0x0f);
if (val >= 0x0f) {
b |= 0x10;
daos.writeByte(b);
writeVInt(val>>>4, daos);
} else {
daos.writeByte(b);
}
} else {
daos.writeByte(INT);
daos.writeInt(val);
}
}
public int readSmallInt(FastInputStream dis) throws IOException {
int v = tagByte & 0x0F;
if ((tagByte & 0x10) != 0)
v = (readVInt(dis)<<4) | v;
return v;
}
public void writeLong(long val) throws IOException {
if ((val & 0xff00000000000000L) == 0) {
int b = SLONG | ((int)val & 0x0f);
if (val >= 0x0f) {
b |= 0x10;
daos.writeByte(b);
writeVLong(val>>>4, daos);
} else {
daos.writeByte(b);
}
} else {
daos.writeByte(LONG);
daos.writeLong(val);
}
}
public long readSmallLong(FastInputStream dis) throws IOException {
long v = tagByte & 0x0F;
if ((tagByte & 0x10) != 0)
v = (readVLong(dis)<<4) | v;
return v;
}
public boolean writePrimitive(Object val) throws IOException {
if (val == null) {
daos.writeByte(NULL);
return true;
} else if (val instanceof String) {
writeStr((String)val);
return true;
} else if (val instanceof Integer) {
writeInt(((Integer)val).intValue());
return true;
} else if (val instanceof Long) {
writeLong(((Long)val).longValue());
return true;
} else if (val instanceof Float) {
daos.writeByte(FLOAT);
daos.writeFloat(((Float) val).floatValue());
return true;
} else if (val instanceof Date) {
daos.writeByte(DATE);
daos.writeLong(((Date) val).getTime());
return true;
} else if (val instanceof Boolean) {
if ((Boolean) val) daos.writeByte(BOOL_TRUE);
else daos.writeByte(BOOL_FALSE);
return true;
} else if (val instanceof Double) {
daos.writeByte(DOUBLE);
daos.writeDouble(((Double) val).doubleValue());
return true;
} else if (val instanceof Byte) {
daos.writeByte(BYTE);
daos.writeByte(((Byte) val).intValue());
return true;
} else if (val instanceof Short) {
daos.writeByte(SHORT);
daos.writeShort(((Short) val).intValue());
return true;
} else if (val instanceof byte[]) {
writeByteArray((byte[])val, 0, ((byte[])val).length);
return true;
} else if (val == END_OBJ) {
writeTag(END);
return true;
}
return false;
}
public void writeMap( Map val)
throws IOException {
writeTag(MAP, val.size());
for (Map.Entry entry : (Set<Map.Entry>) val.entrySet()) {
Object key = entry.getKey();
if (key instanceof String) {
writeExternString((String) key);
} else {
writeVal(key);
}
writeVal(entry.getValue());
}
}
public int readSize(FastInputStream in) throws IOException {
int sz = tagByte & 0x1f;
if (sz == 0x1f) sz += readVInt(in);
return sz;
}
/**
* Special method for variable length int (copied from lucene). Usually used for writing the length of a collection/array/map
* In most of the cases the length can be represented in one byte (length < 127) so it saves 3 bytes/object
*
* @param i
* @param out
* @throws IOException
*/
public static void writeVInt(int i, FastOutputStream out) throws IOException {
while ((i & ~0x7F) != 0) {
out.writeByte((byte) ((i & 0x7f) | 0x80));
i >>>= 7;
}
out.writeByte((byte) i);
}
/**
* The counterpart for the above
*
* @param in
* @return the int value
* @throws IOException
*/
public static int readVInt(FastInputStream in) throws IOException {
byte b = in.readByte();
int i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = in.readByte();
i |= (b & 0x7F) << shift;
}
return i;
}
public static void writeVLong(long i, FastOutputStream out) throws IOException {
while ((i & ~0x7F) != 0) {
out.writeByte((byte)((i & 0x7f) | 0x80));
i >>>= 7;
}
out.writeByte((byte) i);
}
public static long readVLong(FastInputStream in) throws IOException {
byte b = in.readByte();
long i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = in.readByte();
i |= (long)(b & 0x7F) << shift;
}
return i;
}
/** Writes a sequence of UTF-8 encoded characters from a string.
* @param s the source of the characters
* @param start the first character in the sequence
* @param length the number of characters in the sequence
* @see org.apache.lucene.store.IndexInput#readChars(char[],int,int)
*/
public static void writeChars(FastOutputStream os, String s, int start, int length)
throws IOException {
final int end = start + length;
for (int i = start; i < end; i++) {
final int code = (int)s.charAt(i);
if (code >= 0x01 && code <= 0x7F)
os.write(code);
else if (((code >= 0x80) && (code <= 0x7FF)) || code == 0) {
os.write(0xC0 | (code >> 6));
os.write(0x80 | (code & 0x3F));
} else {
os.write(0xE0 | (code >>> 12));
os.write(0x80 | ((code >> 6) & 0x3F));
os.write(0x80 | (code & 0x3F));
}
}
}
/** Reads UTF-8 encoded characters into an array.
* @param buffer the array to read characters into
* @param start the offset in the array to start storing characters
* @param length the number of characters to read
* @see org.apache.lucene.store.IndexOutput#writeChars(String,int,int)
*/
public static void readChars(FastInputStream in, char[] buffer, int start, int length)
throws IOException {
final int end = start + length;
for (int i = start; i < end; i++) {
int b = in.read();
if ((b & 0x80) == 0)
buffer[i] = (char)b;
else if ((b & 0xE0) != 0xE0) {
buffer[i] = (char)(((b & 0x1F) << 6)
| (in.read() & 0x3F));
} else
buffer[i] = (char)(((b & 0x0F) << 12)
| ((in.read() & 0x3F) << 6)
| (in.read() & 0x3F));
}
}
private int stringsCount = 0;
private Map<String,Integer> stringsMap;
private List<String > stringsList;
public void writeExternString(String s) throws IOException {
if(s == null) {
writeTag(NULL) ;
return;
}
Integer idx = stringsMap == null ? null : stringsMap.get(s);
if(idx == null) idx =0;
writeTag(EXTERN_STRING,idx);
if(idx == 0){
writeStr(s);
if(stringsMap == null) stringsMap = new HashMap<String, Integer>();
stringsMap.put(s,++stringsCount);
}
}
public String readExternString(FastInputStream fis) throws IOException {
int idx = readSize(fis);
if (idx != 0) {// idx != 0 is the index of the extern string
return stringsList.get(idx-1);
} else {// idx == 0 means it has a string value
String s = (String) readVal(fis);
if(stringsList == null ) stringsList = new ArrayList<String>();
stringsList.add(s);
return s;
}
}
public static interface ObjectResolver{
public Object resolve(Object o, NamedListCodec codec) throws IOException;
}
@Deprecated
public class NamedListCodec extends JavaBinCodec {
}

4
src/java/org/apache/solr/handler/SnapPuller.java
View File

@ -22,7 +22,7 @@ import org.apache.lucene.index.IndexCommit;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.FastInputStream;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.NamedListCodec;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.core.SolrCore;
import static org.apache.solr.handler.ReplicationHandler.*;
import org.apache.solr.search.SolrIndexSearcher;
@ -171,7 +171,7 @@ public class SnapPuller {
throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
"Request failed for the url " + method);
}
return (NamedList) new NamedListCodec().unmarshal(method.getResponseBodyAsStream());
return (NamedList) new JavaBinCodec().unmarshal(method.getResponseBodyAsStream());
} finally {
try {
method.releaseConnection();

20
src/java/org/apache/solr/request/BinaryResponseWriter.java
View File

@ -21,7 +21,7 @@ import org.apache.lucene.document.Fieldable;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.NamedListCodec;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.schema.*;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
@ -44,7 +44,7 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter {
Resolver resolver = new Resolver(req, response.getReturnFields());
Boolean omitHeader = req.getParams().getBool(CommonParams.OMIT_HEADER);
if (omitHeader != null && omitHeader) response.getValues().remove("responseHeader");
NamedListCodec codec = new NamedListCodec(resolver);
JavaBinCodec codec = new JavaBinCodec(resolver);
codec.marshal(response.getValues(), out);
}
@ -60,7 +60,7 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter {
/* NOOP */
}
private static class Resolver implements NamedListCodec.ObjectResolver {
private static class Resolver implements JavaBinCodec.ObjectResolver {
private final IndexSchema schema;
private final SolrIndexSearcher searcher;
private final Set<String> returnFields;
@ -83,7 +83,7 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter {
this.returnFields = returnFields;
}
public Object resolve(Object o, NamedListCodec codec) throws IOException {
public Object resolve(Object o, JavaBinCodec codec) throws IOException {
if (o instanceof DocList) {
writeDocList((DocList) o, codec);
return null; // null means we completely handled it
@ -100,8 +100,8 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter {
return o;
}
public void writeDocList(DocList ids, NamedListCodec codec) throws IOException {
codec.writeTag(NamedListCodec.SOLRDOCLST);
public void writeDocList(DocList ids, JavaBinCodec codec) throws IOException {
codec.writeTag(JavaBinCodec.SOLRDOCLST);
List l = new ArrayList(3);
l.add((long) ids.matches());
l.add((long) ids.offset());
@ -113,7 +113,7 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter {
codec.writeArray(l);
int sz = ids.size();
codec.writeTag(NamedListCodec.ARR, sz);
codec.writeTag(JavaBinCodec.ARR, sz);
DocIterator iterator = ids.iterator();
for (int i = 0; i < sz; i++) {
@ -181,10 +181,10 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter {
Resolver resolver = new Resolver(req, rsp.getReturnFields());
ByteArrayOutputStream out = new ByteArrayOutputStream();
new NamedListCodec(resolver).marshal(rsp.getValues(), out);
new JavaBinCodec(resolver).marshal(rsp.getValues(), out);
InputStream in = new ByteArrayInputStream(out.toByteArray());
return (NamedList<Object>) new NamedListCodec(resolver).unmarshal(in);
return (NamedList<Object>) new JavaBinCodec(resolver).unmarshal(in);
}
catch (Exception ex) {
throw new RuntimeException(ex);
@ -209,7 +209,7 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter {
KNOWN_TYPES.add(SortableDoubleField.class);
KNOWN_TYPES.add(StrField.class);
KNOWN_TYPES.add(TextField.class);
// We do not add UUIDField because UUID object is not a supported type in NamedListCodec
// We do not add UUIDField because UUID object is not a supported type in JavaBinCodec
// and if we write UUIDField.toObject, we wouldn't know how to handle it in the client side
}
}

4
src/solrj/org/apache/solr/client/solrj/impl/BinaryResponseParser.java
View File

@ -19,7 +19,7 @@ package org.apache.solr.client.solrj.impl;
import org.apache.solr.client.solrj.ResponseParser;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.NamedListCodec;
import org.apache.solr.common.util.JavaBinCodec;
import java.io.IOException;
import java.io.InputStream;
@ -36,7 +36,7 @@ public class BinaryResponseParser extends ResponseParser {
public NamedList<Object> processResponse(InputStream body, String encoding) {
try {
return (NamedList<Object>) new NamedListCodec().unmarshal(body);
return (NamedList<Object>) new JavaBinCodec().unmarshal(body);
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "parsing error", e);

7
src/test/org/apache/solr/TestDistributedSearch.java
View File

@ -18,20 +18,15 @@
package org.apache.solr;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.util.AbstractSolrTestCase;
import java.io.File;
@ -495,7 +490,7 @@ public class TestDistributedSearch extends TestCase {
query("q","*:*", "sort",i1+" desc");
query("q","*:*", "sort",i1+" desc", "fl","*,score");
handle.put("maxScore", SKIPVAL);
query("q","{!func}"+i1);// does not expect maxScore. So if it comes ,ignore it. NamedListCodec.writeSolrDocumentList()
query("q","{!func}"+i1);// does not expect maxScore. So if it comes ,ignore it. JavaBinCodec.writeSolrDocumentList()
//is agnostic of request params.
handle.remove("maxScore");
query("q","{!func}"+i1, "fl","*,score"); // even scores should match exactly here

17
src/test/org/apache/solr/common/util/TestNamedListCodec.java
View File

@ -22,7 +22,6 @@ import org.apache.solr.TestDistributedSearch;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.List;
import java.util.Random;
import java.util.ArrayList;
@ -77,9 +76,9 @@ public class TestNamedListCodec extends TestCase {
nl.add("zzz",doc);
new NamedListCodec(null).marshal(nl,baos);
new JavaBinCodec(null).marshal(nl,baos);
byte[] arr = baos.toByteArray();
nl = (NamedList) new NamedListCodec().unmarshal(new ByteArrayInputStream(arr));
nl = (NamedList) new JavaBinCodec().unmarshal(new ByteArrayInputStream(arr));
assertEquals(3, nl.size());
@ -117,9 +116,9 @@ public class TestNamedListCodec extends TestCase {
nl.add("zzz",list.iterator());
new NamedListCodec(null).marshal(nl,baos);
new JavaBinCodec(null).marshal(nl,baos);
byte[] arr = baos.toByteArray();
nl = (NamedList) new NamedListCodec().unmarshal(new ByteArrayInputStream(arr));
nl = (NamedList) new JavaBinCodec().unmarshal(new ByteArrayInputStream(arr));
List l = (List) nl.get("zzz");
assertEquals(list.size(), l.size());
@ -139,11 +138,11 @@ public class TestNamedListCodec extends TestCase {
r.add("more", "less");
r.add("values", map.values());
r.add("finally", "the end");
new NamedListCodec(null).marshal(r,baos);
new JavaBinCodec(null).marshal(r,baos);
byte[] arr = baos.toByteArray();
try {
NamedList result = (NamedList) new NamedListCodec().unmarshal(new ByteArrayInputStream(arr));
NamedList result = (NamedList) new JavaBinCodec().unmarshal(new ByteArrayInputStream(arr));
assertTrue("result is null and it shouldn't be", result != null);
List keys = (List) result.get("keys");
assertTrue("keys is null and it shouldn't be", keys != null);
@ -244,10 +243,10 @@ public class TestNamedListCodec extends TestCase {
for (int i=0; i<10000; i++) { // pump up the iterations for good stress testing
nl = rNamedList(3);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
new NamedListCodec(null).marshal(nl,baos);
new JavaBinCodec(null).marshal(nl,baos);
byte[] arr = baos.toByteArray();
// System.out.println(arr.length);
res = (NamedList) new NamedListCodec().unmarshal(new ByteArrayInputStream(arr));
res = (NamedList) new JavaBinCodec().unmarshal(new ByteArrayInputStream(arr));
cmp = TestDistributedSearch.compare(nl,res, 0, null);
if (cmp != null) {

4
src/test/org/apache/solr/request/TestBinaryResponseWriter.java
View File

@ -20,7 +20,7 @@ import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.NamedListCodec;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.util.AbstractSolrTestCase;
import java.io.ByteArrayInputStream;
@ -55,7 +55,7 @@ public class TestBinaryResponseWriter extends AbstractSolrTestCase {
BinaryQueryResponseWriter writer = (BinaryQueryResponseWriter) h.getCore().getQueryResponseWriter("javabin");
ByteArrayOutputStream baos = new ByteArrayOutputStream();
writer.write(baos, req, rsp);
NamedList res = (NamedList) new NamedListCodec().unmarshal(new ByteArrayInputStream(baos.toByteArray()));
NamedList res = (NamedList) new JavaBinCodec().unmarshal(new ByteArrayInputStream(baos.toByteArray()));
SolrDocumentList docs = (SolrDocumentList) res.get("response");
for (Object doc : docs) {
SolrDocument document = (SolrDocument) doc;