Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-4224: Refactor JavaBinCodec input stream definition to enhance reuse. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1491459 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2013-06-10 14:10:27 +00:00
parent 5e5ed64b54
commit 4c57013878
7 changed files with 82 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -159,6 +159,9 @@ Other Changes
* SOLR-4448: Allow the solr internal load balancer to be more easily pluggable. * SOLR-4448: Allow the solr internal load balancer to be more easily pluggable.
(Philip Hoy via Robert Muir) (Philip Hoy via Robert Muir)
* SOLR-4224: Refactor JavaBinCodec input stream definition to enhance reuse.
(phunt via Mark Miller)
================== 4.3.1 ================== ================== 4.3.1 ==================

25
solr/core/src/java/org/apache/solr/update/TransactionLog.java
View File

@ -17,15 +17,6 @@
package org.apache.solr.update; package org.apache.solr.update;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.FastInputStream;
import org.apache.solr.common.util.FastOutputStream;
import org.apache.solr.common.util.JavaBinCodec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
@ -34,15 +25,23 @@ import java.nio.ByteBuffer;
import java.nio.channels.Channels; import java.nio.channels.Channels;
import java.nio.channels.FileChannel; import java.nio.channels.FileChannel;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.DataInputInputStream;
import org.apache.solr.common.util.FastInputStream;
import org.apache.solr.common.util.FastOutputStream;
import org.apache.solr.common.util.JavaBinCodec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* Log Format: List{Operation, Version, ...} * Log Format: List{Operation, Version, ...}
* ADD, VERSION, DOC * ADD, VERSION, DOC
@ -121,7 +120,7 @@ public class TransactionLog {
} }
@Override @Override
public String readExternString(FastInputStream fis) throws IOException { public String readExternString(DataInputInputStream fis) throws IOException {
int idx = readSize(fis); int idx = readSize(fis);
if (idx != 0) {// idx != 0 is the index of the extern string if (idx != 0) {// idx != 0 is the index of the extern string
// no need to synchronize globalStringList - it's only updated before the first record is written to the log // no need to synchronize globalStringList - it's only updated before the first record is written to the log
@ -642,7 +641,7 @@ public class TransactionLog {
ChannelFastInputStream fis; ChannelFastInputStream fis;
private LogCodec codec = new LogCodec() { private LogCodec codec = new LogCodec() {
@Override @Override
public SolrInputDocument readSolrInputDocument(FastInputStream dis) { public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) {
// Given that the SolrInputDocument is last in an add record, it's OK to just skip // Given that the SolrInputDocument is last in an add record, it's OK to just skip
// reading it completely. // reading it completely.
return null; return null;

16
solr/solrj/src/java/org/apache/solr/client/solrj/impl/StreamingBinaryResponseParser.java
View File

@ -16,17 +16,17 @@
*/ */
package org.apache.solr.client.solrj.impl; package org.apache.solr.client.solrj.impl;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import org.apache.solr.client.solrj.StreamingResponseCallback; import org.apache.solr.client.solrj.StreamingResponseCallback;
import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.FastInputStream; import org.apache.solr.common.util.DataInputInputStream;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.JavaBinCodec; import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.common.util.NamedList;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
/** /**
* A BinaryResponseParser that sends callback events rather then build * A BinaryResponseParser that sends callback events rather then build
@ -49,14 +49,14 @@ public class StreamingBinaryResponseParser extends BinaryResponseParser {
JavaBinCodec codec = new JavaBinCodec() { JavaBinCodec codec = new JavaBinCodec() {
@Override @Override
public SolrDocument readSolrDocument(FastInputStream dis) throws IOException { public SolrDocument readSolrDocument(DataInputInputStream dis) throws IOException {
SolrDocument doc = super.readSolrDocument(dis); SolrDocument doc = super.readSolrDocument(dis);
callback.streamSolrDocument( doc ); callback.streamSolrDocument( doc );
return null; return null;
} }
@Override @Override
public SolrDocumentList readSolrDocumentList(FastInputStream dis) throws IOException { public SolrDocumentList readSolrDocumentList(DataInputInputStream dis) throws IOException {
SolrDocumentList solrDocs = new SolrDocumentList(); SolrDocumentList solrDocs = new SolrDocumentList();
List list = (List) readVal(dis); List list = (List) readVal(dis);
solrDocs.setNumFound((Long) list.get(0)); solrDocs.setNumFound((Long) list.get(0));

25
solr/solrj/src/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java
View File

@ -16,17 +16,20 @@
*/ */
package org.apache.solr.client.solrj.request; package org.apache.solr.client.solrj.request;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.FastInputStream;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.common.util.NamedList;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.util.*; import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.DataInputInputStream;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.common.util.NamedList;
/** /**
* Provides methods for marshalling an UpdateRequest to a NamedList which can be serialized in the javabin format and * Provides methods for marshalling an UpdateRequest to a NamedList which can be serialized in the javabin format and
@ -94,7 +97,7 @@ public class JavaBinUpdateRequestCodec {
private boolean seenOuterMostDocIterator = false; private boolean seenOuterMostDocIterator = false;
@Override @Override
public NamedList readNamedList(FastInputStream dis) throws IOException { public NamedList readNamedList(DataInputInputStream dis) throws IOException {
int sz = readSize(dis); int sz = readSize(dis);
NamedList nl = new NamedList(); NamedList nl = new NamedList();
if (namedList[0] == null) { if (namedList[0] == null) {
@ -109,7 +112,7 @@ public class JavaBinUpdateRequestCodec {
} }
@Override @Override
public List readIterator(FastInputStream fis) throws IOException { public List readIterator(DataInputInputStream fis) throws IOException {
// default behavior for reading any regular Iterator in the stream // default behavior for reading any regular Iterator in the stream
if (seenOuterMostDocIterator) return super.readIterator(fis); if (seenOuterMostDocIterator) return super.readIterator(fis);
@ -120,7 +123,7 @@ public class JavaBinUpdateRequestCodec {
return readOuterMostDocIterator(fis); return readOuterMostDocIterator(fis);
} }
private List readOuterMostDocIterator(FastInputStream fis) throws IOException { private List readOuterMostDocIterator(DataInputInputStream fis) throws IOException {
NamedList params = (NamedList) namedList[0].getVal(0); NamedList params = (NamedList) namedList[0].getVal(0);
updateRequest.setParams(new ModifiableSolrParams(SolrParams.toSolrParams(params))); updateRequest.setParams(new ModifiableSolrParams(SolrParams.toSolrParams(params)));
if (handler == null) return super.readIterator(fis); if (handler == null) return super.readIterator(fis);

27
solr/solrj/src/java/org/apache/solr/common/util/DataInputInputStream.java Normal file
View File

@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common.util;
import java.io.DataInput;
import java.io.InputStream;
/**
* An abstract DataInput that extends InputStream
*/
public abstract class DataInputInputStream extends InputStream implements DataInput {
}

2
solr/solrj/src/java/org/apache/solr/common/util/FastInputStream.java
View File

@ -22,7 +22,7 @@ import java.io.*;
/** Single threaded buffered InputStream /** Single threaded buffered InputStream
* Internal Solr use only, subject to change. * Internal Solr use only, subject to change.
*/ */
public class FastInputStream extends InputStream implements DataInput { public class FastInputStream extends DataInputInputStream {
protected final InputStream in; protected final InputStream in;
protected final byte[] buf; protected final byte[] buf;
protected int pos; protected int pos;

34
solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
View File

@ -113,7 +113,7 @@ public class JavaBinCodec {
} }
public SimpleOrderedMap<Object> readOrderedMap(FastInputStream dis) throws IOException { public SimpleOrderedMap<Object> readOrderedMap(DataInputInputStream dis) throws IOException {
int sz = readSize(dis); int sz = readSize(dis);
SimpleOrderedMap<Object> nl = new SimpleOrderedMap<Object>(); SimpleOrderedMap<Object> nl = new SimpleOrderedMap<Object>();
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
@ -124,7 +124,7 @@ public class JavaBinCodec {
return nl; return nl;
} }
public NamedList<Object> readNamedList(FastInputStream dis) throws IOException { public NamedList<Object> readNamedList(DataInputInputStream dis) throws IOException {
int sz = readSize(dis); int sz = readSize(dis);
NamedList<Object> nl = new NamedList<Object>(); NamedList<Object> nl = new NamedList<Object>();
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
@ -164,7 +164,7 @@ public class JavaBinCodec {
protected byte tagByte; protected byte tagByte;
public Object readVal(FastInputStream dis) throws IOException { public Object readVal(DataInputInputStream dis) throws IOException {
tagByte = dis.readByte(); tagByte = dis.readByte();
// if ((tagByte & 0xe0) == 0) { // if ((tagByte & 0xe0) == 0) {
@ -304,7 +304,7 @@ public class JavaBinCodec {
daos.write(arr, offset, len); daos.write(arr, offset, len);
} }
public byte[] readByteArray(FastInputStream dis) throws IOException { public byte[] readByteArray(DataInputInputStream dis) throws IOException {
byte[] arr = new byte[readVInt(dis)]; byte[] arr = new byte[readVInt(dis)];
dis.readFully(arr); dis.readFully(arr);
return arr; return arr;
@ -321,7 +321,7 @@ public class JavaBinCodec {
} }
} }
public SolrDocument readSolrDocument(FastInputStream dis) throws IOException { public SolrDocument readSolrDocument(DataInputInputStream dis) throws IOException {
NamedList nl = (NamedList) readVal(dis); NamedList nl = (NamedList) readVal(dis);
SolrDocument doc = new SolrDocument(); SolrDocument doc = new SolrDocument();
for (int i = 0; i < nl.size(); i++) { for (int i = 0; i < nl.size(); i++) {
@ -332,7 +332,7 @@ public class JavaBinCodec {
return doc; return doc;
} }
public SolrDocumentList readSolrDocumentList(FastInputStream dis) throws IOException { public SolrDocumentList readSolrDocumentList(DataInputInputStream dis) throws IOException {
SolrDocumentList solrDocs = new SolrDocumentList(); SolrDocumentList solrDocs = new SolrDocumentList();
List list = (List) readVal(dis); List list = (List) readVal(dis);
solrDocs.setNumFound((Long) list.get(0)); solrDocs.setNumFound((Long) list.get(0));
@ -356,7 +356,7 @@ public class JavaBinCodec {
writeArray(docs); writeArray(docs);
} }
public SolrInputDocument readSolrInputDocument(FastInputStream dis) throws IOException { public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) throws IOException {
int sz = readVInt(dis); int sz = readVInt(dis);
float docBoost = (Float)readVal(dis); float docBoost = (Float)readVal(dis);
SolrInputDocument sdoc = new SolrInputDocument(); SolrInputDocument sdoc = new SolrInputDocument();
@ -390,7 +390,7 @@ public class JavaBinCodec {
} }
public Map<Object,Object> readMap(FastInputStream dis) public Map<Object,Object> readMap(DataInputInputStream dis)
throws IOException { throws IOException {
int sz = readVInt(dis); int sz = readVInt(dis);
Map<Object,Object> m = new LinkedHashMap<Object,Object>(); Map<Object,Object> m = new LinkedHashMap<Object,Object>();
@ -411,7 +411,7 @@ public class JavaBinCodec {
writeVal(END_OBJ); writeVal(END_OBJ);
} }
public List<Object> readIterator(FastInputStream fis) throws IOException { public List<Object> readIterator(DataInputInputStream fis) throws IOException {
ArrayList<Object> l = new ArrayList<Object>(); ArrayList<Object> l = new ArrayList<Object>();
while (true) { while (true) {
Object o = readVal(fis); Object o = readVal(fis);
@ -444,7 +444,7 @@ public class JavaBinCodec {
} }
} }
public List<Object> readArray(FastInputStream dis) throws IOException { public List<Object> readArray(DataInputInputStream dis) throws IOException {
int sz = readSize(dis); int sz = readSize(dis);
ArrayList<Object> l = new ArrayList<Object>(sz); ArrayList<Object> l = new ArrayList<Object>(sz);
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
@ -473,7 +473,7 @@ public class JavaBinCodec {
byte[] bytes; byte[] bytes;
CharArr arr = new CharArr(); CharArr arr = new CharArr();
public String readStr(FastInputStream dis) throws IOException { public String readStr(DataInputInputStream dis) throws IOException {
int sz = readSize(dis); int sz = readSize(dis);
if (bytes == null || bytes.length < sz) bytes = new byte[sz]; if (bytes == null || bytes.length < sz) bytes = new byte[sz];
dis.readFully(bytes, 0, sz); dis.readFully(bytes, 0, sz);
@ -501,7 +501,7 @@ public class JavaBinCodec {
} }
} }
public int readSmallInt(FastInputStream dis) throws IOException { public int readSmallInt(DataInputInputStream dis) throws IOException {
int v = tagByte & 0x0F; int v = tagByte & 0x0F;
if ((tagByte & 0x10) != 0) if ((tagByte & 0x10) != 0)
v = (readVInt(dis) << 4) | v; v = (readVInt(dis) << 4) | v;
@ -525,7 +525,7 @@ public class JavaBinCodec {
} }
} }
public long readSmallLong(FastInputStream dis) throws IOException { public long readSmallLong(DataInputInputStream dis) throws IOException {
long v = tagByte & 0x0F; long v = tagByte & 0x0F;
if ((tagByte & 0x10) != 0) if ((tagByte & 0x10) != 0)
v = (readVLong(dis) << 4) | v; v = (readVLong(dis) << 4) | v;
@ -607,7 +607,7 @@ public class JavaBinCodec {
} }
public int readSize(FastInputStream in) throws IOException { public int readSize(DataInputInputStream in) throws IOException {
int sz = tagByte & 0x1f; int sz = tagByte & 0x1f;
if (sz == 0x1f) sz += readVInt(in); if (sz == 0x1f) sz += readVInt(in);
return sz; return sz;
@ -634,7 +634,7 @@ public class JavaBinCodec {
* *
* @throws IOException If there is a low-level I/O error. * @throws IOException If there is a low-level I/O error.
*/ */
public static int readVInt(FastInputStream in) throws IOException { public static int readVInt(DataInputInputStream in) throws IOException {
byte b = in.readByte(); byte b = in.readByte();
int i = b & 0x7F; int i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) { for (int shift = 7; (b & 0x80) != 0; shift += 7) {
@ -653,7 +653,7 @@ public class JavaBinCodec {
out.writeByte((byte) i); out.writeByte((byte) i);
} }
public static long readVLong(FastInputStream in) throws IOException { public static long readVLong(DataInputInputStream in) throws IOException {
byte b = in.readByte(); byte b = in.readByte();
long i = b & 0x7F; long i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) { for (int shift = 7; (b & 0x80) != 0; shift += 7) {
@ -683,7 +683,7 @@ public class JavaBinCodec {
} }
public String readExternString(FastInputStream fis) throws IOException { public String readExternString(DataInputInputStream fis) throws IOException {
int idx = readSize(fis); int idx = readSize(fis);
if (idx != 0) {// idx != 0 is the index of the extern string if (idx != 0) {// idx != 0 is the index of the extern string
return stringsList.get(idx - 1); return stringsList.get(idx - 1);